Files
Xiaoze Fan 27740fec66 feat(kvcache): add fp8 and int8 per-token-head KV cache formats (#894)
* feat(kvcache): add fp8 and int8 per-token-head KV cache formats

* fix(kernel): keep the bf16 m3 sparse k/v loads inline

* fix(attention): refuse a backend that cannot read the kv format

- filter the m3 page-size fallback by kv format
- release the scale buffer when a qsa or bsa rebuild fails
- keep the bf16 qsa and m3 attend code identical to main
- drop docstrings that claimed no register spills
- drop guard tests

* fix(kernel): widen dsa latent row offsets past int32

* feat(spec): store the mtp draft kv in the target's format
2026-11-11 15:47:50 -08:00

293 lines
12 KiB
Python

"""MLA latent-KV pools (GLM-5.2 / DeepSeek-style latent attention).
``MLAKVCache`` is the MHA pool's sibling for latent-KV models: ONE slab holding the
per-token latent ``ckv (kv_lora_rank) | kpe (qk_rope_head_dim)`` -- there is no
separate V (``v_cache`` aliases ``k_cache``, same convention as dsv4/v4_pool's
single-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attention
index-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer
layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1;
``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild``
resizes ALL slabs atomically so the allocator can never hand out a slot one slab
has and the other lacks.
Storage lives here -- not in the attention backend -- so the engine's rebuild path
(``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views
re-derived by callers per forward) and the KV cost model (which budgets the index-K
bytes off the attention-group spec) stay correct by construction.
"""
from __future__ import annotations
from typing import ClassVar
import torch
from .base import BaseKVCachePool
from .format import NONE, ROW_SCALE_FORMATS, KVFormat
class MLAKVCache(BaseKVCachePool):
"""Paged latent-KV pool: ``[1, num_layers, num_pages, page_size, 1, latent_dim]``.
The leading singleton keeps the buffer shape-compatible with MHAKVCache's
(tokens = shape[2] * shape[3]).
``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x
MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap
contract as MHAKVCache; None keeps the identity mapping.
A quantized row keeps two scales: ckv and kpe have their own ranges.
"""
kv_formats: ClassVar[frozenset[str]] = ROW_SCALE_FORMATS
def __init__(
self,
latent_dim: int,
num_layers: int,
num_pages: int,
page_size: int,
dtype: torch.dtype,
device: torch.device,
layer_ids: "tuple[int, ...] | None" = None,
kv_format: KVFormat = NONE,
) -> None:
self.kv_format = kv_format
self._latent_dim = latent_dim
if layer_ids is None:
self._num_layers = num_layers
self._layer_index: dict[int, int] | None = None
else:
self._num_layers = len(layer_ids)
self._layer_index = {int(g): i for i, g in enumerate(layer_ids)}
self._page_size = page_size
self._dtype = dtype
self._device = device
self._alloc(num_pages)
def _local_layer(self, layer_id: int) -> int:
return layer_id if self._layer_index is None else self._layer_index[layer_id]
def _alloc(self, num_pages: int) -> None:
self._num_pages = num_pages
fmt = self.kv_format
new = torch.zeros if fmt.quantized else torch.empty
self._kv_buffer = new(
(1, self._num_layers, num_pages, self._page_size, 1, fmt.elem_dim(self._latent_dim)),
device=self._device,
dtype=fmt.elem_tensor_dtype(self._dtype),
)
self._scale_buffer = None
if fmt.scale_dtype is not None:
self._scale_buffer = torch.zeros(
(self._num_layers, num_pages * self._page_size, 2),
device=self._device,
dtype=fmt.scale_store_dtype,
)
# -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) --
def k_cache(self, layer_id: int) -> torch.Tensor:
"""Paged latent view ``[num_pages, page_size, latent_dim]``."""
return self._kv_buffer[0, self._local_layer(layer_id)].view(
self._num_pages, self._page_size, -1
)
def v_cache(self, layer_id: int) -> torch.Tensor:
# MLA: K == V (single latent); same buffer, dsv4/v4_pool precedent.
return self.k_cache(layer_id)
def latent_rows(self, layer_id: int) -> torch.Tensor:
"""Row-flat latent view ``[num_pages * page_size, latent_dim]``."""
return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim)
def latent_scale(self, layer_id: int) -> torch.Tensor | None:
"""Row scales ``[num_pages * page_size, 2]`` (ckv, kpe), or None when rows are not quantized."""
return None if self._scale_buffer is None else self._scale_buffer[self._local_layer(layer_id)]
# -- writes -----------------------------------------------------------------
def store_kv(
self,
c_kv: torch.Tensor,
k_rope: torch.Tensor,
out_loc: torch.Tensor,
layer_id: int,
) -> None:
"""Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and
``k_rope`` [T, qk_rope_head_dim] land in the row's two halves.
Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc
store.cu (two-width store).
"""
rows = self.latent_rows(layer_id)
if self.kv_format.quantized:
from freetoken.kernel.triton.kv_rows import store_latent_rows
store_latent_rows(self.kv_format, c_kv, k_rope, out_loc, rows, self.latent_scale(layer_id))
return
split = rows.shape[1] - k_rope.shape[-1]
rows[out_loc, :split] = c_kv
rows[out_loc, split:] = k_rope
def rebuild(self, num_pages: int) -> None:
"""In-place resize (frees the old slab first; object identity preserved --
callers re-derive views per forward, same contract as MHAKVCache.rebuild)."""
self._kv_buffer = None
self._scale_buffer = None
if self._device.type == "cuda":
torch.cuda.synchronize(self._device)
torch.cuda.empty_cache()
self._alloc(num_pages)
@classmethod
def kv_cost(cls, config) -> tuple[int, int, int, int]:
from .base import spec_kv_bytes_per_token
per_token = sum(
spec_kv_bytes_per_token(spec, config, getattr(config, "kv_format", NONE))
for spec in config.model_config.kv_cache_group_specs()
)
return per_token * config.page_size, 0, config.page_size, 0
def rebuild_from_config(
self, config, num_pages: int, *, num_swa_pages: int | None = None
) -> None:
self.rebuild(num_pages + 1) # +1 for the dummy page (matches create_kvcache_pool)
def unit_bytes(self) -> tuple[int, int]:
buf = self._kv_buffer
total = int(buf.numel() * buf.element_size())
if self._scale_buffer is not None:
total += int(self._scale_buffer.numel() * self._scale_buffer.element_size())
return total // (self._num_pages * self._page_size), 0
# -- pool properties ----------------------------------------------------------
@property
def device(self) -> torch.device:
return self._device
@property
def dtype(self) -> torch.dtype:
return self._dtype
@property
def num_layers(self) -> int:
return self._num_layers
class DSAKVCache(MLAKVCache):
"""MLA latent pool + the DSA index-key slab (one row per token per full-indexer
layer, bf16, slot order = the backend's full-layer order)."""
def __init__(
self,
latent_dim: int,
num_layers: int,
num_pages: int,
page_size: int,
dtype: torch.dtype,
device: torch.device,
index_head_dim: int,
num_index_layers: int,
layer_ids: "tuple[int, ...] | None" = None,
kv_format: KVFormat = NONE,
) -> None:
self._index_head_dim = index_head_dim
self._num_index_layers = num_index_layers
super().__init__(
latent_dim, num_layers, num_pages, page_size, dtype, device,
layer_ids=layer_ids, kv_format=kv_format,
)
def _index_rows(self, num_pages: int) -> int:
"""Index-slab row count: one row per token (KpoolDSAKVCache overrides to the
1/ratio shadow + scratch layout)."""
return num_pages * self._page_size
def _alloc(self, num_pages: int) -> None:
# Both slabs in one allocation step so rebuild can never leave one grown
# and the other stale.
super()._alloc(num_pages)
# bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab
# (cache_status._kv_cost_model); keep the two in lockstep.
self._index_k_buffer = torch.zeros(
self._num_index_layers,
self._index_rows(num_pages),
self._index_head_dim,
dtype=torch.bfloat16,
device=self._device,
)
def rebuild(self, num_pages: int) -> None:
self._index_k_buffer = None
super().rebuild(num_pages)
def unit_bytes(self) -> tuple[int, int]:
# The index slab rides the same token budget as the latent slab; each slab's per-token
# cost is floor-divided on its own, matching the cost model's two separate terms.
kv, swa = super().unit_bytes()
idx = self._index_k_buffer
tokens = self._num_pages * self._page_size
return kv + int(idx.numel() * idx.element_size()) // tokens, swa
def index_k_cache(self, slot: int) -> torch.Tensor:
"""Row-flat index keys for a full-indexer layer slot: ``[rows, index_head_dim]``."""
return self._index_k_buffer[slot]
def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> None:
self._index_k_buffer[slot][out_loc] = k
class KpoolDSAKVCache(DSAKVCache):
"""DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus
per-request scratch rows and tail rings.
A pooled entry lives at ``token_slot // index_ratio`` (``page_size %
index_ratio == 0``), so compressed rows follow page sharing/eviction for
free. Rows whose pool does not close write to the request's scratch row
instead (scoring never reads it). The tail rings hold the in-progress
pool's raw K + gate at ``pos % index_ratio`` for pools straddling two
forwards; never cleared -- prefix-cache resume points are page-aligned,
so a stale ring is never read.
"""
def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None:
self._num_req_slots = num_req_slots
self._index_ratio = index_ratio
super().__init__(*args, **kwargs)
assert self._page_size % index_ratio == 0, (
f"kpool needs page_size ({self._page_size}) divisible by "
f"index_ratio ({index_ratio})"
)
def _index_rows(self, num_pages: int) -> int:
# 1/ratio shadow of every token slot + one scratch row per request slot.
return num_pages * self._page_size // self._index_ratio + self._num_req_slots
@property
def cmp_scratch_base(self) -> int:
"""First scratch row (== shadow row count); request ``table_idx`` offsets it."""
return self._num_pages * self._page_size // self._index_ratio
def _alloc(self, num_pages: int) -> None:
super()._alloc(num_pages)
self._tail_k = torch.zeros(
self._num_index_layers, self._num_req_slots, self._index_ratio,
self._index_head_dim, dtype=torch.bfloat16, device=self._device,
)
self._tail_gate = torch.zeros_like(self._tail_k)
def rebuild(self, num_pages: int) -> None:
self._tail_k = None
self._tail_gate = None
super().rebuild(num_pages)
def tail_k(self, slot: int) -> torch.Tensor:
"""Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``."""
return self._tail_k[slot]
def tail_gate(self, slot: int) -> torch.Tensor:
return self._tail_gate[slot]
__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"]