mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(kvcache): add fp8 and int8 per-token-head KV cache formats * fix(kernel): keep the bf16 m3 sparse k/v loads inline * fix(attention): refuse a backend that cannot read the kv format - filter the m3 page-size fallback by kv format - release the scale buffer when a qsa or bsa rebuild fails - keep the bf16 qsa and m3 attend code identical to main - drop docstrings that claimed no register spills - drop guard tests * fix(kernel): widen dsa latent row offsets past int32 * feat(spec): store the mtp draft kv in the target's format
293 lines
12 KiB
Python
293 lines
12 KiB
Python
"""MLA latent-KV pools (GLM-5.2 / DeepSeek-style latent attention).
|
|
|
|
``MLAKVCache`` is the MHA pool's sibling for latent-KV models: ONE slab holding the
|
|
per-token latent ``ckv (kv_lora_rank) | kpe (qk_rope_head_dim)`` -- there is no
|
|
separate V (``v_cache`` aliases ``k_cache``, same convention as dsv4/v4_pool's
|
|
single-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attention
|
|
index-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer
|
|
layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1;
|
|
``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild``
|
|
resizes ALL slabs atomically so the allocator can never hand out a slot one slab
|
|
has and the other lacks.
|
|
|
|
Storage lives here -- not in the attention backend -- so the engine's rebuild path
|
|
(``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views
|
|
re-derived by callers per forward) and the KV cost model (which budgets the index-K
|
|
bytes off the attention-group spec) stay correct by construction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import ClassVar
|
|
|
|
import torch
|
|
|
|
from .base import BaseKVCachePool
|
|
from .format import NONE, ROW_SCALE_FORMATS, KVFormat
|
|
|
|
|
|
class MLAKVCache(BaseKVCachePool):
|
|
"""Paged latent-KV pool: ``[1, num_layers, num_pages, page_size, 1, latent_dim]``.
|
|
|
|
The leading singleton keeps the buffer shape-compatible with MHAKVCache's
|
|
(tokens = shape[2] * shape[3]).
|
|
|
|
``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x
|
|
MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap
|
|
contract as MHAKVCache; None keeps the identity mapping.
|
|
|
|
A quantized row keeps two scales: ckv and kpe have their own ranges.
|
|
"""
|
|
|
|
kv_formats: ClassVar[frozenset[str]] = ROW_SCALE_FORMATS
|
|
|
|
def __init__(
|
|
self,
|
|
latent_dim: int,
|
|
num_layers: int,
|
|
num_pages: int,
|
|
page_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
layer_ids: "tuple[int, ...] | None" = None,
|
|
kv_format: KVFormat = NONE,
|
|
) -> None:
|
|
self.kv_format = kv_format
|
|
self._latent_dim = latent_dim
|
|
if layer_ids is None:
|
|
self._num_layers = num_layers
|
|
self._layer_index: dict[int, int] | None = None
|
|
else:
|
|
self._num_layers = len(layer_ids)
|
|
self._layer_index = {int(g): i for i, g in enumerate(layer_ids)}
|
|
self._page_size = page_size
|
|
self._dtype = dtype
|
|
self._device = device
|
|
self._alloc(num_pages)
|
|
|
|
def _local_layer(self, layer_id: int) -> int:
|
|
return layer_id if self._layer_index is None else self._layer_index[layer_id]
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
self._num_pages = num_pages
|
|
fmt = self.kv_format
|
|
new = torch.zeros if fmt.quantized else torch.empty
|
|
self._kv_buffer = new(
|
|
(1, self._num_layers, num_pages, self._page_size, 1, fmt.elem_dim(self._latent_dim)),
|
|
device=self._device,
|
|
dtype=fmt.elem_tensor_dtype(self._dtype),
|
|
)
|
|
self._scale_buffer = None
|
|
if fmt.scale_dtype is not None:
|
|
self._scale_buffer = torch.zeros(
|
|
(self._num_layers, num_pages * self._page_size, 2),
|
|
device=self._device,
|
|
dtype=fmt.scale_store_dtype,
|
|
)
|
|
|
|
# -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) --
|
|
def k_cache(self, layer_id: int) -> torch.Tensor:
|
|
"""Paged latent view ``[num_pages, page_size, latent_dim]``."""
|
|
return self._kv_buffer[0, self._local_layer(layer_id)].view(
|
|
self._num_pages, self._page_size, -1
|
|
)
|
|
|
|
def v_cache(self, layer_id: int) -> torch.Tensor:
|
|
# MLA: K == V (single latent); same buffer, dsv4/v4_pool precedent.
|
|
return self.k_cache(layer_id)
|
|
|
|
def latent_rows(self, layer_id: int) -> torch.Tensor:
|
|
"""Row-flat latent view ``[num_pages * page_size, latent_dim]``."""
|
|
return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim)
|
|
|
|
def latent_scale(self, layer_id: int) -> torch.Tensor | None:
|
|
"""Row scales ``[num_pages * page_size, 2]`` (ckv, kpe), or None when rows are not quantized."""
|
|
return None if self._scale_buffer is None else self._scale_buffer[self._local_layer(layer_id)]
|
|
|
|
# -- writes -----------------------------------------------------------------
|
|
def store_kv(
|
|
self,
|
|
c_kv: torch.Tensor,
|
|
k_rope: torch.Tensor,
|
|
out_loc: torch.Tensor,
|
|
layer_id: int,
|
|
) -> None:
|
|
"""Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and
|
|
``k_rope`` [T, qk_rope_head_dim] land in the row's two halves.
|
|
|
|
Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc
|
|
store.cu (two-width store).
|
|
"""
|
|
rows = self.latent_rows(layer_id)
|
|
if self.kv_format.quantized:
|
|
from freetoken.kernel.triton.kv_rows import store_latent_rows
|
|
|
|
store_latent_rows(self.kv_format, c_kv, k_rope, out_loc, rows, self.latent_scale(layer_id))
|
|
return
|
|
split = rows.shape[1] - k_rope.shape[-1]
|
|
rows[out_loc, :split] = c_kv
|
|
rows[out_loc, split:] = k_rope
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
"""In-place resize (frees the old slab first; object identity preserved --
|
|
callers re-derive views per forward, same contract as MHAKVCache.rebuild)."""
|
|
self._kv_buffer = None
|
|
self._scale_buffer = None
|
|
if self._device.type == "cuda":
|
|
torch.cuda.synchronize(self._device)
|
|
torch.cuda.empty_cache()
|
|
self._alloc(num_pages)
|
|
|
|
@classmethod
|
|
def kv_cost(cls, config) -> tuple[int, int, int, int]:
|
|
from .base import spec_kv_bytes_per_token
|
|
|
|
per_token = sum(
|
|
spec_kv_bytes_per_token(spec, config, getattr(config, "kv_format", NONE))
|
|
for spec in config.model_config.kv_cache_group_specs()
|
|
)
|
|
return per_token * config.page_size, 0, config.page_size, 0
|
|
|
|
def rebuild_from_config(
|
|
self, config, num_pages: int, *, num_swa_pages: int | None = None
|
|
) -> None:
|
|
self.rebuild(num_pages + 1) # +1 for the dummy page (matches create_kvcache_pool)
|
|
|
|
def unit_bytes(self) -> tuple[int, int]:
|
|
buf = self._kv_buffer
|
|
total = int(buf.numel() * buf.element_size())
|
|
if self._scale_buffer is not None:
|
|
total += int(self._scale_buffer.numel() * self._scale_buffer.element_size())
|
|
return total // (self._num_pages * self._page_size), 0
|
|
|
|
# -- pool properties ----------------------------------------------------------
|
|
@property
|
|
def device(self) -> torch.device:
|
|
return self._device
|
|
|
|
@property
|
|
def dtype(self) -> torch.dtype:
|
|
return self._dtype
|
|
|
|
@property
|
|
def num_layers(self) -> int:
|
|
return self._num_layers
|
|
|
|
|
|
class DSAKVCache(MLAKVCache):
|
|
"""MLA latent pool + the DSA index-key slab (one row per token per full-indexer
|
|
layer, bf16, slot order = the backend's full-layer order)."""
|
|
|
|
def __init__(
|
|
self,
|
|
latent_dim: int,
|
|
num_layers: int,
|
|
num_pages: int,
|
|
page_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
index_head_dim: int,
|
|
num_index_layers: int,
|
|
layer_ids: "tuple[int, ...] | None" = None,
|
|
kv_format: KVFormat = NONE,
|
|
) -> None:
|
|
self._index_head_dim = index_head_dim
|
|
self._num_index_layers = num_index_layers
|
|
super().__init__(
|
|
latent_dim, num_layers, num_pages, page_size, dtype, device,
|
|
layer_ids=layer_ids, kv_format=kv_format,
|
|
)
|
|
|
|
def _index_rows(self, num_pages: int) -> int:
|
|
"""Index-slab row count: one row per token (KpoolDSAKVCache overrides to the
|
|
1/ratio shadow + scratch layout)."""
|
|
return num_pages * self._page_size
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
# Both slabs in one allocation step so rebuild can never leave one grown
|
|
# and the other stale.
|
|
super()._alloc(num_pages)
|
|
# bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab
|
|
# (cache_status._kv_cost_model); keep the two in lockstep.
|
|
self._index_k_buffer = torch.zeros(
|
|
self._num_index_layers,
|
|
self._index_rows(num_pages),
|
|
self._index_head_dim,
|
|
dtype=torch.bfloat16,
|
|
device=self._device,
|
|
)
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
self._index_k_buffer = None
|
|
super().rebuild(num_pages)
|
|
|
|
def unit_bytes(self) -> tuple[int, int]:
|
|
# The index slab rides the same token budget as the latent slab; each slab's per-token
|
|
# cost is floor-divided on its own, matching the cost model's two separate terms.
|
|
kv, swa = super().unit_bytes()
|
|
idx = self._index_k_buffer
|
|
tokens = self._num_pages * self._page_size
|
|
return kv + int(idx.numel() * idx.element_size()) // tokens, swa
|
|
|
|
def index_k_cache(self, slot: int) -> torch.Tensor:
|
|
"""Row-flat index keys for a full-indexer layer slot: ``[rows, index_head_dim]``."""
|
|
return self._index_k_buffer[slot]
|
|
|
|
def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> None:
|
|
self._index_k_buffer[slot][out_loc] = k
|
|
|
|
|
|
class KpoolDSAKVCache(DSAKVCache):
|
|
"""DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus
|
|
per-request scratch rows and tail rings.
|
|
|
|
A pooled entry lives at ``token_slot // index_ratio`` (``page_size %
|
|
index_ratio == 0``), so compressed rows follow page sharing/eviction for
|
|
free. Rows whose pool does not close write to the request's scratch row
|
|
instead (scoring never reads it). The tail rings hold the in-progress
|
|
pool's raw K + gate at ``pos % index_ratio`` for pools straddling two
|
|
forwards; never cleared -- prefix-cache resume points are page-aligned,
|
|
so a stale ring is never read.
|
|
"""
|
|
|
|
def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None:
|
|
self._num_req_slots = num_req_slots
|
|
self._index_ratio = index_ratio
|
|
super().__init__(*args, **kwargs)
|
|
assert self._page_size % index_ratio == 0, (
|
|
f"kpool needs page_size ({self._page_size}) divisible by "
|
|
f"index_ratio ({index_ratio})"
|
|
)
|
|
|
|
def _index_rows(self, num_pages: int) -> int:
|
|
# 1/ratio shadow of every token slot + one scratch row per request slot.
|
|
return num_pages * self._page_size // self._index_ratio + self._num_req_slots
|
|
|
|
@property
|
|
def cmp_scratch_base(self) -> int:
|
|
"""First scratch row (== shadow row count); request ``table_idx`` offsets it."""
|
|
return self._num_pages * self._page_size // self._index_ratio
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
super()._alloc(num_pages)
|
|
self._tail_k = torch.zeros(
|
|
self._num_index_layers, self._num_req_slots, self._index_ratio,
|
|
self._index_head_dim, dtype=torch.bfloat16, device=self._device,
|
|
)
|
|
self._tail_gate = torch.zeros_like(self._tail_k)
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
self._tail_k = None
|
|
self._tail_gate = None
|
|
super().rebuild(num_pages)
|
|
|
|
def tail_k(self, slot: int) -> torch.Tensor:
|
|
"""Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``."""
|
|
return self._tail_k[slot]
|
|
|
|
def tail_gate(self, slot: int) -> torch.Tensor:
|
|
return self._tail_gate[slot]
|
|
|
|
|
|
__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"]
|