mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
261 lines
10 KiB
Python
261 lines
10 KiB
Python
"""MLA latent-KV pools (GLM-5.2 / DeepSeek-style latent attention).
|
|
|
|
``MLAKVCache`` is the MHA pool's sibling for latent-KV models: ONE slab holding the
|
|
per-token latent ``ckv (kv_lora_rank) | kpe (qk_rope_head_dim)`` -- there is no
|
|
separate V (``v_cache`` aliases ``k_cache``, same convention as dsv4/v4_pool's
|
|
single-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attention
|
|
index-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer
|
|
layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1;
|
|
``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild``
|
|
resizes ALL slabs atomically so the allocator can never hand out a slot one slab
|
|
has and the other lacks.
|
|
|
|
Storage lives here -- not in the attention backend -- so the engine's rebuild path
|
|
(``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views
|
|
re-derived by callers per forward) and the KV cost model (which budgets the index-K
|
|
bytes off the attention-group spec) stay correct by construction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
|
|
from .base import BaseKVCachePool
|
|
|
|
|
|
class MLAKVCache(BaseKVCachePool):
|
|
"""Paged latent-KV pool: ``[1, num_layers, num_pages, page_size, 1, latent_dim]``.
|
|
|
|
The leading singleton keeps the buffer shape-compatible with MHAKVCache's
|
|
(tokens = shape[2] * shape[3]).
|
|
|
|
``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x
|
|
MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap
|
|
contract as MHAKVCache; None keeps the identity mapping.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
latent_dim: int,
|
|
num_layers: int,
|
|
num_pages: int,
|
|
page_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
layer_ids: "tuple[int, ...] | None" = None,
|
|
) -> None:
|
|
self._latent_dim = latent_dim
|
|
if layer_ids is None:
|
|
self._num_layers = num_layers
|
|
self._layer_index: dict[int, int] | None = None
|
|
else:
|
|
self._num_layers = len(layer_ids)
|
|
self._layer_index = {int(g): i for i, g in enumerate(layer_ids)}
|
|
self._page_size = page_size
|
|
self._dtype = dtype
|
|
self._device = device
|
|
self._alloc(num_pages)
|
|
|
|
def _local_layer(self, layer_id: int) -> int:
|
|
return layer_id if self._layer_index is None else self._layer_index[layer_id]
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
self._num_pages = num_pages
|
|
self._kv_buffer = torch.empty(
|
|
(1, self._num_layers, num_pages, self._page_size, 1, self._latent_dim),
|
|
device=self._device,
|
|
dtype=self._dtype,
|
|
)
|
|
|
|
# -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) --
|
|
def k_cache(self, layer_id: int) -> torch.Tensor:
|
|
"""Paged latent view ``[num_pages, page_size, latent_dim]``."""
|
|
return self._kv_buffer[0, self._local_layer(layer_id)].view(
|
|
self._num_pages, self._page_size, -1
|
|
)
|
|
|
|
def v_cache(self, layer_id: int) -> torch.Tensor:
|
|
# MLA: K == V (single latent); same buffer, dsv4/v4_pool precedent.
|
|
return self.k_cache(layer_id)
|
|
|
|
def latent_rows(self, layer_id: int) -> torch.Tensor:
|
|
"""Row-flat latent view ``[num_pages * page_size, latent_dim]``."""
|
|
return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim)
|
|
|
|
# -- writes -----------------------------------------------------------------
|
|
def store_kv(
|
|
self,
|
|
c_kv: torch.Tensor,
|
|
k_rope: torch.Tensor,
|
|
out_loc: torch.Tensor,
|
|
layer_id: int,
|
|
) -> None:
|
|
"""Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and
|
|
``k_rope`` [T, qk_rope_head_dim] land in the row's two halves.
|
|
|
|
Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc
|
|
store.cu (two-width store).
|
|
"""
|
|
rows = self.latent_rows(layer_id)
|
|
split = rows.shape[1] - k_rope.shape[-1]
|
|
rows[out_loc, :split] = c_kv
|
|
rows[out_loc, split:] = k_rope
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
"""In-place resize (frees the old slab first; object identity preserved --
|
|
callers re-derive views per forward, same contract as MHAKVCache.rebuild)."""
|
|
self._kv_buffer = None
|
|
if self._device.type == "cuda":
|
|
torch.cuda.synchronize(self._device)
|
|
torch.cuda.empty_cache()
|
|
self._alloc(num_pages)
|
|
|
|
@classmethod
|
|
def kv_cost(cls, config) -> tuple[int, int, int, int]:
|
|
from .base import spec_kv_bytes_per_token
|
|
|
|
per_token = sum(
|
|
spec_kv_bytes_per_token(spec, config)
|
|
for spec in config.model_config.kv_cache_group_specs()
|
|
)
|
|
return per_token * config.page_size, 0, config.page_size, 0
|
|
|
|
def rebuild_from_config(
|
|
self, config, num_pages: int, *, num_swa_pages: int | None = None
|
|
) -> None:
|
|
self.rebuild(num_pages + 1) # +1 for the dummy page (matches create_kvcache_pool)
|
|
|
|
def unit_bytes(self) -> tuple[int, int]:
|
|
buf = self._kv_buffer
|
|
return int(buf.numel() * buf.element_size()) // (self._num_pages * self._page_size), 0
|
|
|
|
# -- pool properties ----------------------------------------------------------
|
|
@property
|
|
def device(self) -> torch.device:
|
|
return self._device
|
|
|
|
@property
|
|
def dtype(self) -> torch.dtype:
|
|
return self._dtype
|
|
|
|
@property
|
|
def num_layers(self) -> int:
|
|
return self._num_layers
|
|
|
|
|
|
class DSAKVCache(MLAKVCache):
|
|
"""MLA latent pool + the DSA index-key slab (one row per token per full-indexer
|
|
layer, bf16, slot order = the backend's full-layer order)."""
|
|
|
|
def __init__(
|
|
self,
|
|
latent_dim: int,
|
|
num_layers: int,
|
|
num_pages: int,
|
|
page_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
index_head_dim: int,
|
|
num_index_layers: int,
|
|
layer_ids: "tuple[int, ...] | None" = None,
|
|
) -> None:
|
|
self._index_head_dim = index_head_dim
|
|
self._num_index_layers = num_index_layers
|
|
super().__init__(
|
|
latent_dim, num_layers, num_pages, page_size, dtype, device,
|
|
layer_ids=layer_ids,
|
|
)
|
|
|
|
def _index_rows(self, num_pages: int) -> int:
|
|
"""Index-slab row count: one row per token (KpoolDSAKVCache overrides to the
|
|
1/ratio shadow + scratch layout)."""
|
|
return num_pages * self._page_size
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
# Both slabs in one allocation step so rebuild can never leave one grown
|
|
# and the other stale.
|
|
super()._alloc(num_pages)
|
|
# bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab
|
|
# (cache_status._kv_cost_model); keep the two in lockstep.
|
|
self._index_k_buffer = torch.zeros(
|
|
self._num_index_layers,
|
|
self._index_rows(num_pages),
|
|
self._index_head_dim,
|
|
dtype=torch.bfloat16,
|
|
device=self._device,
|
|
)
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
self._index_k_buffer = None
|
|
super().rebuild(num_pages)
|
|
|
|
def unit_bytes(self) -> tuple[int, int]:
|
|
# The index slab rides the same token budget as the latent slab; each slab's per-token
|
|
# cost is floor-divided on its own, matching the cost model's two separate terms.
|
|
kv, swa = super().unit_bytes()
|
|
idx = self._index_k_buffer
|
|
tokens = self._num_pages * self._page_size
|
|
return kv + int(idx.numel() * idx.element_size()) // tokens, swa
|
|
|
|
def index_k_cache(self, slot: int) -> torch.Tensor:
|
|
"""Row-flat index keys for a full-indexer layer slot: ``[rows, index_head_dim]``."""
|
|
return self._index_k_buffer[slot]
|
|
|
|
def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> None:
|
|
self._index_k_buffer[slot][out_loc] = k
|
|
|
|
|
|
class KpoolDSAKVCache(DSAKVCache):
|
|
"""DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus
|
|
per-request scratch rows and tail rings.
|
|
|
|
A pooled entry lives at ``token_slot // index_ratio`` (``page_size %
|
|
index_ratio == 0``), so compressed rows follow page sharing/eviction for
|
|
free. Rows whose pool does not close write to the request's scratch row
|
|
instead (scoring never reads it). The tail rings hold the in-progress
|
|
pool's raw K + gate at ``pos % index_ratio`` for pools straddling two
|
|
forwards; never cleared -- prefix-cache resume points are page-aligned,
|
|
so a stale ring is never read.
|
|
"""
|
|
|
|
def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None:
|
|
self._num_req_slots = num_req_slots
|
|
self._index_ratio = index_ratio
|
|
super().__init__(*args, **kwargs)
|
|
assert self._page_size % index_ratio == 0, (
|
|
f"kpool needs page_size ({self._page_size}) divisible by "
|
|
f"index_ratio ({index_ratio})"
|
|
)
|
|
|
|
def _index_rows(self, num_pages: int) -> int:
|
|
# 1/ratio shadow of every token slot + one scratch row per request slot.
|
|
return num_pages * self._page_size // self._index_ratio + self._num_req_slots
|
|
|
|
@property
|
|
def cmp_scratch_base(self) -> int:
|
|
"""First scratch row (== shadow row count); request ``table_idx`` offsets it."""
|
|
return self._num_pages * self._page_size // self._index_ratio
|
|
|
|
def _alloc(self, num_pages: int) -> None:
|
|
super()._alloc(num_pages)
|
|
self._tail_k = torch.zeros(
|
|
self._num_index_layers, self._num_req_slots, self._index_ratio,
|
|
self._index_head_dim, dtype=torch.bfloat16, device=self._device,
|
|
)
|
|
self._tail_gate = torch.zeros_like(self._tail_k)
|
|
|
|
def rebuild(self, num_pages: int) -> None:
|
|
self._tail_k = None
|
|
self._tail_gate = None
|
|
super().rebuild(num_pages)
|
|
|
|
def tail_k(self, slot: int) -> torch.Tensor:
|
|
"""Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``."""
|
|
return self._tail_k[slot]
|
|
|
|
def tail_gate(self, slot: int) -> torch.Tensor:
|
|
return self._tail_gate[slot]
|
|
|
|
|
|
__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"]
|