Files
FreeToken/python/freetoken/kvcache/linear_state_pool.py
Xiaoze Fan dacac5f5fa feat(spec): add MTP drafts for the Qwen3.5 family and Qwen3.8-Flash-Next (#844)
* feat(spec): add the MTP proposer for the qwen3_5 family

* test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints

* feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers

* feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack

* docs(cli): document the speculative decoding flags

* fix(spec): refuse MTP draft sources the draft cannot read before the target loads

* docs(qwen4_exp): describe the verify path in the PLE docstrings

* fix(spec): refuse a missing relative MTP draft path before the target loads

* fix(moe): read MTP head experts on the host and require both stacked tensors

* refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq

* refactor(spec): let each MTP draft family check its own weight source

* fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows

* perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr

* refactor(moe): let each family name its MTP expert tensors

* refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load

* fix(spec): refuse MTP with dummy weights

* refactor(spec): move HiddenRelay under spec/mtp, its only user

* refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens

* refactor(models): note FeatureSink's graph binding, drop its runtime_checkable

* fix(models): read quantized MTP fc projections under the quant config
2026-11-02 11:37:09 -08:00

388 lines
18 KiB
Python

from __future__ import annotations
import math
from typing import List
import torch
from freetoken.distributed import get_tp_info
from freetoken.env import ENV
from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec, SlotVerify
from freetoken.utils import div_even
_SSM_DTYPES = {
"float32": torch.float32,
"bfloat16": torch.bfloat16,
"float16": torch.float16,
}
def ssm_state_dtype() -> torch.dtype:
"""Recurrent (SSM) state dtype, from FREETOKEN_MAMBA_SSM_DTYPE (default fp32)."""
return _SSM_DTYPES.get(str(ENV.MAMBA_SSM_DTYPE).lower(), torch.float32)
def _linear_local_dims(
group: LinearGatedDeltaGroupConfig, tp_size: int
) -> tuple[int, int, int, int]:
"""TP-local ``(n_layers, conv_dim, v_heads, k_heads)`` for the GDN state tensors -- the single
source of the sharding math shared by the pool allocation and the byte estimate."""
local_k_heads = div_even(group.num_key_heads, tp_size, allow_replicate=True)
local_v_heads = div_even(group.num_value_heads, tp_size, allow_replicate=True)
local_conv_dim = 2 * local_k_heads * group.key_head_dim + local_v_heads * group.value_head_dim
return len(group.layer_ids), local_conv_dim, local_v_heads, local_k_heads
class LinearStatePool:
"""Per-request recurrent state (conv + SSM) for GatedDeltaNet layers.
Indexed by ``Req.table_idx`` (0..max_running_req), the same per-request slot the
page table uses, so the scheduler's existing admit/free of ``table_idx`` covers the
state's lifetime. One fixed slot per running request; no paging, no eviction.
A model can declare extra per-request tensors on the same slots through
``ModelConfig.slot_states`` (see ``SlotStateSpec``); they advance, snapshot, COW and
rebuild with the GDN state and are read back through ``slot_state(name, layer_id)``.
Consumers must re-read them each forward: ``rebuild`` replaces the tensors.
"""
def __init__(
self,
group: LinearGatedDeltaGroupConfig,
num_slots: int,
dtype: torch.dtype,
device: torch.device,
tp_size: int | None = None,
slot_states: tuple[SlotStateSpec, ...] = (),
) -> None:
if tp_size is None:
tp_size = get_tp_info().size
self._group = group
self._num_slots = num_slots
self._device = device
self._conv_dtype = dtype
n_layers, local_conv_dim, local_v_heads, self._local_k_heads = _linear_local_dims(group, tp_size)
# conv left-context: the last (kernel-1) timesteps of the conv input stream.
self.conv_states = torch.zeros(
(n_layers, num_slots, local_conv_dim, group.conv_kernel_dim - 1),
dtype=dtype,
device=device,
)
# SSM recurrent state. fp32 by default (matches HF mamba_ssm_dtype); the dtype is
# overridable via FREETOKEN_MAMBA_SSM_DTYPE (see ssm_state_dtype).
self.recurrent_states = torch.zeros(
(n_layers, num_slots, local_v_heads, group.key_head_dim, group.value_head_dim),
dtype=ssm_state_dtype(),
device=device,
)
self._local_index = {layer_id: i for i, layer_id in enumerate(group.layer_ids)}
self._slot_specs = tuple(slot_states)
names = [spec.name for spec in self._slot_specs]
if len(set(names)) != len(names):
raise ValueError(f"duplicate slot_state names: {names}")
self._state_layer_index = {
spec.name: {lid: i for i, lid in enumerate(spec.layer_ids)}
for spec in self._slot_specs
}
self.slot_states: dict[str, torch.Tensor] = self._alloc_slot_states(num_slots)
# Free-list allocator over slots 1..num_slots-1 (slot 0 reserved as a padding sink,
# sglang MambaPool convention). Live working slots, ping-pong track slots, and
# radix-tree-donated snapshots are all drawn from this single free-list, so memory
# flows between them by demand. Unused by the op harness (which assigns slots by hand).
self.padding_slot = 0
# verify batches only read the live state; Engine attaches this when a decode step has more than one row
self.verify_scratch: LinearVerifyScratch | None = None
self._verify_handlers: dict[str, SlotVerify] = {}
self._free_slots: list[int] = list(range(1, num_slots))
def _alloc_slot_states(self, num_slots: int) -> dict[str, torch.Tensor]:
return {
spec.name: torch.full(
(max(1, len(spec.layer_ids)), num_slots, *spec.shape),
spec.fill_value,
dtype=self._state_dtype(spec),
device=self._device,
)
for spec in self._slot_specs
}
def _state_dtype(self, spec: SlotStateSpec) -> torch.dtype:
return spec.dtype if spec.dtype is not None else self._conv_dtype
def has_slot_state(self, name: str) -> bool:
return name in self.slot_states
def slot_state(self, name: str, layer_id: int | None = None) -> torch.Tensor:
"""One declared sibling state, ``[num_slots, *shape]``; ``layer_id`` picks the layer row."""
t = self.slot_states[name]
if layer_id is None:
assert not self._state_layer_index[name], (
f"slot_state {name!r} is per-layer, pass layer_id"
)
return t[0]
return t[self._state_layer_index[name][layer_id]]
@property
def num_free_slots(self) -> int:
return len(self._free_slots)
def alloc(self, n: int = 1) -> list[int]:
"""Pop ``n`` free slot ids (LIFO). Raises if the pool is exhausted."""
if n > len(self._free_slots):
raise RuntimeError(
f"LinearStatePool exhausted: need {n}, have {len(self._free_slots)}"
)
return [self._free_slots.pop() for _ in range(n)]
def reclaim_all_slots(self) -> None:
"""Restore the free-list to all non-padding slots. Idle-only: the caller (e.g. a
CacheManager rebuild that discards the tree owning donated snapshots) must guarantee no
running request holds a slot, otherwise live state would be handed out twice."""
self._free_slots = list(range(1, self._num_slots))
def rebuild(self, num_slots: int) -> None:
"""Reallocate the conv + recurrent state tensors for ``num_slots`` slots IN PLACE.
Geometry (layers, conv dim, head dims) and dtypes are taken from the existing
tensors; only the slot count changes. Object identity is preserved so cached
references (ctx.linear_state_pool) stay valid. Idle-only and destructive: every
live/snapshot state is dropped, so the caller must guarantee no running request
holds a slot and the radix tree owning donated snapshots is discarded too.
"""
n_layers, _, local_conv_dim, km1 = self.conv_states.shape
_, _, local_v_heads, key_head_dim, value_head_dim = self.recurrent_states.shape
conv_dtype, rec_dtype = self.conv_states.dtype, self.recurrent_states.dtype
device = self._device
self.conv_states = None
self.recurrent_states = None
self.slot_states = {}
if device.type == "cuda":
torch.cuda.synchronize(device)
torch.cuda.empty_cache()
self.conv_states = torch.zeros(
(n_layers, num_slots, local_conv_dim, km1), dtype=conv_dtype, device=device
)
self.recurrent_states = torch.zeros(
(n_layers, num_slots, local_v_heads, key_head_dim, value_head_dim),
dtype=rec_dtype,
device=device,
)
self.slot_states = self._alloc_slot_states(num_slots)
self._num_slots = num_slots
self._free_slots = list(range(1, num_slots))
def free(self, slots) -> None:
"""Return slot ids to the free-list. Accepts an int, list, or 1-D tensor."""
if isinstance(slots, torch.Tensor):
slots = slots.flatten().tolist()
elif isinstance(slots, int):
slots = [slots]
self._free_slots.extend(int(s) for s in slots)
def clear_slots(self, slots) -> None:
"""Zero conv + recurrent state at ``slots`` across all linear layers (fresh sequence)."""
if isinstance(slots, (list, tuple)):
slots = torch.as_tensor(slots, dtype=torch.long, device=self._device)
self.conv_states[:, slots] = 0
self.recurrent_states[:, slots] = 0
for spec in self._slot_specs:
self.slot_states[spec.name][:, slots] = spec.fill_value
def copy_from(self, src: int, dst: int) -> None:
"""Copy a whole-sequence snapshot (conv + recurrent, all layers) from slot ``src`` to
``dst``. Used for COW-on-restore (donated snapshot -> fresh live slot)."""
self.conv_states[:, dst].copy_(self.conv_states[:, src])
self.recurrent_states[:, dst].copy_(self.recurrent_states[:, src])
for t in self.slot_states.values():
t[:, dst].copy_(t[:, src])
def is_linear_layer(self, layer_id: int) -> bool:
return layer_id in self._local_index
def local_index(self, layer_id: int) -> int:
return self._local_index[layer_id]
def conv_state(self, layer_id: int, table_idx: int) -> torch.Tensor:
return self.conv_states[self._local_index[layer_id], table_idx]
def recurrent_state(self, layer_id: int, table_idx: int) -> torch.Tensor:
return self.recurrent_states[self._local_index[layer_id], table_idx]
def reset(self, table_idx: int) -> None:
"""Zero a slot across all linear layers (new request takes this table_idx)."""
self.conv_states[:, table_idx].zero_()
self.recurrent_states[:, table_idx].zero_()
for spec in self._slot_specs:
self.slot_states[spec.name][:, table_idx] = spec.fill_value
@property
def num_linear_layers(self) -> int:
return len(self._local_index)
@property
def num_slots(self) -> int:
return self._num_slots
@property
def device(self) -> torch.device:
return self._device
def attach_verify_scratch(self, max_bs: int, rows_per_req: int) -> None:
"""Build the scratch for verify batches of up to ``max_bs`` requests with ``rows_per_req`` rows each, and the verify handler of every slot state; build_fla_metadata hands the GDN scratch to the GDN layers."""
self.verify_scratch = LinearVerifyScratch(self, max_bs, rows_per_req)
self._verify_handlers = {
spec.name: spec.verify(spec, max_bs, rows_per_req, self._device, self._state_dtype(spec))
for spec in self._slot_specs
if spec.verify is not None
}
def verify_handler(self, name: str) -> SlotVerify:
"""The verify handler of slot state ``name``, for its owner's verify forward."""
return self._verify_handlers[name]
def commit_verify(self, slots: List[int], counts: List[int]) -> None:
"""Advance live slot slots[i] over verify row i's first counts[i] tokens."""
self.verify_scratch.commit(self, slots, counts)
if slots and self._verify_handlers:
index = torch.tensor([slots, counts], dtype=torch.int64, pin_memory=self._device.type == "cuda")
index = index.to(self._device, non_blocking=True)
for handler in self._verify_handlers.values():
handler.commit(self, *index)
def bytes_per_slot(self) -> int:
"""Total state bytes for one request (all linear layers)."""
per = (
self.conv_states[:, 0].numel() * self.conv_states.element_size()
+ self.recurrent_states[:, 0].numel() * self.recurrent_states.element_size()
)
for t in self.slot_states.values():
per += t[:, 0].numel() * t.element_size()
return int(per)
class LinearVerifyScratch:
"""Per-row scratch of a GDN verify: each row's recurrence inputs and conv window, keyed by verify-batch row.
The verify leaves the live recurrent state untouched and rolls the live conv window as if every row were
accepted; ``commit`` replays the accepted rows into the recurrent state and restores the conv window that ends
at the last accepted row. Dense and indexed by batch row, never by pool slot.
"""
def __init__(self, pool: LinearStatePool, max_bs: int, rows_per_req: int) -> None:
num_layers, _, hv, k_dim, v_dim = pool.recurrent_states.shape
_, _, conv_dim, km1 = pool.conv_states.shape
hk = pool._local_k_heads
device = pool.recurrent_states.device
rows = (num_layers, max_bs, rows_per_req)
f32 = dict(dtype=torch.float32, device=device)
self.stash = (
torch.zeros(*rows, hk, k_dim, **f32),
torch.zeros(*rows, hv, v_dim, **f32),
torch.zeros(*rows, hv, **f32),
torch.zeros(*rows, hv, **f32),
)
self.conv_window = torch.zeros((*rows, conv_dim, km1), dtype=pool.conv_states.dtype, device=device)
def input_stash(self, li: int) -> tuple:
return tuple(t[li] for t in self.stash)
def commit(self, pool: LinearStatePool, slots: List[int], counts: List[int]) -> None:
"""Advance live slot slots[i] over row i's first counts[i] verified tokens (count >= 1)."""
from freetoken.kernel.fla import gdn_state_replay
if not slots:
return
rows = list(range(len(slots)))
meta = torch.tensor([rows, slots, counts], dtype=torch.int32, pin_memory=pool.recurrent_states.is_cuda)
meta = meta.to(pool.recurrent_states.device, non_blocking=True)
gdn_state_replay(pool.recurrent_states, self.stash, meta)
row_idx, slot_idx, count = meta.long()
pool.conv_states[:, slot_idx] = self.conv_window[:, row_idx, count - 1]
def linear_state_bytes_per_req(
group: LinearGatedDeltaGroupConfig,
tp_size: int,
dtype: torch.dtype,
slot_states: tuple[SlotStateSpec, ...] = (),
) -> int:
"""Linear-state bytes for one request across all linear layers (TP-local), plus any
declared slot_states."""
n_layers, local_conv_dim, local_v_heads, _ = _linear_local_dims(group, tp_size)
conv_elems = local_conv_dim * (group.conv_kernel_dim - 1)
rec_elems = local_v_heads * group.key_head_dim * group.value_head_dim
conv_bytes = conv_elems * dtype.itemsize # conv state in model dtype
rec_bytes = rec_elems * ssm_state_dtype().itemsize # recurrent state (default fp32)
total = n_layers * (conv_bytes + rec_bytes)
for spec in slot_states:
item = (spec.dtype if spec.dtype is not None else dtype).itemsize
total += max(1, len(spec.layer_ids)) * math.prod(spec.shape) * item
return int(total)
__all__ = ["LinearStatePool", "linear_state_bytes_per_req"]
def state_pool_bytes(config, num_slots: int | None = None) -> int:
"""Total GDN state-pool bytes at ``num_slots`` PHYSICAL slots (default: the startup
slot count). The engine adds this to the KV family's fixed cost when budgeting --
the state pool is a sibling pool, not a KV tier."""
linear_group = config.model_config.linear_attention_group()
slot_states = getattr(config.model_config, "slot_states", ())
if linear_group is None:
if slot_states:
raise ValueError("slot_states ride the linear-state slots; model has no linear group")
return 0
slots = num_slots if num_slots is not None else _linear_pool_num_slots(config)
per_req = linear_state_bytes_per_req(
linear_group, config.tp_info.size, config.dtype, slot_states
)
return per_req * slots + verify_scratch_bytes(config, getattr(config, "decode_rows_per_req", 1))
def verify_scratch_bytes(config, rows_per_req: int) -> int:
"""Bytes of the LinearVerifyScratch the engine builds for verify batches of ``rows_per_req`` rows per request (0 when it is 1)."""
group = config.model_config.linear_attention_group()
if group is None or rows_per_req <= 1:
return 0
n_layers, conv_dim, hv, hk = _linear_local_dims(group, config.tp_info.size)
rows = n_layers * config.max_running_req * rows_per_req
stash = rows * (hk * group.key_head_dim + hv * group.value_head_dim + 2 * hv) * 4
window = rows * conv_dim * (group.conv_kernel_dim - 1) * config.dtype.itemsize
handlers = sum(
spec.verify.nbytes(spec, config.max_running_req, rows_per_req, spec.dtype if spec.dtype is not None else config.dtype)
for spec in getattr(config.model_config, "slot_states", ())
if spec.verify is not None
)
return stash + window + handlers
def _linear_pool_num_slots(config) -> int:
"""LinearStatePool slot count. Hybrid-radix non-evictable peak is 4 slots per running request
(1 live + 2 ping-pong + 1 committed snapshot locked through decode), plus a cross-request
snapshot cache and a padding sink; naive GDN keeps the old (max_running_req + 1)."""
mr = config.max_running_req
if config.cache_type != "hybrid_radix":
return mr + 1 # live + dummy/padding
ratio = config.linear_state_cache_ratio
n_cache = max(4, int(ratio * mr))
return 4 * mr + n_cache + 1 # live + 2 ping-pong + locked committed snapshot + cache + padding
def _linear_pool_min_slots(config) -> int:
"""Floor on LinearStatePool slots that still runs: the non-evictable working set with a
zero snapshot cache. Hybrid-radix needs 4 per running request (1 live + 2 ping-pong + 1
committed snapshot locked through decode) + the padding sink; naive needs 1 per request +
padding. Below this, a full max_running_req batch can't get its slots and admission
deadlocks -- so a runtime rebuild rejects a smaller request."""
mr = config.max_running_req
if config.cache_type != "hybrid_radix":
return mr + 1
return 4 * mr + 1