Files
Xiaoze Fan dacac5f5fa feat(spec): add MTP drafts for the Qwen3.5 family and Qwen3.8-Flash-Next (#844)
* feat(spec): add the MTP proposer for the qwen3_5 family

* test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints

* feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers

* feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack

* docs(cli): document the speculative decoding flags

* fix(spec): refuse MTP draft sources the draft cannot read before the target loads

* docs(qwen4_exp): describe the verify path in the PLE docstrings

* fix(spec): refuse a missing relative MTP draft path before the target loads

* fix(moe): read MTP head experts on the host and require both stacked tensors

* refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq

* refactor(spec): let each MTP draft family check its own weight source

* fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows

* perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr

* refactor(moe): let each family name its MTP expert tensors

* refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load

* fix(spec): refuse MTP with dummy weights

* refactor(spec): move HiddenRelay under spec/mtp, its only user

* refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens

* refactor(models): note FeatureSink's graph binding, drop its runtime_checkable

* fix(models): read quantized MTP fc projections under the quant config
2026-11-02 11:37:09 -08:00

233 lines
10 KiB
Python

"""QSA compressed-block sparse KV pool: paged GQA K/V + compressed index keys + pending ring.
Qwen3.8-Flash-Next scores whole ``index_ratio``-token groups instead of single tokens, so
its indexer slab holds ONE compressed key row per group, addressed by ``slot //
index_ratio``. Because ``page_size % index_ratio == 0``, a group's tokens always live in one
page at consecutive slots, which makes that division well-defined: the compressed rows are a
1/ratio shadow of the K/V pages and follow page sharing and eviction for free -- no
allocator, no free, no clear (SGLang qsa_kv_pool / vLLM compressed-region precedent).
Two tiers ride alongside the shadow slab and are NOT per-token:
- ``pending_ring``: the last ``ring_capacity`` pre-RoPE index keys of each running request (sized by ``ring_capacity_for``), indexed by ``Req.table_idx``. A group that straddles two forwards (chunked prefill, and
every decode step) reads its already-consumed members from here. Never cleared: a new
tenant of a table_idx starts at a group boundary (cached_len is 0 or a page multiple), so
its first closing group takes every member from its own forward.
- scratch rows at ``cmp_scratch_base``: one row per request slot, the write target for rows
whose group does not close in this forward, so the compress kernel scatters unconditionally
with no negative index and no cross-row conflict (DSV4 precedent).
The slab is amortized into the per-token KV price (``unit_bytes``); the ring and scratch are
fixed and priced through ``kv_cost``'s ``fixed_cache_size``.
"""
from __future__ import annotations
import math
from typing import Sequence
import torch
from .mha_pool import MHAKVCache
# The index tiers are always 2-byte (compute dtype); spec_kv_bytes_per_token budgets the same.
_INDEX_DTYPE_BYTES = 2
# t/h/w int32 rope position kept per KV slot on mrope models
_ROPE_POS_BYTES = 3 * 4
class QSAKVCache(MHAKVCache):
"""MHA paged pool + the compressed index-key slab + the per-request pending ring.
``cmp_k_cache(slot)`` is row-flat ``[num_pages * page_size // index_ratio + num_req_slots,
index_head_dim]``: row ``r < cmp_scratch_base`` holds the compressed key of the token group
whose K/V slots are ``[r * index_ratio, (r + 1) * index_ratio)``, and the rows from
``cmp_scratch_base`` on are the per-request-slot scratch sinks. ``slot`` is the sparse
layer's order in the attention backend, same convention as BSAKVCache/DSAKVCache.
"""
@classmethod
def ring_capacity_for(cls, index_ratio: int, num_speculative_tokens: int = 0) -> int:
"""Ring depth: one row per pending position, keyed ``position % capacity``; spec decode widens by the draft depth (vLLM sizing)."""
return index_ratio * math.ceil((index_ratio + num_speculative_tokens) / index_ratio)
def __init__(
self,
num_kv_heads: int,
num_layers: int,
head_dim: int,
num_pages: int,
page_size: int,
dtype: torch.dtype,
device: torch.device,
index_head_dim: int,
num_index_layers: int,
index_ratio: int,
num_req_slots: int,
ring_capacity: int | None = None,
layer_ids: Sequence[int] | None = None,
mrope: bool = False,
) -> None:
if index_ratio < 1 or page_size % index_ratio != 0:
# slot // index_ratio only names one group when a group never straddles a page.
raise ValueError(
f"QSA needs page_size ({page_size}) divisible by index_ratio ({index_ratio})"
)
if ring_capacity is None:
ring_capacity = self.ring_capacity_for(index_ratio)
if ring_capacity < index_ratio:
# A closing group reads up to index_ratio - 1 past members plus this forward's.
raise ValueError(
f"QSA needs ring_capacity ({ring_capacity}) >= index_ratio ({index_ratio})"
)
# Index keys ride the compute dtype (the model's index_k is engine-dtype). The KV cost
# model budgets 2 bytes per token per index layer for the slab
# (base.spec_kv_bytes_per_token); keep the two in lockstep.
assert dtype.itemsize == _INDEX_DTYPE_BYTES, (
f"QSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}"
)
self._index_head_dim = index_head_dim
self._num_index_layers = num_index_layers
self._index_ratio = index_ratio
self._num_req_slots = num_req_slots
self._ring_capacity = ring_capacity
self._index_dtype = dtype
self._page_size = page_size
self._mrope = mrope
super().__init__(
num_kv_heads=num_kv_heads,
num_layers=num_layers,
head_dim=head_dim,
num_pages=num_pages,
page_size=page_size,
dtype=dtype,
device=device,
layer_ids=layer_ids,
)
self._zero_kv_slabs()
self._alloc_index_tiers(num_pages)
def _zero_kv_slabs(self) -> None:
# Defense-in-depth: the attend kernels pos-mask every K/V load (the real fix for
# torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab keeps any future
# unmasked read finite instead of model-poisoning. One memset per (re)allocation.
self._kv_buffer.zero_()
def _alloc_index_tiers(self, num_pages: int) -> None:
# ZERO-initialized: the score kernel reads whole rows of blocks unmasked and relies on
# never-written tail rows dotting to a finite 0. Written rows are never cleared again,
# so the kernel must clamp visible blocks to kvlen // index_ratio.
self._cmp_scratch_base = num_pages * self._page_size // self._index_ratio
self._cmp_k_buffer = torch.zeros(
self._num_index_layers,
self._cmp_scratch_base + self._num_req_slots,
self._index_head_dim,
dtype=self._index_dtype,
device=self._device,
)
self._pending_ring = torch.zeros(
self._num_req_slots,
self._num_index_layers,
self._ring_capacity,
self._index_head_dim,
dtype=self._index_dtype,
device=self._device,
)
# 3-axis rope position of every stored token: a compressed group ropes at its first token, which under mrope is not derivable from the logical position
self._rope_positions = (
torch.zeros(num_pages * self._page_size, 3, dtype=torch.int32, device=self._device)
if self._mrope
else None
)
def rebuild(self, num_pages: int) -> None:
# Free the index tiers BEFORE the K/V realloc (super().rebuild frees + syncs +
# empty_cache), then re-derive them at the new page count. If the index alloc itself
# fails (OOM), null the K/V slab too and re-raise: a pool with a grown K/V slab and no
# index slab would mis-serve silently. Rebuild is idle-only, so zeroing the ring here
# cannot drop a live request's pending members.
self._cmp_k_buffer = None
self._pending_ring = None
self._rope_positions = None
super().rebuild(num_pages)
self._zero_kv_slabs()
try:
self._alloc_index_tiers(num_pages)
except Exception:
self._kv_buffer = None
self._k_buffer = None
self._v_buffer = None
raise
@classmethod
def kv_cost(cls, config) -> tuple[int, int, int, int]:
from .base import spec_kv_bytes_per_token
from freetoken.attention import AttnType
num_req_slots = config.max_running_req + 1
per_token = 0
fixed = 0
for spec in config.model_config.kv_cache_group_specs():
if spec.is_swa:
continue
per_token += spec_kv_bytes_per_token(spec, config)
if spec.attn_type is AttnType.QSA:
# One index-key row = all index layers at one position.
row = spec.index_head_dim * spec.num_index_layers * _INDEX_DTYPE_BYTES
ring = cls.ring_capacity_for(spec.index_ratio, getattr(config, "decode_rows_per_req", 1) - 1)
fixed += num_req_slots * row * (ring + 1)
if config.model_config.model_is_mrope:
per_token += _ROPE_POS_BYTES
return per_token * config.page_size, fixed, config.page_size, 0
def unit_bytes(self) -> tuple[int, int]:
# Only the shadow slab scales with pages, and only its non-scratch rows; the ring and
# the scratch rows are the fixed term kv_cost reports separately.
kv, swa = super().unit_bytes()
tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3])
slab = (
self._num_index_layers
* self._cmp_scratch_base
* self._index_head_dim
* self._index_dtype.itemsize
)
return kv + slab // tokens + (_ROPE_POS_BYTES if self._mrope else 0), swa
def cmp_k_cache(self, slot: int) -> torch.Tensor:
"""Compressed index keys of one sparse layer: ``[rows, index_head_dim]``."""
return self._cmp_k_buffer[slot]
def pending_ring(self, slot: int) -> torch.Tensor:
"""One sparse layer's pending ring: ``[num_req_slots, ring_capacity, index_head_dim]``."""
return self._pending_ring[:, slot]
@property
def rope_positions(self) -> torch.Tensor:
"""``[num_tokens, 3]`` int32 t/h/w rope position per KV slot (written by the QSA backend)."""
assert self._rope_positions is not None, "rope positions are only kept on mrope models"
return self._rope_positions
@property
def cmp_scratch_base(self) -> int:
"""First scratch row of ``cmp_k_cache``; row ``cmp_scratch_base + table_idx`` sinks a
forward whose group does not close."""
return self._cmp_scratch_base
@property
def index_ratio(self) -> int:
return self._index_ratio
@property
def index_head_dim(self) -> int:
return self._index_head_dim
@property
def ring_capacity(self) -> int:
return self._ring_capacity
@property
def num_req_slots(self) -> int:
return self._num_req_slots
__all__ = ["QSAKVCache"]