Files
FreeToken/python/freetoken/models/config.py
Xiaoze Fan 08cee44468 feat(spec): add DFlash 2 speculative decoding (#857)
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal

* feat(engine): Sampler.probs and sparse-q rejection

* feat(spec): let a proposer declare slot states riding the GDN slots

* feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families

* feat(attention): record the source slot of each tracked request

* feat(kernel): DFlash2 draft kernels

* feat(models): DFlash2 draft family

* feat(spec): DFlashWindow

* feat(spec): DFlashProposer

* docs(cli): add dflash to the speculative decoding section

* refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs

* fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply

* refactor(spec): share the draft path check between the MTP and DFlash proposers

* test(e2e): compare rebuilt outputs exactly only without hybrid MoE

* refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config

* perf(spec): compute the verify distribution once for sparse-q rejection

* perf(models): let the feature sink add the residual on the rows it keeps

* perf(models): share the DFlash draft K/V weights with the ring projection

* fix(kernel): keep the dflash selector off zero-probability candidates

* test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover

* fix(spec): default and cap DFlash steps at the draft's block_size - 1

* fix(models): reject DFlash draft configs the draft forward cannot serve

* test(e2e): read the MoE strategy from the rebuild test's extra args

* docs(spec): say that slot-state owners write their own prefill track snapshot

* test(models): move the DFlash draft parity tests under tests/models/dflash

* refactor(spec): drop the no-op verify handler of the DFlash window

* refactor(models): fold the DFlash candidate top-k into propose

* refactor(spec): declare the draft step limits through Proposer.num_steps only

* refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring

* refactor(models): name the DFlash draft submodules like the other model families

* refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py

* refactor(spec): hand the verify the draft distribution over the whole vocabulary

* refactor(spec): call the DFlashRing object dflash_ring and its length ring_len

* refactor(models): split the DFlash draft's propose into compute_candidates and sample_path

* refactor(spec): keep the draft distribution on its support in SpecProposal

* fix(spec): drop the step's proposal once the verify has read it

* refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink

* refactor(models): read the declared Batch.features directly instead of through getattr

* test(dflash): check the window rebuild inside the block forward test

* test(spec): share the plain run of the verify contract and run the DFlash draft once per family

* test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes

* test(spec): check the verify probs of each row against a torch top-k and top-p reference

* test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate

* refactor(scheduler): let callers take the speculative counts from the status reporter

* test(spec): keep the shared plain runs of the verify contract in their own temp dirs

* test(models): give the ricursive test's stand-in batch a features field
2026-11-07 12:29:18 -08:00

553 lines
25 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
from typing import Any, ClassVar, Dict, List, Literal, Protocol, Tuple, TypeAlias
from freetoken.attention.base import AttnType
# State-dict key prefixes of the vision stack; load_weight drops them when the engine serves text-only.
VISION_KEY_PREFIXES = ("vision_tower.", "embed_vision.", "vision_embedder.", "visual.")
def detect_expert_quant(hf_config: Any) -> str:
"""Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for
a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export
(``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, or
``format: mixed-precision`` with an nvfp4 config group, e.g.
RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when
unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their
own detector."""
quant = getattr(hf_config, "quantization_config", None)
if quant is None:
return "none"
get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
algo = get("quant_algo") or get("quant_method")
if algo is None:
return "none"
if "fp4" in str(algo).lower():
return "nvfp4"
fmt = str(get("format") or "").lower()
# exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute
if "nvfp4" in fmt:
return "nvfp4"
# llm-compressor writes "mixed-precision" at the top when the groups differ (GLM-5.3-Flash: nvfp4 routed experts, fp8 MTP experts); the real format then sits in each group
if fmt == "mixed-precision":
groups = get("config_groups") or {}
groups = [g or {} for g in (groups.values() if isinstance(groups, dict) else [])]
# groups that target the experts decide; only a generic ["Linear"] group falls back to all of them
expert_groups = [g for g in groups if any("experts" in str(t) for t in (g.get("targets") or []))]
for g in expert_groups or groups:
if "nvfp4" in str(g.get("format") or "").lower():
return "nvfp4"
return str(algo).lower()
def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool:
"""Detect a compressed-tensors NVFP4 (W4A16) checkpoint (dense Qwen3.6-27B,
Muse-Glimmer-30B-NVFP4, ...).
llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups``
map whose ``weights`` are ``num_bits=4, type="float", group_size=16,
strategy="tensor_group"`` -- the NVFP4 layout (``weight_packed`` uint8 +
``weight_scale`` fp8 block + scalar ``weight_global_scale``). All ``targets:
["Linear"]`` are NVFP4 except the per-module ``ignore`` list. A 4-bit float scheme
with a DIFFERENT geometry (MXFP4: group_size 32, real since LLM Compressor 0.9)
raises instead of routing into the NVFP4 loader and dying in a shape assert."""
quant = getattr(hf_config, "quantization_config", None)
if quant is None:
return False
get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
if str(get("quant_method") or "").lower() != "compressed-tensors":
return False
groups = get("config_groups") or {}
# Verdicts are collected across ALL groups before returning: an early return on
# the first NVFP4 group would accept a mixed {nvfp4, mxfp4} checkpoint (and the
# error would depend on the groups' key order).
saw_nvfp4 = False
for g in (groups.values() if isinstance(groups, dict) else []):
w = (g or {}).get("weights") or {}
if int(w.get("num_bits", 0) or 0) != 4 or str(w.get("type", "")).lower() != "float":
continue
# vLLM's _is_nvfp4_format gates on the same two fields.
group_size = int(w.get("group_size", 0) or 0)
strategy = str(w.get("strategy", "")).lower()
if group_size == 16 and strategy == "tensor_group":
saw_nvfp4 = True
continue
raise ValueError(
"unsupported compressed-tensors 4-bit float scheme "
f"(group_size={group_size}, strategy={strategy!r}); FreeToken serves "
"NVFP4 (group_size=16, strategy=tensor_group) only"
)
return saw_nvfp4
@dataclass(frozen=True)
class RotaryConfig:
head_dim: int
rotary_dim: int
max_position: int
base: float
scaling: Dict[str, Any] | None
# 3-axis rope sections; None keeps the 1-D rope path, set only when the model serves vision
mrope_section: list | None = None
mrope_layout: str = "contiguous" # see freetoken.layers.rotary.build_section_table
@property
def table_positions(self) -> int:
"""Positions the cos/sin table covers: the longest sequence this rope can serve."""
from freetoken.layers.rotary import rope_table_positions
return rope_table_positions(self.max_position, self.scaling)
def mrope_layout_from_rope_params(rope_params: Any) -> str:
"""rope_parameters flags -> layout name: mrope_interleaved_glm, else mrope_interleaved, else contiguous."""
if rope_params.get("mrope_interleaved_glm"):
return "interleaved_glm"
if rope_params.get("mrope_interleaved"):
return "interleaved"
return "contiguous"
@dataclass(frozen=True)
class KVCacheGroupSpec:
name: str
layer_ids: Tuple[int, ...]
num_kv_heads: int
head_dim: int
sliding_window: int | None
# Latent-KV MLA group: the pool stores ONE latent slab per token (no separate V)
# and the cost model budgets 1x instead of 2x per token. When the model also runs
# DSA, ``index_head_dim``/``num_index_layers`` size the index-key slab, so the
# pool factory and the cost model derive the same bytes from the same spec.
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
# Grouped index-key compression: one index-key row per ``index_ratio`` tokens
# (QSA groups, glm5_next kpool pools; 1 keeps the per-token BSA/DSA slab). The
# pool factory and the KV cost model divide by the same value.
index_ratio: int = 1
# Attention-type taxonomy value for this group; drives the backend capability
# matrix and (with the pool factory) selects the KV pool family.
attn_type: AttnType = AttnType.FULL
@property
def num_layers(self) -> int:
return len(self.layer_ids)
@property
def is_swa(self) -> bool:
return self.name == "swa"
@dataclass(frozen=True)
class BaseAttentionGroupConfig:
name: str
layer_ids: Tuple[int, ...]
def owns_layer(self, layer_id: int) -> bool:
return layer_id in self.layer_ids
@dataclass(frozen=True)
class FullAttentionGroupConfig(BaseAttentionGroupConfig):
kind: ClassVar[Literal["full"]] = "full"
cache_kind: ClassVar[Literal["paged_kv"]] = "paged_kv"
num_kv_heads: int
head_dim: int
rotary_config: RotaryConfig
k_eq_v: bool = False
# Latent-KV MLA (single latent slab, V aliases K). With DSA, ``index_head_dim`` /
# ``num_index_layers`` size the index-key slab -- the pool factory and the KV cost
# model both read these, so they can never disagree.
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
# Grouped index-key compression ratio (see KVCacheGroupSpec.index_ratio).
# GQA + ratio > 1 -> AttnType.QSA (Qwen3.8); MLA + ratio > 1 -> the glm5_next
# kpool DSA layout (attn type stays DSA; the pool factory branches on mla).
index_ratio: int = 1
@dataclass(frozen=True)
class SWAAttentionGroupConfig(BaseAttentionGroupConfig):
kind: ClassVar[Literal["swa"]] = "swa"
cache_kind: ClassVar[Literal["swa_kv"]] = "swa_kv"
num_kv_heads: int
head_dim: int
rotary_config: RotaryConfig
sliding_window: int
# image token spans attend to each other in both directions on these layers
bidirectional_mm_blocks: bool = False
@dataclass(frozen=True)
class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig):
kind: ClassVar[Literal["linear_gated_delta"]] = "linear_gated_delta"
cache_kind: ClassVar[Literal["linear_state"]] = "linear_state"
num_key_heads: int
num_value_heads: int
key_head_dim: int
value_head_dim: int
conv_kernel_dim: int
# Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated.
output_gate: str
# "gdn" and "kda" share the same state geometry (one LinearStatePool serves
# both); the variant selects the kernels.
variant: Literal["gdn", "kda"] = "gdn"
@dataclass(frozen=True)
class DSV4AttentionGroupConfig(BaseAttentionGroupConfig):
"""DSV4 sparse attention (window + compressed tiers + Lightning Indexer). Standalone
class on purpose: subclassing SWAAttentionGroupConfig would flip has_swa_attention
and reroute DSV4 through the SWA gates. Geometry beyond the latent width lives in
dsv4_args (``dsv41_args`` for the ``v41`` variant); the pool prices itself from there,
not from this spec."""
kind: ClassVar[Literal["dsv4"]] = "dsv4"
cache_kind: ClassVar[Literal["dsv4_paged"]] = "dsv4_paged"
num_kv_heads: int
head_dim: int
sliding_window: int # the P-token window page
# "v41": DeepSeek-V4.1's cross-layer-shared packed tiers (AttnType.DSV41, its own pool and backend)
variant: Literal["v4", "v41"] = "v4"
# image token spans attend to each other in both directions, so prefill chunks end
# before a block instead of splitting it (the scheduler's keep_images_whole gate)
bidirectional_mm_blocks: bool = False
AttentionGroupConfig: TypeAlias = (
FullAttentionGroupConfig
| SWAAttentionGroupConfig
| LinearGatedDeltaGroupConfig
| DSV4AttentionGroupConfig
)
def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType:
# Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache,
# GQA (non-mla) + index slab -> QSAKVCache when the index keys are compressed
# (index_ratio > 1, Qwen3.8-Flash-Next) else BSAKVCache (MiniMax-M3 block-sparse).
if not group.mla:
if group.index_head_dim > 0 and group.num_index_layers > 0:
return AttnType.QSA if group.index_ratio > 1 else AttnType.BSA
return AttnType.FULL
if group.index_head_dim > 0 and group.num_index_layers > 0:
return AttnType.DSA
return AttnType.MLA
class SlotVerify(Protocol):
"""How a speculative verify step stages and commits one slot state; the state's owner supplies it and the pool only calls it.
The pool builds one per engine as ``cls(spec, max_bs, rows_per_req, device, dtype)``, ``dtype`` being the state's own, and keeps it across ``rebuild``.
A verify forward reads the live state only and writes what the commit needs into this handler's scratch; the owner gets it back with ``pool.verify_handler(spec.name)``.
Scratch row = verify-batch row, never a pool slot.
The scratch is single-buffered, which is safe because speculative decoding forces the normal loop.
A handler reads only its own scratch and writes only its own state, so the handlers commit in any order.
"""
@staticmethod
def nbytes(spec: SlotStateSpec, max_bs: int, rows_per_req: int, dtype) -> int:
"""Scratch bytes, from the spec alone, before any pool exists."""
...
def commit(self, pool, slots, counts) -> None:
"""Roll live slot ``slots[i]`` to where it stands after the first ``counts[i]`` rows of verify request i; int64 device tensors, one entry per request.
Reads the scratch without consuming it and may run more than once per step; reads the state through ``pool.slot_state`` on every call, since ``rebuild`` replaces it.
"""
...
@dataclass(frozen=True)
class SlotStateSpec:
"""One extra per-request tensor riding the LinearStatePool slots.
Allocated as ``[max(1, len(layer_ids)), num_slots, *shape]``; ``copy_from``, ``reset`` and ``rebuild`` cover it with the GDN state. The owner advances it in the forward and writes its own prefill track snapshot into ``fla_metadata.track_dst``, after the chunk's rows; it reads the state back through ``pool.slot_state(name, layer_id)``. ``shape`` is per slot and TP-replicated.
"""
name: str
shape: Tuple[int, ...]
layer_ids: Tuple[int, ...] = ()
dtype: Any | None = None # a torch dtype; None -> the pool's compute dtype
fill_value: float = 0.0
# None on a model's own state: speculative decoding cannot serve the model; a proposer's state may leave it None and keep itself right in post_verify
verify: type[SlotVerify] | None = None
@dataclass(frozen=True)
class ModelConfig:
num_layers: int
num_qo_heads: int
num_kv_heads: int
head_dim: int
hidden_size: int
vocab_size: int
intermediate_size: int
rms_norm_eps: float
rotary_config: RotaryConfig
hidden_act: str
tie_word_embeddings: bool
num_experts: int
num_experts_per_tok: int
moe_intermediate_size: int
norm_topk_prob: bool
model_type: str
architectures: list[str]
moe_strategy: str = "fused"
# where routed experts decode (gpu / cpu / hybrid); set by the engine from the flags, it gates which expert kernels can serve
decode_target: str = "gpu"
# The QuantConfig the engine builds from the checkpoint; layers ask it for their method.
quant: Any | None = None
# ----- optional, model-specific extensions (default keeps other models intact) -----
moe_enabled: bool = False
# Weight quantization of the MoE experts only. "none" keeps the default BF16
# offload/fused path; "nvfp4" stores experts as packed FP4 + block scales;
# "fp8_block" is DeepSeek-V3-style 128x128 block-fp8 (weight fp8-e4m3 +
# weight_scale_inv per block), also applied to the dense projections.
expert_quant: str = "none"
# Block size (out, in) for block-wise weight quantization (fp8_block: (128, 128)).
weight_block_size: tuple[int, int] | None = None
# the checkpoint's quant kind for the dense attention / GatedDeltaNet projections, detected by the family's parse_config for its reader
attn_quant: str = "none"
# the checkpoint's quant kind for the dense MLP projections (shared expert, dense layers), detected the same way
dense_quant: str = "none"
# the checkpoint's quant kind for the lm_head, detected the same way (only some NVFP4 exports quantize it)
lm_head_quant: str = "none"
shared_expert_intermediate_size: int = 0
use_qk_norm: bool = False
# ----- DeepSeek/GLM-style MoE extensions (default keeps other models intact) -----
# The first ``first_k_dense_replace`` decoder layers use a dense MLP instead of the
# sparse MoE block (GLM-4: 3). Experts (and the offload cache) therefore only exist
# for layers ``[first_k_dense_replace, num_layers)``.
first_k_dense_replace: int = 0
# Always-on shared expert(s) added to every MoE layer's output (GLM-4: 1).
n_shared_experts: int = 0
# Selected-expert weights are multiplied by this after renormalization (GLM-4: 2.5).
routed_scaling_factor: float = 1.0
# Group-limited routing (DeepSeek-V3 noaux_tc); GLM-4 uses n_group=topk_group=1 so the
# grouping is a no-op, but we keep the knobs for completeness.
n_group: int = 1
topk_group: int = 1
attn_sm_scale: float | None = None # None -> 1/sqrt(head_dim)
final_logit_softcapping: float | None = None
embedding_scale: float | None = None
# Second RMSNorm eps for models whose post-sublayer norms use a different eps than the
# pre-sublayer ones (muse_glimmer: post_attention/post_feedforward at 1e-8 vs 1e-5).
post_norm_eps: float | None = None
# Scalar the raw lm_head logits are multiplied by BEFORE final_logit_softcapping
# (muse_glimmer: 1/sqrt(hidden/256)); None leaves the logits unscaled.
output_multiplier: float | None = None
vision_config: Any | None = None
image_token_id: int | None = None
attention_groups: Tuple[AttentionGroupConfig, ...] = ()
has_attn_bias: bool = False
has_router_bias: bool = False
moe_weight_format: str | None = None
swiglu_limit: float | None = None
hidden_act_alpha: float = 1.702
# Full DeepseekV4Args payload for the DSV4-specific machinery (MLA sparse attention,
# CSA/HCA compressors, Lightning Indexer, manifold-constrained Hyper-Connections,
# hash routing). Opaque to model-agnostic engine code; None for non-DSV4 models.
dsv4_args: Any | None = None
# DeepSeek-V4.1 payload (DeepseekV41Args): DSV41 layer roles, Engram, replay policy. Opaque
# to model-agnostic engine code; None for every other model.
dsv41_args: Any | None = None
# GLM-5.2 (glm_moe_dsa) MLA/DSA payload (GlmMoeDsaArgs): the MLA low-rank dims and the
# DSA indexer geometry the model module needs. Opaque to model-agnostic engine code;
# None for every other model.
glm_dsa_args: Any | None = None
# GLM-5.3-Flash (glm5_next) payload (Glm5NextArgs): NoPE-MLA dims, the kpool indexer
# geometry, the KDA head config, and the mHC knobs. Opaque to model-agnostic engine
# code; None for every other model.
glm5_args: Any | None = None
# MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry
# (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the
# swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine
# code; None for every other model.
m3_args: Any | None = None
# Qwen3.8-Flash-Next (qwen4_exp) payload (Qwen4ExpArgs): hyper-connection widths, PLE
# n-gram embedding geometry and the QSA indexer scoring geometry the model module
# needs. Opaque to model-agnostic engine code; None for every other model.
qwen4_args: Any | None = None
# Generic execution-path capability flags (set by a model's parse_config) so the engine and
# factories stay model-agnostic instead of branching on dsv4_args:
single_stream_only: bool = False # model runs one sequence at a time -> force bs=1
# Extra per-request tensors riding the LinearStatePool slots (see SlotStateSpec);
# () for models without any. Requires a linear-attention group to ride on.
slot_states: Tuple[SlotStateSpec, ...] = ()
@property
def is_moe(self) -> bool:
return "moe" in self.model_type or self.moe_enabled
@property
def num_moe_layers(self) -> int:
"""Number of layers that own a sparse MoE block (and offload-cache expert slots).
Models with leading dense layers (``first_k_dense_replace`` > 0, e.g. GLM-4)
only store experts for the trailing layers; everything else has all layers MoE.
"""
return self.num_layers - self.first_k_dense_replace
@property
def is_multimodal(self) -> bool:
return self.vision_config is not None
@property
def model_is_mrope(self) -> bool:
"""True when any full-attention layer uses 3-axis (t/h/w) rope positions."""
return any(
getattr(getattr(g, "rotary_config", None), "mrope_section", None) is not None
for g in self.attention_groups
)
@property
def has_hybrid_attention(self) -> bool:
return len(self.attention_groups) > 1
@property
def has_swa_attention(self) -> bool:
return any(isinstance(group, SWAAttentionGroupConfig) for group in self.attention_groups)
@property
def has_linear_attention(self) -> bool:
return any(
isinstance(group, LinearGatedDeltaGroupConfig)
for group in self.attention_groups
)
def default_full_attention_group(self) -> FullAttentionGroupConfig:
return FullAttentionGroupConfig(
name="full",
layer_ids=tuple(range(self.num_layers)),
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
rotary_config=self.rotary_config,
)
def attention_group_for_layer(self, layer_id: int) -> AttentionGroupConfig:
groups = self.attention_groups or (self.default_full_attention_group(),)
matches = [group for group in groups if group.owns_layer(layer_id)]
if len(matches) != 1:
raise ValueError(
f"Expected exactly one attention group for layer {layer_id}, "
f"got {len(matches)}"
)
return matches[0]
def swa_attention_group(self) -> SWAAttentionGroupConfig | None:
groups = [
group
for group in self.attention_groups
if isinstance(group, SWAAttentionGroupConfig)
]
if len(groups) > 1:
raise ValueError("Expected at most one SWA attention group")
return groups[0] if groups else None
def linear_attention_group(self) -> LinearGatedDeltaGroupConfig | None:
groups = [
group
for group in self.attention_groups
if isinstance(group, LinearGatedDeltaGroupConfig)
]
if len(groups) > 1:
raise ValueError("Expected at most one linear-gated-delta attention group")
return groups[0] if groups else None
def is_swa_layer(self, layer_id: int) -> bool:
return isinstance(self.attention_group_for_layer(layer_id), SWAAttentionGroupConfig)
def is_linear_layer(self, layer_id: int) -> bool:
return isinstance(
self.attention_group_for_layer(layer_id),
LinearGatedDeltaGroupConfig,
)
def attn_type_for_layer(self, layer_id: int) -> AttnType:
"""Canonical per-layer attention-type lookup (the taxonomy is declared
top-down on the attention groups; this is the layer-granular view)."""
group = self.attention_group_for_layer(layer_id)
if isinstance(group, LinearGatedDeltaGroupConfig):
return AttnType.LINEAR
if isinstance(group, SWAAttentionGroupConfig):
return AttnType.SWA
if isinstance(group, DSV4AttentionGroupConfig):
return AttnType.DSV41 if group.variant == "v41" else AttnType.DSV4
return _full_group_attn_type(group)
def kv_cache_group_specs(self) -> Tuple[KVCacheGroupSpec, ...]:
if not self.attention_groups:
return (
KVCacheGroupSpec(
name="full",
layer_ids=tuple(range(self.num_layers)),
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
sliding_window=None,
),
)
specs: list[KVCacheGroupSpec] = []
for group in self.attention_groups:
if isinstance(group, FullAttentionGroupConfig):
specs.append(
KVCacheGroupSpec(
name=group.name,
layer_ids=group.layer_ids,
num_kv_heads=group.num_kv_heads,
head_dim=group.head_dim,
sliding_window=None,
mla=group.mla,
index_head_dim=group.index_head_dim,
num_index_layers=group.num_index_layers,
index_ratio=group.index_ratio,
attn_type=_full_group_attn_type(group),
)
)
elif isinstance(group, SWAAttentionGroupConfig):
specs.append(
KVCacheGroupSpec(
name=group.name,
layer_ids=group.layer_ids,
num_kv_heads=group.num_kv_heads,
head_dim=group.head_dim,
sliding_window=group.sliding_window,
attn_type=AttnType.SWA,
)
)
elif isinstance(group, DSV4AttentionGroupConfig):
# Matrix/taxonomy entry only: DSV4 sizing never reads this spec
# (the pool prices itself from dsv4_args), and is_swa/mla stay
# False so no generic spec walker treats it as SWA or MLA.
specs.append(
KVCacheGroupSpec(
name=group.name,
layer_ids=group.layer_ids,
num_kv_heads=group.num_kv_heads,
head_dim=group.head_dim,
sliding_window=group.sliding_window,
attn_type=AttnType.DSV41 if group.variant == "v41" else AttnType.DSV4,
)
)
return tuple(specs)
def kv_cache_groups(self) -> List[Tuple[int, int, int]]:
"""Return ``[(num_layers, num_kv_heads, head_dim), ...]`` per KV group.
This legacy memory-accounting helper is derived from the SGLang-style
group specs used by hybrid full/SWA KV cache storage.
"""
return [
(group.num_layers, group.num_kv_heads, group.head_dim)
for group in self.kv_cache_group_specs()
if group.num_layers > 0
]