from __future__ import annotations from dataclasses import dataclass from typing import Any, ClassVar, Dict, List, Literal, Protocol, Tuple, TypeAlias from freetoken.attention.base import AttnType # State-dict key prefixes of the vision stack; load_weight drops them when the engine serves text-only. VISION_KEY_PREFIXES = ("vision_tower.", "embed_vision.", "vision_embedder.", "visual.") def detect_expert_quant(hf_config: Any) -> str: """Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, or ``format: mixed-precision`` with an nvfp4 config group, e.g. RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their own detector.""" quant = getattr(hf_config, "quantization_config", None) if quant is None: return "none" get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d)) algo = get("quant_algo") or get("quant_method") if algo is None: return "none" if "fp4" in str(algo).lower(): return "nvfp4" fmt = str(get("format") or "").lower() # exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute if "nvfp4" in fmt: return "nvfp4" # llm-compressor writes "mixed-precision" at the top when the groups differ (GLM-5.3-Flash: nvfp4 routed experts, fp8 MTP experts); the real format then sits in each group if fmt == "mixed-precision": groups = get("config_groups") or {} groups = [g or {} for g in (groups.values() if isinstance(groups, dict) else [])] # groups that target the experts decide; only a generic ["Linear"] group falls back to all of them expert_groups = [g for g in groups if any("experts" in str(t) for t in (g.get("targets") or []))] for g in expert_groups or groups: if "nvfp4" in str(g.get("format") or "").lower(): return "nvfp4" return str(algo).lower() def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool: """Detect a compressed-tensors NVFP4 (W4A16) checkpoint (dense Qwen3.6-27B, Muse-Glimmer-30B-NVFP4, ...). llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups`` map whose ``weights`` are ``num_bits=4, type="float", group_size=16, strategy="tensor_group"`` -- the NVFP4 layout (``weight_packed`` uint8 + ``weight_scale`` fp8 block + scalar ``weight_global_scale``). All ``targets: ["Linear"]`` are NVFP4 except the per-module ``ignore`` list. A 4-bit float scheme with a DIFFERENT geometry (MXFP4: group_size 32, real since LLM Compressor 0.9) raises instead of routing into the NVFP4 loader and dying in a shape assert.""" quant = getattr(hf_config, "quantization_config", None) if quant is None: return False get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d)) if str(get("quant_method") or "").lower() != "compressed-tensors": return False groups = get("config_groups") or {} # Verdicts are collected across ALL groups before returning: an early return on # the first NVFP4 group would accept a mixed {nvfp4, mxfp4} checkpoint (and the # error would depend on the groups' key order). saw_nvfp4 = False for g in (groups.values() if isinstance(groups, dict) else []): w = (g or {}).get("weights") or {} if int(w.get("num_bits", 0) or 0) != 4 or str(w.get("type", "")).lower() != "float": continue # vLLM's _is_nvfp4_format gates on the same two fields. group_size = int(w.get("group_size", 0) or 0) strategy = str(w.get("strategy", "")).lower() if group_size == 16 and strategy == "tensor_group": saw_nvfp4 = True continue raise ValueError( "unsupported compressed-tensors 4-bit float scheme " f"(group_size={group_size}, strategy={strategy!r}); FreeToken serves " "NVFP4 (group_size=16, strategy=tensor_group) only" ) return saw_nvfp4 @dataclass(frozen=True) class RotaryConfig: head_dim: int rotary_dim: int max_position: int base: float scaling: Dict[str, Any] | None # 3-axis rope sections; None keeps the 1-D rope path, set only when the model serves vision mrope_section: list | None = None mrope_layout: str = "contiguous" # see freetoken.layers.rotary.build_section_table @property def table_positions(self) -> int: """Positions the cos/sin table covers: the longest sequence this rope can serve.""" from freetoken.layers.rotary import rope_table_positions return rope_table_positions(self.max_position, self.scaling) def mrope_layout_from_rope_params(rope_params: Any) -> str: """rope_parameters flags -> layout name: mrope_interleaved_glm, else mrope_interleaved, else contiguous.""" if rope_params.get("mrope_interleaved_glm"): return "interleaved_glm" if rope_params.get("mrope_interleaved"): return "interleaved" return "contiguous" @dataclass(frozen=True) class KVCacheGroupSpec: name: str layer_ids: Tuple[int, ...] num_kv_heads: int head_dim: int sliding_window: int | None # Latent-KV MLA group: the pool stores ONE latent slab per token (no separate V) # and the cost model budgets 1x instead of 2x per token. When the model also runs # DSA, ``index_head_dim``/``num_index_layers`` size the index-key slab, so the # pool factory and the cost model derive the same bytes from the same spec. mla: bool = False index_head_dim: int = 0 num_index_layers: int = 0 # Grouped index-key compression: one index-key row per ``index_ratio`` tokens # (QSA groups, glm5_next kpool pools; 1 keeps the per-token BSA/DSA slab). The # pool factory and the KV cost model divide by the same value. index_ratio: int = 1 # Attention-type taxonomy value for this group; drives the backend capability # matrix and (with the pool factory) selects the KV pool family. attn_type: AttnType = AttnType.FULL @property def num_layers(self) -> int: return len(self.layer_ids) @property def is_swa(self) -> bool: return self.name == "swa" @dataclass(frozen=True) class BaseAttentionGroupConfig: name: str layer_ids: Tuple[int, ...] def owns_layer(self, layer_id: int) -> bool: return layer_id in self.layer_ids @dataclass(frozen=True) class FullAttentionGroupConfig(BaseAttentionGroupConfig): kind: ClassVar[Literal["full"]] = "full" cache_kind: ClassVar[Literal["paged_kv"]] = "paged_kv" num_kv_heads: int head_dim: int rotary_config: RotaryConfig k_eq_v: bool = False # Latent-KV MLA (single latent slab, V aliases K). With DSA, ``index_head_dim`` / # ``num_index_layers`` size the index-key slab -- the pool factory and the KV cost # model both read these, so they can never disagree. mla: bool = False index_head_dim: int = 0 num_index_layers: int = 0 # Grouped index-key compression ratio (see KVCacheGroupSpec.index_ratio). # GQA + ratio > 1 -> AttnType.QSA (Qwen3.8); MLA + ratio > 1 -> the glm5_next # kpool DSA layout (attn type stays DSA; the pool factory branches on mla). index_ratio: int = 1 @dataclass(frozen=True) class SWAAttentionGroupConfig(BaseAttentionGroupConfig): kind: ClassVar[Literal["swa"]] = "swa" cache_kind: ClassVar[Literal["swa_kv"]] = "swa_kv" num_kv_heads: int head_dim: int rotary_config: RotaryConfig sliding_window: int # image token spans attend to each other in both directions on these layers bidirectional_mm_blocks: bool = False @dataclass(frozen=True) class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig): kind: ClassVar[Literal["linear_gated_delta"]] = "linear_gated_delta" cache_kind: ClassVar[Literal["linear_state"]] = "linear_state" num_key_heads: int num_value_heads: int key_head_dim: int value_head_dim: int conv_kernel_dim: int # Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated. output_gate: str # "gdn" and "kda" share the same state geometry (one LinearStatePool serves # both); the variant selects the kernels. variant: Literal["gdn", "kda"] = "gdn" @dataclass(frozen=True) class DSV4AttentionGroupConfig(BaseAttentionGroupConfig): """DSV4 sparse attention (window + compressed tiers + Lightning Indexer). Standalone class on purpose: subclassing SWAAttentionGroupConfig would flip has_swa_attention and reroute DSV4 through the SWA gates. Geometry beyond the latent width lives in dsv4_args (``dsv41_args`` for the ``v41`` variant); the pool prices itself from there, not from this spec.""" kind: ClassVar[Literal["dsv4"]] = "dsv4" cache_kind: ClassVar[Literal["dsv4_paged"]] = "dsv4_paged" num_kv_heads: int head_dim: int sliding_window: int # the P-token window page # "v41": DeepSeek-V4.1's cross-layer-shared packed tiers (AttnType.DSV41, its own pool and backend) variant: Literal["v4", "v41"] = "v4" # image token spans attend to each other in both directions, so prefill chunks end # before a block instead of splitting it (the scheduler's keep_images_whole gate) bidirectional_mm_blocks: bool = False AttentionGroupConfig: TypeAlias = ( FullAttentionGroupConfig | SWAAttentionGroupConfig | LinearGatedDeltaGroupConfig | DSV4AttentionGroupConfig ) def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType: # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache, # GQA (non-mla) + index slab -> QSAKVCache when the index keys are compressed # (index_ratio > 1, Qwen3.8-Flash-Next) else BSAKVCache (MiniMax-M3 block-sparse). if not group.mla: if group.index_head_dim > 0 and group.num_index_layers > 0: return AttnType.QSA if group.index_ratio > 1 else AttnType.BSA return AttnType.FULL if group.index_head_dim > 0 and group.num_index_layers > 0: return AttnType.DSA return AttnType.MLA class SlotVerify(Protocol): """How a speculative verify step stages and commits one slot state; the state's owner supplies it and the pool only calls it. The pool builds one per engine as ``cls(spec, max_bs, rows_per_req, device, dtype)``, ``dtype`` being the state's own, and keeps it across ``rebuild``. A verify forward reads the live state only and writes what the commit needs into this handler's scratch; the owner gets it back with ``pool.verify_handler(spec.name)``. Scratch row = verify-batch row, never a pool slot. The scratch is single-buffered, which is safe because speculative decoding forces the normal loop. A handler reads only its own scratch and writes only its own state, so the handlers commit in any order. """ @staticmethod def nbytes(spec: SlotStateSpec, max_bs: int, rows_per_req: int, dtype) -> int: """Scratch bytes, from the spec alone, before any pool exists.""" ... def commit(self, pool, slots, counts) -> None: """Roll live slot ``slots[i]`` to where it stands after the first ``counts[i]`` rows of verify request i; int64 device tensors, one entry per request. Reads the scratch without consuming it and may run more than once per step; reads the state through ``pool.slot_state`` on every call, since ``rebuild`` replaces it. """ ... @dataclass(frozen=True) class SlotStateSpec: """One extra per-request tensor riding the LinearStatePool slots. Allocated as ``[max(1, len(layer_ids)), num_slots, *shape]`` and advanced, snapshot, COW'd and rebuilt with the GDN state; the owner reads it back through ``pool.slot_state(name, layer_id)``. ``shape`` is per slot and TP-replicated. """ name: str shape: Tuple[int, ...] layer_ids: Tuple[int, ...] = () dtype: Any | None = None # a torch dtype; None -> the pool's compute dtype fill_value: float = 0.0 # None: speculative decoding cannot serve a model carrying this state verify: type[SlotVerify] | None = None @dataclass(frozen=True) class ModelConfig: num_layers: int num_qo_heads: int num_kv_heads: int head_dim: int hidden_size: int vocab_size: int intermediate_size: int rms_norm_eps: float rotary_config: RotaryConfig hidden_act: str tie_word_embeddings: bool num_experts: int num_experts_per_tok: int moe_intermediate_size: int norm_topk_prob: bool model_type: str architectures: list[str] moe_strategy: str = "fused" # where routed experts decode (gpu / cpu / hybrid); set by the engine from the flags, it gates which expert kernels can serve decode_target: str = "gpu" # The QuantConfig the engine builds from the checkpoint; layers ask it for their method. quant: Any | None = None # ----- optional, model-specific extensions (default keeps other models intact) ----- moe_enabled: bool = False # Weight quantization of the MoE experts only. "none" keeps the default BF16 # offload/fused path; "nvfp4" stores experts as packed FP4 + block scales; # "fp8_block" is DeepSeek-V3-style 128x128 block-fp8 (weight fp8-e4m3 + # weight_scale_inv per block), also applied to the dense projections. expert_quant: str = "none" # Block size (out, in) for block-wise weight quantization (fp8_block: (128, 128)). weight_block_size: tuple[int, int] | None = None # the checkpoint's quant kind for the dense attention / GatedDeltaNet projections, detected by the family's parse_config for its reader attn_quant: str = "none" # the checkpoint's quant kind for the dense MLP projections (shared expert, dense layers), detected the same way dense_quant: str = "none" # the checkpoint's quant kind for the lm_head, detected the same way (only some NVFP4 exports quantize it) lm_head_quant: str = "none" shared_expert_intermediate_size: int = 0 use_qk_norm: bool = False # ----- DeepSeek/GLM-style MoE extensions (default keeps other models intact) ----- # The first ``first_k_dense_replace`` decoder layers use a dense MLP instead of the # sparse MoE block (GLM-4: 3). Experts (and the offload cache) therefore only exist # for layers ``[first_k_dense_replace, num_layers)``. first_k_dense_replace: int = 0 # Always-on shared expert(s) added to every MoE layer's output (GLM-4: 1). n_shared_experts: int = 0 # Selected-expert weights are multiplied by this after renormalization (GLM-4: 2.5). routed_scaling_factor: float = 1.0 # Group-limited routing (DeepSeek-V3 noaux_tc); GLM-4 uses n_group=topk_group=1 so the # grouping is a no-op, but we keep the knobs for completeness. n_group: int = 1 topk_group: int = 1 attn_sm_scale: float | None = None # None -> 1/sqrt(head_dim) final_logit_softcapping: float | None = None embedding_scale: float | None = None # Second RMSNorm eps for models whose post-sublayer norms use a different eps than the # pre-sublayer ones (muse_glimmer: post_attention/post_feedforward at 1e-8 vs 1e-5). post_norm_eps: float | None = None # Scalar the raw lm_head logits are multiplied by BEFORE final_logit_softcapping # (muse_glimmer: 1/sqrt(hidden/256)); None leaves the logits unscaled. output_multiplier: float | None = None vision_config: Any | None = None image_token_id: int | None = None attention_groups: Tuple[AttentionGroupConfig, ...] = () has_attn_bias: bool = False has_router_bias: bool = False moe_weight_format: str | None = None swiglu_limit: float | None = None hidden_act_alpha: float = 1.702 # Full DeepseekV4Args payload for the DSV4-specific machinery (MLA sparse attention, # CSA/HCA compressors, Lightning Indexer, manifold-constrained Hyper-Connections, # hash routing). Opaque to model-agnostic engine code; None for non-DSV4 models. dsv4_args: Any | None = None # DeepSeek-V4.1 payload (DeepseekV41Args): DSV41 layer roles, Engram, replay policy. Opaque # to model-agnostic engine code; None for every other model. dsv41_args: Any | None = None # GLM-5.2 (glm_moe_dsa) MLA/DSA payload (GlmMoeDsaArgs): the MLA low-rank dims and the # DSA indexer geometry the model module needs. Opaque to model-agnostic engine code; # None for every other model. glm_dsa_args: Any | None = None # GLM-5.3-Flash (glm5_next) payload (Glm5NextArgs): NoPE-MLA dims, the kpool indexer # geometry, the KDA head config, and the mHC knobs. Opaque to model-agnostic engine # code; None for every other model. glm5_args: Any | None = None # MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry # (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the # swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine # code; None for every other model. m3_args: Any | None = None # Qwen3.8-Flash-Next (qwen4_exp) payload (Qwen4ExpArgs): hyper-connection widths, PLE # n-gram embedding geometry and the QSA indexer scoring geometry the model module # needs. Opaque to model-agnostic engine code; None for every other model. qwen4_args: Any | None = None # Generic execution-path capability flags (set by a model's parse_config) so the engine and # factories stay model-agnostic instead of branching on dsv4_args: single_stream_only: bool = False # model runs one sequence at a time -> force bs=1 # Extra per-request tensors riding the LinearStatePool slots (see SlotStateSpec); # () for models without any. Requires a linear-attention group to ride on. slot_states: Tuple[SlotStateSpec, ...] = () @property def is_moe(self) -> bool: return "moe" in self.model_type or self.moe_enabled @property def num_moe_layers(self) -> int: """Number of layers that own a sparse MoE block (and offload-cache expert slots). Models with leading dense layers (``first_k_dense_replace`` > 0, e.g. GLM-4) only store experts for the trailing layers; everything else has all layers MoE. """ return self.num_layers - self.first_k_dense_replace @property def is_multimodal(self) -> bool: return self.vision_config is not None @property def model_is_mrope(self) -> bool: """True when any full-attention layer uses 3-axis (t/h/w) rope positions.""" return any( getattr(getattr(g, "rotary_config", None), "mrope_section", None) is not None for g in self.attention_groups ) @property def has_hybrid_attention(self) -> bool: return len(self.attention_groups) > 1 @property def has_swa_attention(self) -> bool: return any(isinstance(group, SWAAttentionGroupConfig) for group in self.attention_groups) @property def has_linear_attention(self) -> bool: return any( isinstance(group, LinearGatedDeltaGroupConfig) for group in self.attention_groups ) def default_full_attention_group(self) -> FullAttentionGroupConfig: return FullAttentionGroupConfig( name="full", layer_ids=tuple(range(self.num_layers)), num_kv_heads=self.num_kv_heads, head_dim=self.head_dim, rotary_config=self.rotary_config, ) def attention_group_for_layer(self, layer_id: int) -> AttentionGroupConfig: groups = self.attention_groups or (self.default_full_attention_group(),) matches = [group for group in groups if group.owns_layer(layer_id)] if len(matches) != 1: raise ValueError( f"Expected exactly one attention group for layer {layer_id}, " f"got {len(matches)}" ) return matches[0] def swa_attention_group(self) -> SWAAttentionGroupConfig | None: groups = [ group for group in self.attention_groups if isinstance(group, SWAAttentionGroupConfig) ] if len(groups) > 1: raise ValueError("Expected at most one SWA attention group") return groups[0] if groups else None def linear_attention_group(self) -> LinearGatedDeltaGroupConfig | None: groups = [ group for group in self.attention_groups if isinstance(group, LinearGatedDeltaGroupConfig) ] if len(groups) > 1: raise ValueError("Expected at most one linear-gated-delta attention group") return groups[0] if groups else None def is_swa_layer(self, layer_id: int) -> bool: return isinstance(self.attention_group_for_layer(layer_id), SWAAttentionGroupConfig) def is_linear_layer(self, layer_id: int) -> bool: return isinstance( self.attention_group_for_layer(layer_id), LinearGatedDeltaGroupConfig, ) def attn_type_for_layer(self, layer_id: int) -> AttnType: """Canonical per-layer attention-type lookup (the taxonomy is declared top-down on the attention groups; this is the layer-granular view).""" group = self.attention_group_for_layer(layer_id) if isinstance(group, LinearGatedDeltaGroupConfig): return AttnType.LINEAR if isinstance(group, SWAAttentionGroupConfig): return AttnType.SWA if isinstance(group, DSV4AttentionGroupConfig): return AttnType.DSV41 if group.variant == "v41" else AttnType.DSV4 return _full_group_attn_type(group) def kv_cache_group_specs(self) -> Tuple[KVCacheGroupSpec, ...]: if not self.attention_groups: return ( KVCacheGroupSpec( name="full", layer_ids=tuple(range(self.num_layers)), num_kv_heads=self.num_kv_heads, head_dim=self.head_dim, sliding_window=None, ), ) specs: list[KVCacheGroupSpec] = [] for group in self.attention_groups: if isinstance(group, FullAttentionGroupConfig): specs.append( KVCacheGroupSpec( name=group.name, layer_ids=group.layer_ids, num_kv_heads=group.num_kv_heads, head_dim=group.head_dim, sliding_window=None, mla=group.mla, index_head_dim=group.index_head_dim, num_index_layers=group.num_index_layers, index_ratio=group.index_ratio, attn_type=_full_group_attn_type(group), ) ) elif isinstance(group, SWAAttentionGroupConfig): specs.append( KVCacheGroupSpec( name=group.name, layer_ids=group.layer_ids, num_kv_heads=group.num_kv_heads, head_dim=group.head_dim, sliding_window=group.sliding_window, attn_type=AttnType.SWA, ) ) elif isinstance(group, DSV4AttentionGroupConfig): # Matrix/taxonomy entry only: DSV4 sizing never reads this spec # (the pool prices itself from dsv4_args), and is_swa/mla stay # False so no generic spec walker treats it as SWA or MLA. specs.append( KVCacheGroupSpec( name=group.name, layer_ids=group.layer_ids, num_kv_heads=group.num_kv_heads, head_dim=group.head_dim, sliding_window=group.sliding_window, attn_type=AttnType.DSV41 if group.variant == "v41" else AttnType.DSV4, ) ) return tuple(specs) def kv_cache_groups(self) -> List[Tuple[int, int, int]]: """Return ``[(num_layers, num_kv_heads, head_dim), ...]`` per KV group. This legacy memory-accounting helper is derived from the SGLang-style group specs used by hybrid full/SWA KV cache storage. """ return [ (group.num_layers, group.num_kv_heads, group.head_dim) for group in self.kv_cache_group_specs() if group.num_layers > 0 ]