Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

226 lines
9.6 KiB
Python

"""DeepSeek-V4.1-Flash hyperparameters and the static per-layer attention roles.
``DeepseekV41Args`` mirrors the reference ``ModelArgs`` (``inference/model.py``) field for field,
read from the HF ``config.json``'s ``text_config`` (the ``inference/config.json`` dialect is also
accepted). Everything the engine reconciles at runtime (``max_seq_len``, the replay policy) is
attached by ``parse_config`` / the engine, not read from the checkpoint.
``LayerRole`` is the attention mode table (tech report sec. 2.3.1): every compressing layer is Full
(owns the compressed KV + indexer K), Reindex (own indexer over a shared K) or Reuse (shared
Top-K); the candidate pool of the Hierarchical Sparse Indexer (sec. 2.3.2) is produced by the
first decoder Full layer and consumed by the decoder Reindex layers after it.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Any, Literal
# text_config key -> ModelArgs key, where the two dialects differ
_HF_TO_ARGS = {
"hidden_size": "dim",
"moe_intermediate_size": "moe_inter_dim",
"num_hidden_layers": "n_layers",
"num_nextn_predict_layers": "n_mtp_layers",
"num_attention_heads": "n_heads",
"num_experts_per_tok": "n_activated_experts",
"scoring_func": "score_func",
"routed_scaling_factor": "route_scale",
"qk_rope_head_dim": "rope_head_dim",
"rms_norm_eps": "norm_eps",
"sliding_window": "window_size",
"kv_source_layer_ids": "kv_source_layers",
"index_source_layer_ids": "index_source_layers",
"candidate_source_layer_id": "candidate_source_layer",
"engram_pad_token_id": "engram_pad_id",
"dspark_num_experts_per_tok": "dspark_n_activated_experts",
}
class Mode(str, Enum):
WINDOW = "window" # sliding window only (compress_ratio == 0)
FULL = "full" # computes main KV + indexer K, runs its indexer
REINDEX = "reindex" # reuses main KV + indexer K, runs its own indexer
REUSE = "reuse" # reuses main KV and the latest Top-K
@dataclass(frozen=True)
class LayerRole:
layer_id: int
ratio: int
mode: Mode
kv_source: int | None # the layer whose main KV / indexer K this layer reads (itself when Full)
is_candidate_source: bool # builds the hierarchical candidate pool
uses_candidates: bool # indexes only inside the candidate pool
@property
def compresses(self) -> bool:
return self.ratio > 0
@property
def has_indexer(self) -> bool:
return self.mode in (Mode.FULL, Mode.REINDEX)
@dataclass
class DeepseekV41Args:
vocab_size: int = 129280
dim: int = 5120
moe_inter_dim: int = 2304
n_layers: int = 40
n_mtp_layers: int = 3
n_heads: int = 64
# moe
n_routed_experts: int = 384
n_shared_experts: int = 1
n_activated_experts: int = 6
score_func: Literal["softmax", "sigmoid", "sqrtsoftplus"] = "sqrtsoftplus"
norm_topk_prob: bool = True
route_scale: float = 1.5
swiglu_limit: float = 10.0
# attention: latent q, one shared latent kv head, grouped low-rank output projection
q_lora_rank: int = 1280
head_dim: int = 512
rope_head_dim: int = 64
norm_eps: float = 1e-20
o_groups: int = 8
o_lora_rank: int = 1024
# sparse attention
window_size: int = 128
compress_ratios: tuple[int, ...] = () # one per layer, MTP layers included
kv_source_layers: tuple[int, ...] = ()
index_source_layers: tuple[int, ...] = ()
compress_rope_theta: float = 160000.0
original_seq_len: int = 65536
rope_theta: float = 10000.0
rope_factor: float = 16.0
beta_fast: int = 32
beta_slow: int = 1
index_n_heads: int = 32
index_head_dim: int = 128
index_topk: int = 512
candidate_source_layer: int = -1
candidate_topk_blocks: int = 0
candidate_block_size: int = 0
# hyper-connections (single-pass mHC)
hc_mult: int = 4
hc_sinkhorn_iters: int = 20
hc_eps: float = 1e-6
# engram
engram_layer_ids: tuple[int, ...] = ()
engram_num_embeddings: tuple[int, ...] = ()
engram_max_ngram_size: int = 4
engram_vocab_size: int = 0
engram_n_heads: int = 0
engram_head_dim: int = 0
engram_pad_id: int = 2
engram_compressed_vocab_size: int = 0
# dspark (weights are not served in v1; kept so the config round-trips)
dspark_block_size: int = 0
dspark_target_layer_ids: tuple[int, ...] = ()
# ---- runtime (engine-reconciled, never read from the checkpoint) ----
max_seq_len: int = 4096
max_batch_size: int = 1
# "bounded": the decoder runs on the prompt's last window_size tokens with the sliding window
# truncated there (Decoder SWA Bounded Replay, tech report sec. 3.2.2); "exact": on every token.
swa_decoder_replay: Literal["bounded", "exact"] = "bounded"
roles: tuple[LayerRole, ...] = field(default_factory=tuple, repr=False)
def __post_init__(self) -> None:
for name in ("compress_ratios", "kv_source_layers", "index_source_layers", "engram_layer_ids",
"engram_num_embeddings", "dspark_target_layer_ids"):
setattr(self, name, tuple(int(v) for v in getattr(self, name)))
if len(self.compress_ratios) < self.n_layers:
raise ValueError(f"compress_ratios has {len(self.compress_ratios)} entries for {self.n_layers} layers")
if self.n_shared_experts != 1:
raise ValueError("DeepSeek-V4.1 has exactly one shared expert")
if self.hc_mult < 1:
raise ValueError("hc_mult must be positive")
self.roles = tuple(self._role(layer) for layer in range(self.n_layers))
if len(self.engram_layer_ids) != len(self.engram_num_embeddings):
raise ValueError("engram_layer_ids and engram_num_embeddings differ in length")
# ---- derived ----
def _role(self, layer: int) -> LayerRole:
ratio = self.compress_ratios[layer]
if ratio == 0:
return LayerRole(layer, 0, Mode.WINDOW, None, False, False)
sources = [s for s in self.kv_source_layers if s <= layer]
if not sources:
raise ValueError(f"layer {layer} compresses but no kv source precedes it")
source = max(sources)
if self.compress_ratios[source] != ratio:
raise ValueError(f"layer {layer} (ratio {ratio}) reads kv source {source} (ratio {self.compress_ratios[source]})")
if layer == source:
mode = Mode.FULL
elif layer in self.index_source_layers:
mode = Mode.REINDEX
else:
mode = Mode.REUSE
cand = self.candidate_source_layer
is_cand_src = cand >= 0 and layer == cand
uses = cand >= 0 and cand < layer and mode is Mode.REINDEX
if uses and self.kv_source_layers and max(s for s in self.kv_source_layers if s <= cand) != source:
raise ValueError(f"layer {layer} would index the candidate pool of layer {cand} over a different kv source")
return LayerRole(layer, ratio, mode, source, is_cand_src, uses)
@property
def nope_head_dim(self) -> int:
return self.head_dim - self.rope_head_dim
@property
def backbone_kv_sources(self) -> tuple[int, ...]:
return tuple(s for s in self.kv_source_layers if s < self.n_layers)
@property
def decoder_start_layer(self) -> int:
"""The first decoder layer: the last kv source (CED projects the decoder's global KV from
its input, the final encoder hidden state)."""
return max(self.backbone_kv_sources) if self.backbone_kv_sources else self.n_layers
@property
def engram_hash_cols(self) -> int:
return (self.engram_max_ngram_size - 1) * self.engram_n_heads
@property
def index_softmax_scale(self) -> float:
return self.index_head_dim**-0.5
def freqs_params(self, layer: int) -> tuple[int, int, float, float, int, int]:
"""``(rotary_dim, original_seq_len, theta, factor, beta_fast, beta_slow)``: compressing layers
rotate at the compressed theta under YaRN, window-only layers at the base theta without it."""
if self.compress_ratios[layer]:
return self.rope_head_dim, self.original_seq_len, self.compress_rope_theta, self.rope_factor, self.beta_fast, self.beta_slow
return self.rope_head_dim, 0, self.rope_theta, self.rope_factor, self.beta_fast, self.beta_slow
@classmethod
def from_hf(cls, hf_config: Any) -> "DeepseekV41Args":
"""From the HF config (``text_config`` of the multimodal wrapper, or the text config itself,
or the ``inference/config.json`` dialect)."""
text = getattr(hf_config, "text_config", None) or hf_config
if isinstance(text, dict):
raw = text
elif hasattr(text, "to_dict"): # PretrainedConfig / RawConfigShim
raw = text.to_dict()
else:
raw = vars(text)
names = {f.name for f in cls.__dataclass_fields__.values()} - {"roles", "max_seq_len", "max_batch_size", "swa_decoder_replay"}
kwargs: dict[str, Any] = {}
for key, value in raw.items():
key = _HF_TO_ARGS.get(key, key)
if key in names:
kwargs[key] = value
# HF nests the YaRN parameters; the inference dialect keeps them flat
scaling = raw.get("rope_scaling") or raw.get("rope_parameters")
if isinstance(scaling, dict):
if "factor" in scaling:
kwargs["rope_factor"] = scaling["factor"]
for src, dst in (("beta_fast", "beta_fast"), ("beta_slow", "beta_slow"), ("original_max_position_embeddings", "original_seq_len")):
if src in scaling:
kwargs[dst] = scaling[src]
return cls(**kwargs)
__all__ = ["DeepseekV41Args", "LayerRole", "Mode"]