mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
226 lines
9.6 KiB
Python
226 lines
9.6 KiB
Python
"""DeepSeek-V4.1-Flash hyperparameters and the static per-layer attention roles.
|
|
|
|
``DeepseekV41Args`` mirrors the reference ``ModelArgs`` (``inference/model.py``) field for field,
|
|
read from the HF ``config.json``'s ``text_config`` (the ``inference/config.json`` dialect is also
|
|
accepted). Everything the engine reconciles at runtime (``max_seq_len``, the replay policy) is
|
|
attached by ``parse_config`` / the engine, not read from the checkpoint.
|
|
|
|
``LayerRole`` is the attention mode table (tech report sec. 2.3.1): every compressing layer is Full
|
|
(owns the compressed KV + indexer K), Reindex (own indexer over a shared K) or Reuse (shared
|
|
Top-K); the candidate pool of the Hierarchical Sparse Indexer (sec. 2.3.2) is produced by the
|
|
first decoder Full layer and consumed by the decoder Reindex layers after it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from enum import Enum
|
|
from typing import Any, Literal
|
|
|
|
# text_config key -> ModelArgs key, where the two dialects differ
|
|
_HF_TO_ARGS = {
|
|
"hidden_size": "dim",
|
|
"moe_intermediate_size": "moe_inter_dim",
|
|
"num_hidden_layers": "n_layers",
|
|
"num_nextn_predict_layers": "n_mtp_layers",
|
|
"num_attention_heads": "n_heads",
|
|
"num_experts_per_tok": "n_activated_experts",
|
|
"scoring_func": "score_func",
|
|
"routed_scaling_factor": "route_scale",
|
|
"qk_rope_head_dim": "rope_head_dim",
|
|
"rms_norm_eps": "norm_eps",
|
|
"sliding_window": "window_size",
|
|
"kv_source_layer_ids": "kv_source_layers",
|
|
"index_source_layer_ids": "index_source_layers",
|
|
"candidate_source_layer_id": "candidate_source_layer",
|
|
"engram_pad_token_id": "engram_pad_id",
|
|
"dspark_num_experts_per_tok": "dspark_n_activated_experts",
|
|
}
|
|
|
|
|
|
class Mode(str, Enum):
|
|
WINDOW = "window" # sliding window only (compress_ratio == 0)
|
|
FULL = "full" # computes main KV + indexer K, runs its indexer
|
|
REINDEX = "reindex" # reuses main KV + indexer K, runs its own indexer
|
|
REUSE = "reuse" # reuses main KV and the latest Top-K
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LayerRole:
|
|
layer_id: int
|
|
ratio: int
|
|
mode: Mode
|
|
kv_source: int | None # the layer whose main KV / indexer K this layer reads (itself when Full)
|
|
is_candidate_source: bool # builds the hierarchical candidate pool
|
|
uses_candidates: bool # indexes only inside the candidate pool
|
|
|
|
@property
|
|
def compresses(self) -> bool:
|
|
return self.ratio > 0
|
|
|
|
@property
|
|
def has_indexer(self) -> bool:
|
|
return self.mode in (Mode.FULL, Mode.REINDEX)
|
|
|
|
|
|
@dataclass
|
|
class DeepseekV41Args:
|
|
vocab_size: int = 129280
|
|
dim: int = 5120
|
|
moe_inter_dim: int = 2304
|
|
n_layers: int = 40
|
|
n_mtp_layers: int = 3
|
|
n_heads: int = 64
|
|
# moe
|
|
n_routed_experts: int = 384
|
|
n_shared_experts: int = 1
|
|
n_activated_experts: int = 6
|
|
score_func: Literal["softmax", "sigmoid", "sqrtsoftplus"] = "sqrtsoftplus"
|
|
norm_topk_prob: bool = True
|
|
route_scale: float = 1.5
|
|
swiglu_limit: float = 10.0
|
|
# attention: latent q, one shared latent kv head, grouped low-rank output projection
|
|
q_lora_rank: int = 1280
|
|
head_dim: int = 512
|
|
rope_head_dim: int = 64
|
|
norm_eps: float = 1e-20
|
|
o_groups: int = 8
|
|
o_lora_rank: int = 1024
|
|
# sparse attention
|
|
window_size: int = 128
|
|
compress_ratios: tuple[int, ...] = () # one per layer, MTP layers included
|
|
kv_source_layers: tuple[int, ...] = ()
|
|
index_source_layers: tuple[int, ...] = ()
|
|
compress_rope_theta: float = 160000.0
|
|
original_seq_len: int = 65536
|
|
rope_theta: float = 10000.0
|
|
rope_factor: float = 16.0
|
|
beta_fast: int = 32
|
|
beta_slow: int = 1
|
|
index_n_heads: int = 32
|
|
index_head_dim: int = 128
|
|
index_topk: int = 512
|
|
candidate_source_layer: int = -1
|
|
candidate_topk_blocks: int = 0
|
|
candidate_block_size: int = 0
|
|
# hyper-connections (single-pass mHC)
|
|
hc_mult: int = 4
|
|
hc_sinkhorn_iters: int = 20
|
|
hc_eps: float = 1e-6
|
|
# engram
|
|
engram_layer_ids: tuple[int, ...] = ()
|
|
engram_num_embeddings: tuple[int, ...] = ()
|
|
engram_max_ngram_size: int = 4
|
|
engram_vocab_size: int = 0
|
|
engram_n_heads: int = 0
|
|
engram_head_dim: int = 0
|
|
engram_pad_id: int = 2
|
|
engram_compressed_vocab_size: int = 0
|
|
# dspark (weights are not served in v1; kept so the config round-trips)
|
|
dspark_block_size: int = 0
|
|
dspark_target_layer_ids: tuple[int, ...] = ()
|
|
# ---- runtime (engine-reconciled, never read from the checkpoint) ----
|
|
max_seq_len: int = 4096
|
|
max_batch_size: int = 1
|
|
# "bounded": the decoder runs on the prompt's last window_size tokens with the sliding window
|
|
# truncated there (Decoder SWA Bounded Replay, tech report sec. 3.2.2); "exact": on every token.
|
|
swa_decoder_replay: Literal["bounded", "exact"] = "bounded"
|
|
roles: tuple[LayerRole, ...] = field(default_factory=tuple, repr=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
for name in ("compress_ratios", "kv_source_layers", "index_source_layers", "engram_layer_ids",
|
|
"engram_num_embeddings", "dspark_target_layer_ids"):
|
|
setattr(self, name, tuple(int(v) for v in getattr(self, name)))
|
|
if len(self.compress_ratios) < self.n_layers:
|
|
raise ValueError(f"compress_ratios has {len(self.compress_ratios)} entries for {self.n_layers} layers")
|
|
if self.n_shared_experts != 1:
|
|
raise ValueError("DeepSeek-V4.1 has exactly one shared expert")
|
|
if self.hc_mult < 1:
|
|
raise ValueError("hc_mult must be positive")
|
|
self.roles = tuple(self._role(layer) for layer in range(self.n_layers))
|
|
if len(self.engram_layer_ids) != len(self.engram_num_embeddings):
|
|
raise ValueError("engram_layer_ids and engram_num_embeddings differ in length")
|
|
|
|
# ---- derived ----
|
|
def _role(self, layer: int) -> LayerRole:
|
|
ratio = self.compress_ratios[layer]
|
|
if ratio == 0:
|
|
return LayerRole(layer, 0, Mode.WINDOW, None, False, False)
|
|
sources = [s for s in self.kv_source_layers if s <= layer]
|
|
if not sources:
|
|
raise ValueError(f"layer {layer} compresses but no kv source precedes it")
|
|
source = max(sources)
|
|
if self.compress_ratios[source] != ratio:
|
|
raise ValueError(f"layer {layer} (ratio {ratio}) reads kv source {source} (ratio {self.compress_ratios[source]})")
|
|
if layer == source:
|
|
mode = Mode.FULL
|
|
elif layer in self.index_source_layers:
|
|
mode = Mode.REINDEX
|
|
else:
|
|
mode = Mode.REUSE
|
|
cand = self.candidate_source_layer
|
|
is_cand_src = cand >= 0 and layer == cand
|
|
uses = cand >= 0 and cand < layer and mode is Mode.REINDEX
|
|
if uses and self.kv_source_layers and max(s for s in self.kv_source_layers if s <= cand) != source:
|
|
raise ValueError(f"layer {layer} would index the candidate pool of layer {cand} over a different kv source")
|
|
return LayerRole(layer, ratio, mode, source, is_cand_src, uses)
|
|
|
|
@property
|
|
def nope_head_dim(self) -> int:
|
|
return self.head_dim - self.rope_head_dim
|
|
|
|
@property
|
|
def backbone_kv_sources(self) -> tuple[int, ...]:
|
|
return tuple(s for s in self.kv_source_layers if s < self.n_layers)
|
|
|
|
@property
|
|
def decoder_start_layer(self) -> int:
|
|
"""The first decoder layer: the last kv source (CED projects the decoder's global KV from
|
|
its input, the final encoder hidden state)."""
|
|
return max(self.backbone_kv_sources) if self.backbone_kv_sources else self.n_layers
|
|
|
|
@property
|
|
def engram_hash_cols(self) -> int:
|
|
return (self.engram_max_ngram_size - 1) * self.engram_n_heads
|
|
|
|
@property
|
|
def index_softmax_scale(self) -> float:
|
|
return self.index_head_dim**-0.5
|
|
|
|
def freqs_params(self, layer: int) -> tuple[int, int, float, float, int, int]:
|
|
"""``(rotary_dim, original_seq_len, theta, factor, beta_fast, beta_slow)``: compressing layers
|
|
rotate at the compressed theta under YaRN, window-only layers at the base theta without it."""
|
|
if self.compress_ratios[layer]:
|
|
return self.rope_head_dim, self.original_seq_len, self.compress_rope_theta, self.rope_factor, self.beta_fast, self.beta_slow
|
|
return self.rope_head_dim, 0, self.rope_theta, self.rope_factor, self.beta_fast, self.beta_slow
|
|
|
|
@classmethod
|
|
def from_hf(cls, hf_config: Any) -> "DeepseekV41Args":
|
|
"""From the HF config (``text_config`` of the multimodal wrapper, or the text config itself,
|
|
or the ``inference/config.json`` dialect)."""
|
|
text = getattr(hf_config, "text_config", None) or hf_config
|
|
if isinstance(text, dict):
|
|
raw = text
|
|
elif hasattr(text, "to_dict"): # PretrainedConfig / RawConfigShim
|
|
raw = text.to_dict()
|
|
else:
|
|
raw = vars(text)
|
|
names = {f.name for f in cls.__dataclass_fields__.values()} - {"roles", "max_seq_len", "max_batch_size", "swa_decoder_replay"}
|
|
kwargs: dict[str, Any] = {}
|
|
for key, value in raw.items():
|
|
key = _HF_TO_ARGS.get(key, key)
|
|
if key in names:
|
|
kwargs[key] = value
|
|
# HF nests the YaRN parameters; the inference dialect keeps them flat
|
|
scaling = raw.get("rope_scaling") or raw.get("rope_parameters")
|
|
if isinstance(scaling, dict):
|
|
if "factor" in scaling:
|
|
kwargs["rope_factor"] = scaling["factor"]
|
|
for src, dst in (("beta_fast", "beta_fast"), ("beta_slow", "beta_slow"), ("original_max_position_embeddings", "original_seq_len")):
|
|
if src in scaling:
|
|
kwargs[dst] = scaling[src]
|
|
return cls(**kwargs)
|
|
|
|
|
|
__all__ = ["DeepseekV41Args", "LayerRole", "Mode"]
|