Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

79 lines
4.0 KiB
Python

from __future__ import annotations
import dataclasses
from typing import Any, ClassVar
from ..linear import LinearConfig
from ..moe import MoEConfig
from ..names import is_routed_expert, name_set, substr_set
from ..registry import LayerKind, register_dialect
from ..scheme import QuantKind, QuantScheme
from ..scheme import FP8_BLOCK_SIZES, fp8_block_scheme, fp8_tensor_scheme, mxfp4_scheme
from .base import QuantConfig, Stored, cfg_get
@register_dialect
class Fp8BlockConfig(QuantConfig):
"""HF ``quant_method: fp8`` (DeepSeek-V3 style 128x128 block scales) plus the DeepSeek-V4 e8m0 / fp4-expert
variant; DeepSeek-V4.1 keeps that dialect with 32x32 blocks (``weight_block_size: [32, 32]``)."""
dialect = "fp8"
# the 128-block schemes every dialect table names; ``block_scheme`` carries the checkpoint's actual block edge
SCHEMES: ClassVar[dict[str, QuantScheme]] = {
"BLOCK": fp8_block_scheme("float"),
"BLOCK_E8M0": fp8_block_scheme("e8m0"),
# HF ``modules_to_convert``: a table (Qwen3.8-Flash-Next PLE) stored e4m3 with one scalar scale
"TABLE": fp8_tensor_scheme("float"),
"EXPERT_MXFP4": mxfp4_scheme(),
}
# transformers' fp8 names; DeepSeek-V4's e8m0 export calls every scale ``scale`` (see storage)
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {
QuantKind.FP8_BLOCK: {"weight": "weight", "weight_scale_inv": "weight_scale_inv"},
QuantKind.FP8_TENSOR: {"weight": "weight", "weight_scale": "weight_scale"},
QuantKind.MXFP4: {"weight": "weight", "weight_scale": "scale"},
}
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
super().__init__(name_map, unquantized)
block = tuple(int(x) for x in (q.get("weight_block_size") or ()))
square = len(block) == 2 and block[0] == block[1] and block[0] in FP8_BLOCK_SIZES
if q.get("weight_per_tensor") or not square:
raise NotImplementedError(
f"fp8 checkpoint with weight_block_size={block} per_tensor={q.get('weight_per_tensor')} is not supported; "
f"only square blocks of {FP8_BLOCK_SIZES} are"
)
self.block = block[0]
# transformers skips lm_head when the checkpoint gives no list
not_convert = tuple(q.get("modules_to_not_convert") or ("lm_head",))
self.not_convert = name_set(not_convert)
self.not_convert_substr = substr_set(not_convert)
self.convert_tables = name_set(tuple(q.get("modules_to_convert") or ()))
self.e8m0 = str(q.get("scale_fmt") or "").lower() == "ue8m0"
# ``expert_dtype`` sits at the config top level (V4) or inside quantization_config (V4.1)
expert_dtype = q.get("expert_dtype") or cfg_get(hf_config, "expert_dtype")
self.expert_fp4 = str(expert_dtype or "").lower() == "fp4"
self.block_scheme = fp8_block_scheme("e8m0" if self.e8m0 else "float", self.block)
def storage(self, scheme: QuantScheme) -> dict[str, Stored]:
names = super().storage(scheme)
if self.e8m0 and scheme.kind is QuantKind.FP8_BLOCK:
names["weight_scale_inv"] = Stored("scale")
return names
def scheme_for_name(self, name: str) -> QuantScheme | None:
if self.convert_tables(name):
return self.SCHEMES["TABLE"]
if self.not_convert(name) or self.not_convert_substr(name):
return None
if self.expert_fp4 and is_routed_expert(name):
return self.SCHEMES["EXPERT_MXFP4"]
return self.block_scheme
def layer_config(self, layer: Any, layer_kind: LayerKind, scheme: QuantScheme | None) -> LinearConfig | MoEConfig:
cfg = super().layer_config(layer, layer_kind, scheme)
if layer_kind is LayerKind.MOE and scheme is not None and scheme.kind is QuantKind.MXFP4:
# the fp4 experts quantize their activations at the same block as the dense linears
cfg = dataclasses.replace(cfg, act_block=self.block)
return cfg