mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
79 lines
4.0 KiB
Python
79 lines
4.0 KiB
Python
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
from typing import Any, ClassVar
|
|
|
|
from ..linear import LinearConfig
|
|
from ..moe import MoEConfig
|
|
from ..names import is_routed_expert, name_set, substr_set
|
|
from ..registry import LayerKind, register_dialect
|
|
from ..scheme import QuantKind, QuantScheme
|
|
from ..scheme import FP8_BLOCK_SIZES, fp8_block_scheme, fp8_tensor_scheme, mxfp4_scheme
|
|
from .base import QuantConfig, Stored, cfg_get
|
|
|
|
|
|
@register_dialect
|
|
class Fp8BlockConfig(QuantConfig):
|
|
"""HF ``quant_method: fp8`` (DeepSeek-V3 style 128x128 block scales) plus the DeepSeek-V4 e8m0 / fp4-expert
|
|
variant; DeepSeek-V4.1 keeps that dialect with 32x32 blocks (``weight_block_size: [32, 32]``)."""
|
|
|
|
dialect = "fp8"
|
|
|
|
# the 128-block schemes every dialect table names; ``block_scheme`` carries the checkpoint's actual block edge
|
|
SCHEMES: ClassVar[dict[str, QuantScheme]] = {
|
|
"BLOCK": fp8_block_scheme("float"),
|
|
"BLOCK_E8M0": fp8_block_scheme("e8m0"),
|
|
# HF ``modules_to_convert``: a table (Qwen3.8-Flash-Next PLE) stored e4m3 with one scalar scale
|
|
"TABLE": fp8_tensor_scheme("float"),
|
|
"EXPERT_MXFP4": mxfp4_scheme(),
|
|
}
|
|
# transformers' fp8 names; DeepSeek-V4's e8m0 export calls every scale ``scale`` (see storage)
|
|
STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {
|
|
QuantKind.FP8_BLOCK: {"weight": "weight", "weight_scale_inv": "weight_scale_inv"},
|
|
QuantKind.FP8_TENSOR: {"weight": "weight", "weight_scale": "weight_scale"},
|
|
QuantKind.MXFP4: {"weight": "weight", "weight_scale": "scale"},
|
|
}
|
|
|
|
def __init__(self, q: dict[str, Any], hf_config: Any = None, *, name_map=None, unquantized=()):
|
|
super().__init__(name_map, unquantized)
|
|
block = tuple(int(x) for x in (q.get("weight_block_size") or ()))
|
|
square = len(block) == 2 and block[0] == block[1] and block[0] in FP8_BLOCK_SIZES
|
|
if q.get("weight_per_tensor") or not square:
|
|
raise NotImplementedError(
|
|
f"fp8 checkpoint with weight_block_size={block} per_tensor={q.get('weight_per_tensor')} is not supported; "
|
|
f"only square blocks of {FP8_BLOCK_SIZES} are"
|
|
)
|
|
self.block = block[0]
|
|
# transformers skips lm_head when the checkpoint gives no list
|
|
not_convert = tuple(q.get("modules_to_not_convert") or ("lm_head",))
|
|
self.not_convert = name_set(not_convert)
|
|
self.not_convert_substr = substr_set(not_convert)
|
|
self.convert_tables = name_set(tuple(q.get("modules_to_convert") or ()))
|
|
self.e8m0 = str(q.get("scale_fmt") or "").lower() == "ue8m0"
|
|
# ``expert_dtype`` sits at the config top level (V4) or inside quantization_config (V4.1)
|
|
expert_dtype = q.get("expert_dtype") or cfg_get(hf_config, "expert_dtype")
|
|
self.expert_fp4 = str(expert_dtype or "").lower() == "fp4"
|
|
self.block_scheme = fp8_block_scheme("e8m0" if self.e8m0 else "float", self.block)
|
|
|
|
def storage(self, scheme: QuantScheme) -> dict[str, Stored]:
|
|
names = super().storage(scheme)
|
|
if self.e8m0 and scheme.kind is QuantKind.FP8_BLOCK:
|
|
names["weight_scale_inv"] = Stored("scale")
|
|
return names
|
|
|
|
def scheme_for_name(self, name: str) -> QuantScheme | None:
|
|
if self.convert_tables(name):
|
|
return self.SCHEMES["TABLE"]
|
|
if self.not_convert(name) or self.not_convert_substr(name):
|
|
return None
|
|
if self.expert_fp4 and is_routed_expert(name):
|
|
return self.SCHEMES["EXPERT_MXFP4"]
|
|
return self.block_scheme
|
|
|
|
def layer_config(self, layer: Any, layer_kind: LayerKind, scheme: QuantScheme | None) -> LinearConfig | MoEConfig:
|
|
cfg = super().layer_config(layer, layer_kind, scheme)
|
|
if layer_kind is LayerKind.MOE and scheme is not None and scheme.kind is QuantKind.MXFP4:
|
|
# the fp4 experts quantize their activations at the same block as the dense linears
|
|
cfg = dataclasses.replace(cfg, act_block=self.block)
|
|
return cfg
|