mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
71 lines
2.8 KiB
Python
71 lines
2.8 KiB
Python
"""fp8 e4m3 weight with square block scales: 128x128 (DeepSeek-V3 style) or 32x32 (DeepSeek-V4.1)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import FP8_BLOCK, QuantKind, fp8_block_size
|
|
from .base import LinearConfig, LinearKernel, LinearMethod
|
|
|
|
FP8 = torch.float8_e4m3fn
|
|
E8M0 = torch.float8_e8m0fnu
|
|
|
|
|
|
def _e8m0(cfg: LinearConfig) -> bool:
|
|
return cfg.scheme is not None and cfg.scheme.weight.scale == "e8m0"
|
|
|
|
|
|
def layer_block_size(layer: Any) -> int:
|
|
"""The block edge a finalized layer was declared with: ``K / (scale columns)``, exact by construction."""
|
|
return layer.weight.shape[1] // layer.weight_scale_inv.shape[1]
|
|
|
|
|
|
class Dsv4Fp8BlockLinearKernel(LinearKernel):
|
|
"""DeepSeek-V4's reference path: activations quantized to fp8 with power-of-two block scales, e8m0 weight scales read as codes."""
|
|
|
|
name = "dsv4"
|
|
|
|
def unusable_reason(self, cfg: LinearConfig) -> str | None:
|
|
return None if _e8m0(cfg) else "serves e8m0 block scales only"
|
|
|
|
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel.triton.dsv4.fp8_linear import block_fp8_linear
|
|
|
|
return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias, block=layer_block_size(layer))
|
|
|
|
|
|
class TritonFp8BlockLinearKernel(LinearKernel):
|
|
"""W8A16 GEMV at M=1, dynamic 1x128 W8A8 GEMM above."""
|
|
|
|
name = "triton"
|
|
|
|
def unusable_reason(self, cfg: LinearConfig) -> str | None:
|
|
if _e8m0(cfg):
|
|
return "reads float block scales; e8m0 codes go to the dsv4 kernel"
|
|
if fp8_block_size(cfg.scheme) != FP8_BLOCK:
|
|
return f"float-scale kernel serves {FP8_BLOCK}x{FP8_BLOCK} blocks only"
|
|
return None
|
|
|
|
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel.triton.fp8_block_linear import block_fp8_linear
|
|
|
|
return block_fp8_linear(x, layer.weight, layer.weight_scale_inv, layer.bias)
|
|
|
|
|
|
@register_method(QuantKind.FP8_BLOCK, LayerKind.LINEAR)
|
|
class Fp8BlockLinearMethod(LinearMethod):
|
|
candidates = (Dsv4Fp8BlockLinearKernel, TritonFp8BlockLinearKernel)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
g = self.cfg
|
|
block = fp8_block_size(g.scheme)
|
|
if g.in_features % block or any(o % block for o in g.output_sizes):
|
|
raise ValueError(f"block-fp8 needs in/out sizes divisible by {block}, got K={g.in_features} N={g.output_sizes}")
|
|
layer.weight = torch.empty(g.out_features, g.in_features, dtype=FP8)
|
|
# e8m0 codes stay codes for the dsv4 kernel; float scales are bf16 as the readers push them today
|
|
scale_dtype = E8M0 if _e8m0(g) else torch.bfloat16
|
|
layer.weight_scale_inv = torch.empty(g.out_features // block, g.in_features // block, dtype=scale_dtype)
|