Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

93 lines
3.5 KiB
Python

"""Device-side row-format helpers shared by the DSV41 kernels.
The Python-side descriptors live in ``freetoken.kvcache.dsv4.v41_row_format``; the ``FMT`` constexpr a
kernel receives is ``RowFormat.code``:
0 bf16 2 B / channel, no scale
1 fp8_e8m0_b32 e4m3 byte / channel, ue8m0 scale byte per 32 channels
2 fp4_e4m3_b16 e2m1 nibble / channel (even channel low nibble), e4m3 scale byte per 16
3 fp4_e8m0_b32 e2m1 nibble / channel, ue8m0 scale byte per 32
A row is ``[values | scales]``; ``ROW_BYTES`` is the pool's row stride. Every dequantized value
is rounded through bf16 before use -- the reference bakes its quant round-trip into a bf16 cache,
so this is what its attention / indexer kernels actually read.
"""
from __future__ import annotations
import triton
import triton.language as tl
from freetoken.kernel.triton.e4m3_compat import e4m3_u8_to_f32
FMT_BF16 = 0
FMT_FP8_E8M0_B32 = 1
FMT_FP4_E4M3_B16 = 2
FMT_FP4_E8M0_B32 = 3
@triton.jit
def fp4_nibble_to_f32(nib):
"""e2m1 code (uint8, 0..15) -> fp32: bit 3 sign, bits 2-1 exponent, bit 0 mantissa;
codes 0/1 are the subnormals 0 and 0.5, the rest ``(1 + m/2) * 2^(e-1)``."""
c = (nib & 7).to(tl.int32)
neg = (nib & 8) != 0
sub = c.to(tl.float32) * 0.5
normal = (1.0 + 0.5 * (c & 1).to(tl.float32)) * tl.exp2(((c >> 1) - 1).to(tl.float32))
mag = tl.where(c < 2, sub, normal)
return tl.where(neg, -mag, mag)
@triton.jit
def fp4_f32_to_nibble(q):
"""fp32 value already on the e2m1 grid ({0, .5, 1, 1.5, 2, 3, 4, 6}, signed) -> e2m1 code.
Thresholds sit between grid points, so a value a hair off the grid still maps to its point."""
a = tl.abs(q)
c = tl.where(
a < 1.75,
tl.where(a < 0.75, tl.where(a < 0.25, 0, 1), tl.where(a < 1.25, 2, 3)),
tl.where(a < 3.5, tl.where(a < 2.5, 4, 5), tl.where(a < 5.0, 6, 7)),
)
return (c + tl.where(q < 0, 8, 0)).to(tl.uint8)
@triton.jit
def load_rows(pool_ptr, rows, valid, offs_d, D: tl.constexpr, ROW_BYTES: tl.constexpr, FMT: tl.constexpr):
"""Gather + dequantize ``[T, D]`` fp32 from a packed pool.
``pool_ptr`` uint8 pool base; ``rows`` [T] row ids (any int width; masked-out rows may be -1);
``valid`` [T] bool; ``offs_d`` [D] = ``tl.arange(0, D)``. Invalid rows read as 0.
"""
row_i64 = tl.maximum(rows, 0).to(tl.int64)
base = pool_ptr + row_i64[:, None] * ROW_BYTES
m = valid[:, None] & (offs_d >= 0)[None, :]
if FMT == 0:
lo = tl.load(base + 2 * offs_d[None, :], mask=m, other=0).to(tl.uint16)
hi = tl.load(base + 2 * offs_d[None, :] + 1, mask=m, other=0).to(tl.uint16)
val = (lo | (hi << 8)).to(tl.bfloat16, bitcast=True).to(tl.float32)
elif FMT == 1:
v = tl.load(base + offs_d[None, :], mask=m, other=0)
s = tl.load(base + D + (offs_d // 32)[None, :], mask=m, other=127)
val = e4m3_u8_to_f32(v) * tl.exp2(s.to(tl.float32) - 127.0)
else:
b = tl.load(base + (offs_d // 2)[None, :], mask=m, other=0)
nib = tl.where((offs_d % 2 == 0)[None, :], b & 0xF, b >> 4)
q = fp4_nibble_to_f32(nib)
if FMT == 2:
s = e4m3_u8_to_f32(tl.load(base + D // 2 + (offs_d // 16)[None, :], mask=m, other=0))
else:
s = tl.exp2(tl.load(base + D // 2 + (offs_d // 32)[None, :], mask=m, other=127).to(tl.float32) - 127.0)
val = q * s
return val.to(tl.bfloat16).to(tl.float32)
__all__ = [
"FMT_BF16",
"FMT_FP8_E8M0_B32",
"FMT_FP4_E4M3_B16",
"FMT_FP4_E8M0_B32",
"fp4_nibble_to_f32",
"fp4_f32_to_nibble",
"load_rows",
]