mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
93 lines
3.5 KiB
Python
93 lines
3.5 KiB
Python
"""Device-side row-format helpers shared by the DSV41 kernels.
|
|
|
|
The Python-side descriptors live in ``freetoken.kvcache.dsv4.v41_row_format``; the ``FMT`` constexpr a
|
|
kernel receives is ``RowFormat.code``:
|
|
|
|
0 bf16 2 B / channel, no scale
|
|
1 fp8_e8m0_b32 e4m3 byte / channel, ue8m0 scale byte per 32 channels
|
|
2 fp4_e4m3_b16 e2m1 nibble / channel (even channel low nibble), e4m3 scale byte per 16
|
|
3 fp4_e8m0_b32 e2m1 nibble / channel, ue8m0 scale byte per 32
|
|
|
|
A row is ``[values | scales]``; ``ROW_BYTES`` is the pool's row stride. Every dequantized value
|
|
is rounded through bf16 before use -- the reference bakes its quant round-trip into a bf16 cache,
|
|
so this is what its attention / indexer kernels actually read.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import triton
|
|
import triton.language as tl
|
|
|
|
from freetoken.kernel.triton.e4m3_compat import e4m3_u8_to_f32
|
|
|
|
FMT_BF16 = 0
|
|
FMT_FP8_E8M0_B32 = 1
|
|
FMT_FP4_E4M3_B16 = 2
|
|
FMT_FP4_E8M0_B32 = 3
|
|
|
|
|
|
@triton.jit
|
|
def fp4_nibble_to_f32(nib):
|
|
"""e2m1 code (uint8, 0..15) -> fp32: bit 3 sign, bits 2-1 exponent, bit 0 mantissa;
|
|
codes 0/1 are the subnormals 0 and 0.5, the rest ``(1 + m/2) * 2^(e-1)``."""
|
|
c = (nib & 7).to(tl.int32)
|
|
neg = (nib & 8) != 0
|
|
sub = c.to(tl.float32) * 0.5
|
|
normal = (1.0 + 0.5 * (c & 1).to(tl.float32)) * tl.exp2(((c >> 1) - 1).to(tl.float32))
|
|
mag = tl.where(c < 2, sub, normal)
|
|
return tl.where(neg, -mag, mag)
|
|
|
|
|
|
@triton.jit
|
|
def fp4_f32_to_nibble(q):
|
|
"""fp32 value already on the e2m1 grid ({0, .5, 1, 1.5, 2, 3, 4, 6}, signed) -> e2m1 code.
|
|
Thresholds sit between grid points, so a value a hair off the grid still maps to its point."""
|
|
a = tl.abs(q)
|
|
c = tl.where(
|
|
a < 1.75,
|
|
tl.where(a < 0.75, tl.where(a < 0.25, 0, 1), tl.where(a < 1.25, 2, 3)),
|
|
tl.where(a < 3.5, tl.where(a < 2.5, 4, 5), tl.where(a < 5.0, 6, 7)),
|
|
)
|
|
return (c + tl.where(q < 0, 8, 0)).to(tl.uint8)
|
|
|
|
|
|
@triton.jit
|
|
def load_rows(pool_ptr, rows, valid, offs_d, D: tl.constexpr, ROW_BYTES: tl.constexpr, FMT: tl.constexpr):
|
|
"""Gather + dequantize ``[T, D]`` fp32 from a packed pool.
|
|
|
|
``pool_ptr`` uint8 pool base; ``rows`` [T] row ids (any int width; masked-out rows may be -1);
|
|
``valid`` [T] bool; ``offs_d`` [D] = ``tl.arange(0, D)``. Invalid rows read as 0.
|
|
"""
|
|
row_i64 = tl.maximum(rows, 0).to(tl.int64)
|
|
base = pool_ptr + row_i64[:, None] * ROW_BYTES
|
|
m = valid[:, None] & (offs_d >= 0)[None, :]
|
|
if FMT == 0:
|
|
lo = tl.load(base + 2 * offs_d[None, :], mask=m, other=0).to(tl.uint16)
|
|
hi = tl.load(base + 2 * offs_d[None, :] + 1, mask=m, other=0).to(tl.uint16)
|
|
val = (lo | (hi << 8)).to(tl.bfloat16, bitcast=True).to(tl.float32)
|
|
elif FMT == 1:
|
|
v = tl.load(base + offs_d[None, :], mask=m, other=0)
|
|
s = tl.load(base + D + (offs_d // 32)[None, :], mask=m, other=127)
|
|
val = e4m3_u8_to_f32(v) * tl.exp2(s.to(tl.float32) - 127.0)
|
|
else:
|
|
b = tl.load(base + (offs_d // 2)[None, :], mask=m, other=0)
|
|
nib = tl.where((offs_d % 2 == 0)[None, :], b & 0xF, b >> 4)
|
|
q = fp4_nibble_to_f32(nib)
|
|
if FMT == 2:
|
|
s = e4m3_u8_to_f32(tl.load(base + D // 2 + (offs_d // 16)[None, :], mask=m, other=0))
|
|
else:
|
|
s = tl.exp2(tl.load(base + D // 2 + (offs_d // 32)[None, :], mask=m, other=127).to(tl.float32) - 127.0)
|
|
val = q * s
|
|
return val.to(tl.bfloat16).to(tl.float32)
|
|
|
|
|
|
__all__ = [
|
|
"FMT_BF16",
|
|
"FMT_FP8_E8M0_B32",
|
|
"FMT_FP4_E4M3_B16",
|
|
"FMT_FP4_E8M0_B32",
|
|
"fp4_nibble_to_f32",
|
|
"fp4_f32_to_nibble",
|
|
"load_rows",
|
|
]
|