Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

348 lines
14 KiB
Python

"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels.
One program per token fuses the sublayer-boundary mix: apply the previous
sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre --
the fn GEMV over the flattened streams, flat-RMS normalization, and the three
gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an
n x n held in registers) -- and the pre-mixed layer input.
Semantics are defined by layers/mhc.py's torch reference (bit-comparable in
fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N
(hc_mult) is a constexpr; only N == 4 is exercised.
"""
from __future__ import annotations
import torch
import triton
import triton.language as tl
@triton.jit
def _mhc_stage1_kernel(
x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr,
res_out_ptr,
sq_part_ptr, # [T, NS] fp32
mix_part_ptr, # [T, NS, BLK_MIX] fp32
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
SPLIT: tl.constexpr, # hidden elems per split (multiple of BLOCK_H)
BLOCK_H: tl.constexpr,
NS: tl.constexpr, # splits per token (the grid's second axis; need not be a power of two)
HAS_POST: tl.constexpr,
):
"""Split-K stage of the fused mHC: each program owns one hidden slice of one
token -- applies hc_post there, stores the updated streams, and reduces its
partial sq-sum + fn-GEMV contribution over NS splits."""
t = tl.program_id(0).to(tl.int64)
s = tl.program_id(1)
offs_mix = tl.arange(0, BLK_MIX)
mix_mask = offs_mix < MIX
offs_n = tl.arange(0, N)
if HAS_POST:
b_post = tl.load(post_ptr + t * N + offs_n)
b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :])
sqsum = 0.0
acc = tl.zeros([BLK_MIX], dtype=tl.float32)
for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H):
offs_h = h0 + tl.arange(0, BLOCK_H)
h_mask = offs_h < H
if HAS_POST:
b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
for n in tl.static_range(N):
if HAS_POST:
r_new = tl.zeros([BLOCK_H], dtype=tl.float32)
for i in tl.static_range(N):
r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0))
r_new += c_in * r_i
p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0))
r_new += p_n * b_x
# Round through the STORAGE dtype before the sq-sum and fn-GEMV:
# the torch reference reads back what it stored (dtype-generic --
# fp16/fp32 residuals must not be silently bf16-rounded).
r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32)
tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
else:
r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
sqsum += tl.sum(r_new * r_new)
fn_tile = tl.load(
fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :],
mask=mix_mask[:, None] & h_mask[None, :], other=0.0,
)
acc += tl.sum(fn_tile * r_new[None, :], axis=1)
tl.store(sq_part_ptr + t * NS + s, sqsum)
tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc)
@triton.jit
def _mhc_gates(
t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
post_out_ptr, comb_out_ptr, pre_out_ptr,
rms_eps, hc_eps, post_mult,
SINKHORN: tl.constexpr,
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
NS: tl.constexpr, BLK_NS: tl.constexpr,
):
"""Reduce token ``t``'s split partials and run the tiny gate math (sigmoid gates,
row-softmax + in-register 4x4 Sinkhorn); writes post / comb / pre for the next sublayer."""
offs_mix = tl.arange(0, BLK_MIX)
mix_mask = offs_mix < MIX
offs_s = tl.arange(0, BLK_NS)
s_mask = offs_s < NS
sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s, mask=s_mask, other=0.0))
acc = tl.sum(
tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :], mask=s_mask[:, None], other=0.0),
axis=0,
)
inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps)
mixes = acc * inv_rms
s0 = tl.load(scale_ptr + 0)
s1 = tl.load(scale_ptr + 1)
s2 = tl.load(scale_ptr + 2)
b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0)
is_pre = offs_mix < N
is_post = (offs_mix >= N) & (offs_mix < 2 * N)
gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2))
logits = mixes * gate_scale + b_base
pre = tl.sigmoid(logits) + hc_eps
post_new = tl.sigmoid(logits) * post_mult
offs_n2 = tl.arange(0, N)
comb_logits = tl.zeros([N, N], dtype=tl.float32)
for r in tl.static_range(N):
for c in tl.static_range(N):
lane = 2 * N + r * N + c
v = tl.sum(tl.where(offs_mix == lane, logits, 0.0))
comb_logits += tl.where(
(offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0
)
row_max = tl.max(comb_logits, axis=1)
e = tl.exp(comb_logits - row_max[:, None])
comb = e / tl.sum(e, axis=1)[:, None] + hc_eps
comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
for _ in range(SINKHORN - 1):
comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps)
comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
post_g = tl.sum(
tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0),
axis=1,
)
pre_g = tl.sum(
tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1
)
tl.store(post_out_ptr + t * N + offs_n2, post_g)
tl.store(pre_out_ptr + t * N + offs_n2, pre_g)
tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb)
@triton.jit
def _mhc_stage2_kernel(
sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
post_out_ptr, comb_out_ptr, pre_out_ptr,
rms_eps, hc_eps, post_mult,
SINKHORN: tl.constexpr,
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
NS: tl.constexpr, BLK_NS: tl.constexpr,
):
"""Gate math alone (the two-pass variant, whose stage 3 consumes these pre gates)."""
_mhc_gates(
tl.program_id(0).to(tl.int64), sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
)
@triton.jit
def _mhc_stage3_kernel(
res_out_ptr, pre_ptr, li_out_ptr,
H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr,
):
"""layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks."""
t = tl.program_id(0).to(tl.int64)
hb = tl.program_id(1)
offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = offs_h < H
offs_n = tl.arange(0, N)
pre = tl.load(pre_ptr + t * N + offs_n)
li = tl.zeros([BLOCK_H], dtype=tl.float32)
for n in tl.static_range(N):
r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
@triton.jit
def _mhc_stage23_kernel(
res_out_ptr, pre_ptr, li_out_ptr,
sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
post_out_ptr, comb_out_ptr, pre_out_ptr,
rms_eps, hc_eps, post_mult,
SINKHORN: tl.constexpr,
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
NS: tl.constexpr, BLK_NS: tl.constexpr, BLOCK_H: tl.constexpr,
):
"""Single-pass mHC: the layer input mixes with the PREVIOUS sublayer's pre gates, so the
gate math for the next sublayer (stage 2) does not gate stage 3 -- one launch does both:
every program mixes one hidden chunk, program 0 of each token also reduces the split
partials and writes the next sublayer's gates."""
t = tl.program_id(0).to(tl.int64)
hb = tl.program_id(1)
offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
h_mask = offs_h < H
offs_n = tl.arange(0, N)
pre = tl.load(pre_ptr + t * N + offs_n)
li = tl.zeros([BLOCK_H], dtype=tl.float32)
for n in tl.static_range(N):
r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
if hb == 0:
_mhc_gates(
t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
)
def _stage1_hidden_tile(h: int) -> int:
"""The stage-1 hidden split. It fixes the fp32 reduction order over the hidden width, so it
depends on the width alone and results reproduce across processes, machines and chunk sizes.
128 gives 32 / 40 programs per token at widths 4096 / 5120."""
return min(128, triton.next_power_of_2(h))
def _mhc_fused_launch(
x: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor | None,
comb_mix: torch.Tensor | None,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_eps: float,
post_mult: float,
sinkhorn_repeat: int,
pre_mix_in: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""The three stages behind both variants. ``pre_mix_in`` selects single-pass: the
layer input is mixed with it instead of with the pre gates stage 2 just computed
(those are still returned, for the next sublayer). Returns (residual_new, post,
comb, pre_new [T,N] fp32, layer_input)."""
t, n, h = residual.shape
mix = 2 * n + n * n
assert fn.shape == (mix, n * h) and fn.dtype == torch.float32
residual = residual.contiguous()
has_post = post_mix is not None
dev = residual.device
res_out = torch.empty_like(residual)
post_out = torch.empty(t, n, dtype=torch.float32, device=dev)
comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev)
li_out = torch.empty(t, h, dtype=residual.dtype, device=dev)
block_h = split = _stage1_hidden_tile(h)
ns = triton.cdiv(h, split)
blk_mix = triton.next_power_of_2(mix)
sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev)
mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev)
pre_out = torch.empty(t, n, dtype=torch.float32, device=dev)
_mhc_stage1_kernel[(t, ns)](
x.contiguous() if has_post else residual, # dummy ptr when unused
residual,
post_mix.contiguous().view(t, n) if has_post else post_out,
comb_mix.contiguous() if has_post else comb_out,
fn, res_out, sq_part, mix_part,
H=h, N=n, MIX=mix, BLK_MIX=blk_mix,
SPLIT=split, BLOCK_H=block_h, NS=ns,
HAS_POST=has_post,
num_warps=4, num_stages=2,
)
block_h3 = min(1024, triton.next_power_of_2(h))
if pre_mix_in is None:
# two-pass: stage 3 mixes with the gates stage 2 just produced
_mhc_stage2_kernel[(t,)](
sq_part, mix_part, hc_scale, hc_base,
post_out, comb_out, pre_out,
rms_eps, hc_eps, post_mult,
SINKHORN=sinkhorn_repeat,
H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns),
num_warps=1,
)
_mhc_stage3_kernel[(t, triton.cdiv(h, block_h3))](
res_out, pre_out, li_out,
H=h, N=n, BLOCK_H=block_h3,
num_warps=4,
)
else:
# single-pass: the mix uses the previous sublayer's gates, so stages 2 and 3 share a launch
assert pre_mix_in.shape == (t, n), (pre_mix_in.shape, (t, n))
_mhc_stage23_kernel[(t, triton.cdiv(h, block_h3))](
res_out, pre_mix_in.to(torch.float32).contiguous(), li_out,
sq_part, mix_part, hc_scale, hc_base,
post_out, comb_out, pre_out,
rms_eps, hc_eps, post_mult,
SINKHORN=sinkhorn_repeat,
H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns), BLOCK_H=block_h3,
num_warps=4,
)
return res_out, post_out.view(t, n, 1), comb_out, pre_out, li_out
def mhc_fused_post_pre_triton(
x: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor | None,
comb_mix: torch.Tensor | None,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_eps: float,
post_mult: float,
sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage
split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns
(residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32,
layer_input [T,H] bf16)."""
res_out, post_out, comb_out, _pre_out, li_out = _mhc_fused_launch(
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=None,
)
return res_out, post_out, comb_out, li_out
def mhc_fused_post_pre_single_pass_triton(
x: torch.Tensor,
residual: torch.Tensor,
post_mix: torch.Tensor | None,
comb_mix: torch.Tensor | None,
pre_mix_in: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_eps: float,
post_mult: float,
sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Single-pass mHC (DeepSeek-V4.1): same fused stages, but the layer input is mixed
with the PREVIOUS sublayer's ``pre_mix_in`` and this sublayer's pre gates come back
as ``pre_new`` for the next one. Returns (residual_new, post [T,N,1], comb [T,N,N],
pre_new [T,N] fp32, layer_input)."""
return _mhc_fused_launch(
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=pre_mix_in,
)
__all__ = ["mhc_fused_post_pre_triton", "mhc_fused_post_pre_single_pass_triton"]