mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 05:49:20 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
348 lines
14 KiB
Python
348 lines
14 KiB
Python
"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels.
|
|
|
|
One program per token fuses the sublayer-boundary mix: apply the previous
|
|
sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre --
|
|
the fn GEMV over the flattened streams, flat-RMS normalization, and the three
|
|
gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an
|
|
n x n held in registers) -- and the pre-mixed layer input.
|
|
|
|
Semantics are defined by layers/mhc.py's torch reference (bit-comparable in
|
|
fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N
|
|
(hc_mult) is a constexpr; only N == 4 is exercised.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import triton
|
|
import triton.language as tl
|
|
|
|
|
|
@triton.jit
|
|
def _mhc_stage1_kernel(
|
|
x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr,
|
|
res_out_ptr,
|
|
sq_part_ptr, # [T, NS] fp32
|
|
mix_part_ptr, # [T, NS, BLK_MIX] fp32
|
|
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
|
|
SPLIT: tl.constexpr, # hidden elems per split (multiple of BLOCK_H)
|
|
BLOCK_H: tl.constexpr,
|
|
NS: tl.constexpr, # splits per token (the grid's second axis; need not be a power of two)
|
|
HAS_POST: tl.constexpr,
|
|
):
|
|
"""Split-K stage of the fused mHC: each program owns one hidden slice of one
|
|
token -- applies hc_post there, stores the updated streams, and reduces its
|
|
partial sq-sum + fn-GEMV contribution over NS splits."""
|
|
t = tl.program_id(0).to(tl.int64)
|
|
s = tl.program_id(1)
|
|
offs_mix = tl.arange(0, BLK_MIX)
|
|
mix_mask = offs_mix < MIX
|
|
offs_n = tl.arange(0, N)
|
|
|
|
if HAS_POST:
|
|
b_post = tl.load(post_ptr + t * N + offs_n)
|
|
b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :])
|
|
|
|
sqsum = 0.0
|
|
acc = tl.zeros([BLK_MIX], dtype=tl.float32)
|
|
for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H):
|
|
offs_h = h0 + tl.arange(0, BLOCK_H)
|
|
h_mask = offs_h < H
|
|
if HAS_POST:
|
|
b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
|
|
for n in tl.static_range(N):
|
|
if HAS_POST:
|
|
r_new = tl.zeros([BLOCK_H], dtype=tl.float32)
|
|
for i in tl.static_range(N):
|
|
r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
|
|
c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0))
|
|
r_new += c_in * r_i
|
|
p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0))
|
|
r_new += p_n * b_x
|
|
# Round through the STORAGE dtype before the sq-sum and fn-GEMV:
|
|
# the torch reference reads back what it stored (dtype-generic --
|
|
# fp16/fp32 residuals must not be silently bf16-rounded).
|
|
r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32)
|
|
tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
|
|
else:
|
|
r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
|
|
tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask)
|
|
sqsum += tl.sum(r_new * r_new)
|
|
fn_tile = tl.load(
|
|
fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :],
|
|
mask=mix_mask[:, None] & h_mask[None, :], other=0.0,
|
|
)
|
|
acc += tl.sum(fn_tile * r_new[None, :], axis=1)
|
|
|
|
tl.store(sq_part_ptr + t * NS + s, sqsum)
|
|
tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc)
|
|
|
|
|
|
@triton.jit
|
|
def _mhc_gates(
|
|
t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
|
|
post_out_ptr, comb_out_ptr, pre_out_ptr,
|
|
rms_eps, hc_eps, post_mult,
|
|
SINKHORN: tl.constexpr,
|
|
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
|
|
NS: tl.constexpr, BLK_NS: tl.constexpr,
|
|
):
|
|
"""Reduce token ``t``'s split partials and run the tiny gate math (sigmoid gates,
|
|
row-softmax + in-register 4x4 Sinkhorn); writes post / comb / pre for the next sublayer."""
|
|
offs_mix = tl.arange(0, BLK_MIX)
|
|
mix_mask = offs_mix < MIX
|
|
offs_s = tl.arange(0, BLK_NS)
|
|
s_mask = offs_s < NS
|
|
|
|
sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s, mask=s_mask, other=0.0))
|
|
acc = tl.sum(
|
|
tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :], mask=s_mask[:, None], other=0.0),
|
|
axis=0,
|
|
)
|
|
inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps)
|
|
mixes = acc * inv_rms
|
|
s0 = tl.load(scale_ptr + 0)
|
|
s1 = tl.load(scale_ptr + 1)
|
|
s2 = tl.load(scale_ptr + 2)
|
|
b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0)
|
|
|
|
is_pre = offs_mix < N
|
|
is_post = (offs_mix >= N) & (offs_mix < 2 * N)
|
|
gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2))
|
|
logits = mixes * gate_scale + b_base
|
|
pre = tl.sigmoid(logits) + hc_eps
|
|
post_new = tl.sigmoid(logits) * post_mult
|
|
|
|
offs_n2 = tl.arange(0, N)
|
|
comb_logits = tl.zeros([N, N], dtype=tl.float32)
|
|
for r in tl.static_range(N):
|
|
for c in tl.static_range(N):
|
|
lane = 2 * N + r * N + c
|
|
v = tl.sum(tl.where(offs_mix == lane, logits, 0.0))
|
|
comb_logits += tl.where(
|
|
(offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0
|
|
)
|
|
row_max = tl.max(comb_logits, axis=1)
|
|
e = tl.exp(comb_logits - row_max[:, None])
|
|
comb = e / tl.sum(e, axis=1)[:, None] + hc_eps
|
|
comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
|
|
for _ in range(SINKHORN - 1):
|
|
comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps)
|
|
comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps)
|
|
|
|
post_g = tl.sum(
|
|
tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0),
|
|
axis=1,
|
|
)
|
|
pre_g = tl.sum(
|
|
tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1
|
|
)
|
|
tl.store(post_out_ptr + t * N + offs_n2, post_g)
|
|
tl.store(pre_out_ptr + t * N + offs_n2, pre_g)
|
|
tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb)
|
|
|
|
|
|
|
|
@triton.jit
|
|
def _mhc_stage2_kernel(
|
|
sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
|
|
post_out_ptr, comb_out_ptr, pre_out_ptr,
|
|
rms_eps, hc_eps, post_mult,
|
|
SINKHORN: tl.constexpr,
|
|
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
|
|
NS: tl.constexpr, BLK_NS: tl.constexpr,
|
|
):
|
|
"""Gate math alone (the two-pass variant, whose stage 3 consumes these pre gates)."""
|
|
_mhc_gates(
|
|
tl.program_id(0).to(tl.int64), sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
|
|
rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _mhc_stage3_kernel(
|
|
res_out_ptr, pre_ptr, li_out_ptr,
|
|
H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr,
|
|
):
|
|
"""layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks."""
|
|
t = tl.program_id(0).to(tl.int64)
|
|
hb = tl.program_id(1)
|
|
offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
|
|
h_mask = offs_h < H
|
|
offs_n = tl.arange(0, N)
|
|
pre = tl.load(pre_ptr + t * N + offs_n)
|
|
li = tl.zeros([BLOCK_H], dtype=tl.float32)
|
|
for n in tl.static_range(N):
|
|
r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
|
|
li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
|
|
tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
|
|
|
|
|
|
@triton.jit
|
|
def _mhc_stage23_kernel(
|
|
res_out_ptr, pre_ptr, li_out_ptr,
|
|
sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr,
|
|
post_out_ptr, comb_out_ptr, pre_out_ptr,
|
|
rms_eps, hc_eps, post_mult,
|
|
SINKHORN: tl.constexpr,
|
|
H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr,
|
|
NS: tl.constexpr, BLK_NS: tl.constexpr, BLOCK_H: tl.constexpr,
|
|
):
|
|
"""Single-pass mHC: the layer input mixes with the PREVIOUS sublayer's pre gates, so the
|
|
gate math for the next sublayer (stage 2) does not gate stage 3 -- one launch does both:
|
|
every program mixes one hidden chunk, program 0 of each token also reduces the split
|
|
partials and writes the next sublayer's gates."""
|
|
t = tl.program_id(0).to(tl.int64)
|
|
hb = tl.program_id(1)
|
|
offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H)
|
|
h_mask = offs_h < H
|
|
offs_n = tl.arange(0, N)
|
|
pre = tl.load(pre_ptr + t * N + offs_n)
|
|
li = tl.zeros([BLOCK_H], dtype=tl.float32)
|
|
for n in tl.static_range(N):
|
|
r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32)
|
|
li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r
|
|
tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask)
|
|
if hb == 0:
|
|
_mhc_gates(
|
|
t, sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, post_out_ptr, comb_out_ptr, pre_out_ptr,
|
|
rms_eps, hc_eps, post_mult, SINKHORN, H, N, MIX, BLK_MIX, NS, BLK_NS,
|
|
)
|
|
|
|
|
|
def _stage1_hidden_tile(h: int) -> int:
|
|
"""The stage-1 hidden split. It fixes the fp32 reduction order over the hidden width, so it
|
|
depends on the width alone and results reproduce across processes, machines and chunk sizes.
|
|
128 gives 32 / 40 programs per token at widths 4096 / 5120."""
|
|
return min(128, triton.next_power_of_2(h))
|
|
|
|
|
|
def _mhc_fused_launch(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor | None,
|
|
comb_mix: torch.Tensor | None,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
pre_mix_in: torch.Tensor | None,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""The three stages behind both variants. ``pre_mix_in`` selects single-pass: the
|
|
layer input is mixed with it instead of with the pre gates stage 2 just computed
|
|
(those are still returned, for the next sublayer). Returns (residual_new, post,
|
|
comb, pre_new [T,N] fp32, layer_input)."""
|
|
t, n, h = residual.shape
|
|
mix = 2 * n + n * n
|
|
assert fn.shape == (mix, n * h) and fn.dtype == torch.float32
|
|
residual = residual.contiguous()
|
|
has_post = post_mix is not None
|
|
dev = residual.device
|
|
|
|
res_out = torch.empty_like(residual)
|
|
post_out = torch.empty(t, n, dtype=torch.float32, device=dev)
|
|
comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev)
|
|
li_out = torch.empty(t, h, dtype=residual.dtype, device=dev)
|
|
|
|
block_h = split = _stage1_hidden_tile(h)
|
|
ns = triton.cdiv(h, split)
|
|
blk_mix = triton.next_power_of_2(mix)
|
|
sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev)
|
|
mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev)
|
|
pre_out = torch.empty(t, n, dtype=torch.float32, device=dev)
|
|
|
|
_mhc_stage1_kernel[(t, ns)](
|
|
x.contiguous() if has_post else residual, # dummy ptr when unused
|
|
residual,
|
|
post_mix.contiguous().view(t, n) if has_post else post_out,
|
|
comb_mix.contiguous() if has_post else comb_out,
|
|
fn, res_out, sq_part, mix_part,
|
|
H=h, N=n, MIX=mix, BLK_MIX=blk_mix,
|
|
SPLIT=split, BLOCK_H=block_h, NS=ns,
|
|
HAS_POST=has_post,
|
|
num_warps=4, num_stages=2,
|
|
)
|
|
block_h3 = min(1024, triton.next_power_of_2(h))
|
|
if pre_mix_in is None:
|
|
# two-pass: stage 3 mixes with the gates stage 2 just produced
|
|
_mhc_stage2_kernel[(t,)](
|
|
sq_part, mix_part, hc_scale, hc_base,
|
|
post_out, comb_out, pre_out,
|
|
rms_eps, hc_eps, post_mult,
|
|
SINKHORN=sinkhorn_repeat,
|
|
H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns),
|
|
num_warps=1,
|
|
)
|
|
_mhc_stage3_kernel[(t, triton.cdiv(h, block_h3))](
|
|
res_out, pre_out, li_out,
|
|
H=h, N=n, BLOCK_H=block_h3,
|
|
num_warps=4,
|
|
)
|
|
else:
|
|
# single-pass: the mix uses the previous sublayer's gates, so stages 2 and 3 share a launch
|
|
assert pre_mix_in.shape == (t, n), (pre_mix_in.shape, (t, n))
|
|
_mhc_stage23_kernel[(t, triton.cdiv(h, block_h3))](
|
|
res_out, pre_mix_in.to(torch.float32).contiguous(), li_out,
|
|
sq_part, mix_part, hc_scale, hc_base,
|
|
post_out, comb_out, pre_out,
|
|
rms_eps, hc_eps, post_mult,
|
|
SINKHORN=sinkhorn_repeat,
|
|
H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, BLK_NS=triton.next_power_of_2(ns), BLOCK_H=block_h3,
|
|
num_warps=4,
|
|
)
|
|
return res_out, post_out.view(t, n, 1), comb_out, pre_out, li_out
|
|
|
|
|
|
def mhc_fused_post_pre_triton(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor | None,
|
|
comb_mix: torch.Tensor | None,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage
|
|
split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns
|
|
(residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32,
|
|
layer_input [T,H] bf16)."""
|
|
res_out, post_out, comb_out, _pre_out, li_out = _mhc_fused_launch(
|
|
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=None,
|
|
)
|
|
return res_out, post_out, comb_out, li_out
|
|
|
|
|
|
def mhc_fused_post_pre_single_pass_triton(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor | None,
|
|
comb_mix: torch.Tensor | None,
|
|
pre_mix_in: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Single-pass mHC (DeepSeek-V4.1): same fused stages, but the layer input is mixed
|
|
with the PREVIOUS sublayer's ``pre_mix_in`` and this sublayer's pre gates come back
|
|
as ``pre_new`` for the next one. Returns (residual_new, post [T,N,1], comb [T,N,N],
|
|
pre_new [T,N] fp32, layer_input)."""
|
|
return _mhc_fused_launch(
|
|
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat, pre_mix_in=pre_mix_in,
|
|
)
|
|
|
|
|
|
__all__ = ["mhc_fused_post_pre_triton", "mhc_fused_post_pre_single_pass_triton"]
|