mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
254 lines
10 KiB
Python
254 lines
10 KiB
Python
"""mHC -- Manifold-Constrained Hyper-Connections (GLM-5.3-Flash; arXiv 2512.24880).
|
|
|
|
The residual stream is widened to ``hc_mult`` (n) parallel streams. Around every
|
|
sublayer the streams are mixed by three learned, token-dependent maps computed
|
|
from ONE fp32 GEMM over the flattened streams (``fn [2n+n^2, n*hidden]``),
|
|
RMS-normalized over the full ``n*hidden`` vector and split into:
|
|
|
|
* ``pre_mix [n]`` sigmoid gates: the sublayer input is ``sum_i pre_i * res_i``
|
|
* ``post_mix [n]`` sigmoid*mult gates: how much sublayer output enters each stream
|
|
* ``comb_mix [n, n]`` softmax + Sinkhorn-projected (approximately doubly-stochastic)
|
|
stream-mixing matrix -- the "manifold constraint": mixing
|
|
neither amplifies nor loses residual mass.
|
|
|
|
``mhc_post`` then rebuilds the streams: ``out_j = sum_i comb_ij * res_i + post_j * x``.
|
|
|
|
Semantics match vLLM's reference (``model_executor/kernels/mhc/torch.py``,
|
|
PR #53906) bit-for-bit in fp32; the per-layer weights are ``hc_{attn,ffn}_fn`` /
|
|
``_scale`` / ``_base`` from the checkpoint. This torch implementation is the
|
|
correctness baseline; the fused triton kernel (``kernel/triton/mhc.py``,
|
|
dispatched by ``mhc_fused_post_pre`` below) replaces it on CUDA and is
|
|
validated against this file.
|
|
|
|
Two variants share the gate math:
|
|
|
|
* **multi-pass** (GLM-5.3, DeepSeek-V4): ``pre`` is computed from the residual the
|
|
sublayer is about to read and applied immediately -- ``mhc_pre`` / ``mhc_fused_post_pre``.
|
|
* **single-pass** (DeepSeek-V4.1 "Single-Pass mHC", tech report sec. 2.4.1): every
|
|
sublayer consumes the ``pre`` gates the *previous* sublayer produced, so the input
|
|
mix no longer waits on this sublayer's own reduction -- ``mhc_pre_single_pass`` /
|
|
``mhc_fused_post_pre_single_pass``. The returned ``pre_new`` is what the next
|
|
sublayer applies; the model seeds the chain with a one-hot pre on stream 0.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
|
|
|
|
def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
|
"""[T, hidden] -> [T, n, hidden] by replication (model entry)."""
|
|
return x.unsqueeze(1).expand(-1, n, -1).contiguous()
|
|
|
|
|
|
def hc_contract(x: torch.Tensor) -> torch.Tensor:
|
|
"""[T, n, hidden] -> [T, hidden] by averaging (model exit)."""
|
|
return x.mean(dim=1)
|
|
|
|
|
|
def mhc_gates(
|
|
residual: torch.Tensor, # [T, n, hidden]
|
|
fn: torch.Tensor, # [2n + n^2, n*hidden] fp32
|
|
hc_scale: torch.Tensor, # [3] fp32
|
|
hc_base: torch.Tensor, # [2n + n^2] fp32
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""The coefficient predictor H(X): returns (pre_mix [T, n] fp32, post_mix [T, n, 1]
|
|
fp32, comb_mix [T, n, n] fp32) from one fp32 GEMM over the flattened streams."""
|
|
n, hidden = residual.shape[-2], residual.shape[-1]
|
|
t = residual.shape[0]
|
|
|
|
x = residual.reshape(t, n * hidden).to(torch.float32)
|
|
mixes = x @ fn.t()
|
|
# RMS over the FULL flattened n*hidden vector (not per stream).
|
|
mixes = mixes * torch.rsqrt(x.square().sum(-1, keepdim=True) / (n * hidden) + rms_eps)
|
|
|
|
pre_mix = torch.sigmoid(mixes[:, :n] * hc_scale[0] + hc_base[:n]) + hc_eps
|
|
post_mix = torch.sigmoid(mixes[:, n : 2 * n] * hc_scale[1] + hc_base[n : 2 * n])
|
|
post_mix = post_mix * post_mult
|
|
|
|
comb = mixes[:, 2 * n :].view(t, n, n) * hc_scale[2] + hc_base[2 * n :].view(1, n, n)
|
|
comb = torch.softmax(comb, dim=-1) + hc_eps
|
|
# Sinkhorn-Knopp projection toward the doubly-stochastic manifold: alternate
|
|
# column / row normalization, ``sinkhorn_repeat`` column steps in total.
|
|
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)
|
|
for _ in range(sinkhorn_repeat - 1):
|
|
comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_eps)
|
|
comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps)
|
|
return pre_mix, post_mix.view(t, n, 1), comb
|
|
|
|
|
|
def mhc_mix_input(residual: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
|
|
"""The sublayer input ``sum_i pre_i * res_i``: [T, n, hidden] x [T, n] -> [T, hidden]."""
|
|
return (pre_mix.unsqueeze(-1) * residual.to(torch.float32)).sum(dim=1).to(residual.dtype)
|
|
|
|
|
|
def mhc_pre(
|
|
residual: torch.Tensor, # [T, n, hidden] bf16
|
|
fn: torch.Tensor, # [2n + n^2, n*hidden] fp32
|
|
hc_scale: torch.Tensor, # [3] fp32
|
|
hc_base: torch.Tensor, # [2n + n^2] fp32
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Multi-pass: gates from ``residual``, pre applied to it right away. Returns
|
|
(post_mix [T, n, 1] fp32, comb_mix [T, n, n] fp32, layer_input [T, hidden] bf16)."""
|
|
pre_mix, post_mix, comb = mhc_gates(residual, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat)
|
|
return post_mix, comb, mhc_mix_input(residual, pre_mix)
|
|
|
|
|
|
def mhc_pre_single_pass(
|
|
residual: torch.Tensor, # [T, n, hidden]
|
|
pre_mix_in: torch.Tensor, # [T, n] fp32 -- the previous sublayer's pre gates
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Single-pass: gates from ``residual`` for THIS sublayer's post/comb and the NEXT
|
|
sublayer's pre; the input is mixed with ``pre_mix_in``. Returns (pre_new [T, n] fp32,
|
|
post_mix [T, n, 1] fp32, comb_mix [T, n, n] fp32, layer_input [T, hidden])."""
|
|
pre_new, post_mix, comb = mhc_gates(residual, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat)
|
|
return pre_new, post_mix, comb, mhc_mix_input(residual, pre_mix_in.to(torch.float32))
|
|
|
|
|
|
def mhc_post(
|
|
x: torch.Tensor, # [T, hidden] sublayer output
|
|
residual: torch.Tensor, # [T, n, hidden]
|
|
post_mix: torch.Tensor, # [T, n, 1] fp32
|
|
comb_mix: torch.Tensor, # [T, n, n] fp32
|
|
) -> torch.Tensor:
|
|
"""out_j = sum_i comb_ij * residual_i + post_j * x; returns [T, n, hidden]."""
|
|
mixed = torch.einsum(
|
|
"tij,tih->tjh", comb_mix.to(torch.float32), residual.to(torch.float32)
|
|
)
|
|
post = post_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
|
|
return (mixed + post).to(residual.dtype)
|
|
|
|
|
|
def mhc_fused_post_pre_torch(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor,
|
|
comb_mix: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Decomposed reference: hc_post then hc_pre; the fused triton kernel
|
|
replaces it on CUDA and is validated against this function."""
|
|
residual_new = mhc_post(x, residual, post_mix, comb_mix)
|
|
post_new, comb_new, layer_input = mhc_pre(
|
|
residual_new, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat
|
|
)
|
|
return residual_new, post_new, comb_new, layer_input
|
|
|
|
|
|
def mhc_fused_post_pre(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor,
|
|
comb_mix: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Apply the previous sublayer's hc_post, then this sublayer's hc_pre on the
|
|
updated streams. The fused triton kernel serves every batch size on CUDA
|
|
(deliberately no T threshold); the decomposed torch path serves CPU/tests."""
|
|
if residual.is_cuda:
|
|
from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton
|
|
|
|
return mhc_fused_post_pre_triton(
|
|
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat,
|
|
)
|
|
return mhc_fused_post_pre_torch(
|
|
x, residual, post_mix, comb_mix, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat,
|
|
)
|
|
|
|
|
|
def mhc_fused_post_pre_single_pass_torch(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor | None,
|
|
comb_mix: torch.Tensor | None,
|
|
pre_mix_in: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Decomposed single-pass reference: the previous sublayer's hc_post (skipped when
|
|
``post_mix is None`` -- the model's first sublayer), then this sublayer's gates with
|
|
the input mixed by ``pre_mix_in``. Returns (residual_new, post_new [T, n, 1],
|
|
comb_new [T, n, n], pre_new [T, n], layer_input)."""
|
|
residual_new = residual if post_mix is None else mhc_post(x, residual, post_mix, comb_mix)
|
|
pre_new, post_new, comb_new, layer_input = mhc_pre_single_pass(
|
|
residual_new, pre_mix_in, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat
|
|
)
|
|
return residual_new, post_new, comb_new, pre_new, layer_input
|
|
|
|
|
|
def mhc_fused_post_pre_single_pass(
|
|
x: torch.Tensor,
|
|
residual: torch.Tensor,
|
|
post_mix: torch.Tensor | None,
|
|
comb_mix: torch.Tensor | None,
|
|
pre_mix_in: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_eps: float,
|
|
post_mult: float,
|
|
sinkhorn_repeat: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Single-pass sublayer boundary (DeepSeek-V4.1): fused triton on CUDA, torch elsewhere."""
|
|
if residual.is_cuda:
|
|
from freetoken.kernel.triton.mhc import mhc_fused_post_pre_single_pass_triton
|
|
|
|
return mhc_fused_post_pre_single_pass_triton(
|
|
x, residual, post_mix, comb_mix, pre_mix_in, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat,
|
|
)
|
|
return mhc_fused_post_pre_single_pass_torch(
|
|
x, residual, post_mix, comb_mix, pre_mix_in, fn, hc_scale, hc_base,
|
|
rms_eps, hc_eps, post_mult, sinkhorn_repeat,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"hc_expand",
|
|
"hc_contract",
|
|
"mhc_gates",
|
|
"mhc_mix_input",
|
|
"mhc_pre",
|
|
"mhc_pre_single_pass",
|
|
"mhc_post",
|
|
"mhc_fused_post_pre",
|
|
"mhc_fused_post_pre_torch",
|
|
"mhc_fused_post_pre_single_pass",
|
|
"mhc_fused_post_pre_single_pass_torch",
|
|
]
|