Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

132 lines
6.5 KiB
Python

"""DSV41 KV compressor: ``compress_ratio`` consecutive tokens -> one latent (reference ``Compressor``).
* ratio 1 (the CED decoder source): a plain bf16 projection + RMSNorm, one latent per token.
* ratio > 1 (the encoder sources): ``wkv`` / ``wgate`` promoted to fp32, a softmax over the group's
gate scores pools the group's ``wkv`` rows; the trailing partial group is carried across chunks
and decode steps in the pool's compress-state ring (per window page, so a page-aligned radix hit
resumes it by value -- see ``DSV41SparseAttnBackend`` for the addressing).
Returns the latent BEFORE RoPE: the indexer key is projected from the unrotated latent, the
attention layer rotates and quantizes it afterwards.
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
from freetoken.core import get_global_ctx
from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
from .args import DeepseekV41Args
class Compressor(BaseOP):
def __init__(self, args: DeepseekV41Args, layer_id: int, ratio: int, *, quant_config=None, prefix: str = ""):
assert ratio >= 1
self.layer_id = layer_id
self.ratio = ratio
self.dim = args.dim
self.head_dim = args.head_dim
self.window = args.window_size
if ratio > 1:
# fp32 like the reference (the softmax pooling runs in fp32); the checkpoint ships bf16
self.wkv = torch.empty(args.head_dim, args.dim, dtype=torch.float32)
self.wgate = torch.empty(args.head_dim, args.dim, dtype=torch.float32)
else:
self.wkv = LinearReplicated(args.dim, args.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")
self.norm = RMSNorm(args.head_dim, args.norm_eps)
@property
def attn(self):
return get_global_ctx().attn_backend
def _project(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""fp32 ``wkv`` / ``wgate`` projections."""
xf = x.float()
return F.linear(xf, self.wkv), F.linear(xf, self.wgate)
# ----- prefill -----------------------------------------------------------------------
def needs_tail_carry(self, start_pos: int) -> bool:
"""Whether a prefill from ``start_pos`` resumes a partial group (never at a page-aligned start)."""
return self.ratio > 1 and start_pos % self.ratio != 0
def forward_prefill(
self, x: torch.Tensor, start_pos: int, window_slots: torch.Tensor, tail_window_slot: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Compress the tokens ``[start_pos, start_pos + n)`` of one request.
``window_slots`` are the tokens' window slots (the ring block of each window page carries the
partial group); ``tail_window_slot`` is the previous token's slot, needed only when
``needs_tail_carry(start_pos)``. Returns ``(latent [G, head_dim] bf16 pre-RoPE, group_starts [G]
int64)`` for every group that completed in this call.
"""
n = x.shape[0]
device = x.device
if self.ratio == 1:
latent = self.norm.forward(self.wkv.forward(x))
return latent, torch.arange(start_pos, start_pos + n, device=device)
ratio, d = self.ratio, self.head_dim
kv, score = self._project(x)
# the partial group in progress at start_pos: its rows live in the previous token's ring block
offset = start_pos % ratio
if offset:
assert tail_window_slot is not None, "resuming mid-group needs the previous token's window slot"
carry = self.attn.read_carry(self.layer_id, tail_window_slot) # [ratio, 2d]
kv = torch.cat([carry[:offset, :d], kv], dim=0)
score = torch.cat([carry[:offset, d:], score], dim=0)
total = kv.shape[0]
groups = total // ratio
remainder = total % ratio
cut = groups * ratio
if groups:
kv_g = kv[:cut].view(groups, ratio, d)
score_g = score[:cut].view(groups, ratio, d)
pooled = (kv_g * score_g.softmax(dim=1)).sum(dim=1)
latent = self.norm.forward(pooled.to(x.dtype))
else:
latent = x.new_empty(0, d)
group_starts = (start_pos - offset) + ratio * torch.arange(groups, device=device)
# ring: the reset block at every page boundary crossed (a page holds whole groups), and the
# in-progress tail (the remainder rows) on the last page so decode / the next chunk resume
end = start_pos + n
self.attn.write_boundary_carries(self.layer_id, lo=start_pos, hi=end, window_slots=window_slots)
if end % self.window != 0:
block = self._carry_block(kv[cut:], score[cut:], remainder)
self.attn.write_carry(self.layer_id, int(window_slots[-1].item()), block)
return latent, group_starts
def _carry_block(self, kv_tail: torch.Tensor, score_tail: torch.Tensor, filled: int) -> torch.Tensor:
ratio, d = self.ratio, self.head_dim
ks = kv_tail.new_zeros(ratio, d)
ss = kv_tail.new_full((ratio, d), float("-inf"))
if filled:
ks[:filled] = kv_tail
ss[:filled] = score_tail
return torch.cat([ks, ss], dim=-1)
# ----- decode ------------------------------------------------------------------------
def forward_decode(self, x: torch.Tensor, pos: torch.Tensor, prev_window_slots: torch.Tensor, window_slots: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""One token per row. Returns ``(latent [B, head_dim] bf16 pre-RoPE, completed [B] bool)``;
a row whose group did not complete carries a meaningless latent the caller discards."""
B = x.shape[0]
if self.ratio == 1:
latent = self.norm.forward(self.wkv.forward(x))
return latent, torch.ones(B, dtype=torch.bool, device=x.device)
from freetoken.kernel.triton.dsv4.compress import gated_pool
ratio, d = self.ratio, self.head_dim
kv, score = self._project(x)
idx = (pos % ratio).view(B, 1, 1).expand(B, 1, d)
block = self.attn.read_carry_blocks(self.layer_id, prev_window_slots) # [B, ratio, 2d]
ks = block[..., :d].clone().scatter_(1, idx, kv.unsqueeze(1))
ss = block[..., d:].clone().scatter_(1, idx, score.unsqueeze(1))
self.attn.write_carry_blocks(self.layer_id, window_slots, torch.cat([ks, ss], dim=-1))
pooled = gated_pool(ks, ss, x.dtype).view(B, d)
completed = (pos + 1) % ratio == 0
return self.norm.forward(pooled), completed
__all__ = ["Compressor"]