mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
132 lines
6.5 KiB
Python
132 lines
6.5 KiB
Python
"""DSV41 KV compressor: ``compress_ratio`` consecutive tokens -> one latent (reference ``Compressor``).
|
|
|
|
* ratio 1 (the CED decoder source): a plain bf16 projection + RMSNorm, one latent per token.
|
|
* ratio > 1 (the encoder sources): ``wkv`` / ``wgate`` promoted to fp32, a softmax over the group's
|
|
gate scores pools the group's ``wkv`` rows; the trailing partial group is carried across chunks
|
|
and decode steps in the pool's compress-state ring (per window page, so a page-aligned radix hit
|
|
resumes it by value -- see ``DSV41SparseAttnBackend`` for the addressing).
|
|
|
|
Returns the latent BEFORE RoPE: the indexer key is projected from the unrotated latent, the
|
|
attention layer rotates and quantizes it afterwards.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
|
|
|
|
from .args import DeepseekV41Args
|
|
|
|
|
|
class Compressor(BaseOP):
|
|
def __init__(self, args: DeepseekV41Args, layer_id: int, ratio: int, *, quant_config=None, prefix: str = ""):
|
|
assert ratio >= 1
|
|
self.layer_id = layer_id
|
|
self.ratio = ratio
|
|
self.dim = args.dim
|
|
self.head_dim = args.head_dim
|
|
self.window = args.window_size
|
|
if ratio > 1:
|
|
# fp32 like the reference (the softmax pooling runs in fp32); the checkpoint ships bf16
|
|
self.wkv = torch.empty(args.head_dim, args.dim, dtype=torch.float32)
|
|
self.wgate = torch.empty(args.head_dim, args.dim, dtype=torch.float32)
|
|
else:
|
|
self.wkv = LinearReplicated(args.dim, args.head_dim, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.wkv")
|
|
self.norm = RMSNorm(args.head_dim, args.norm_eps)
|
|
|
|
@property
|
|
def attn(self):
|
|
return get_global_ctx().attn_backend
|
|
|
|
def _project(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""fp32 ``wkv`` / ``wgate`` projections."""
|
|
xf = x.float()
|
|
return F.linear(xf, self.wkv), F.linear(xf, self.wgate)
|
|
|
|
# ----- prefill -----------------------------------------------------------------------
|
|
def needs_tail_carry(self, start_pos: int) -> bool:
|
|
"""Whether a prefill from ``start_pos`` resumes a partial group (never at a page-aligned start)."""
|
|
return self.ratio > 1 and start_pos % self.ratio != 0
|
|
|
|
def forward_prefill(
|
|
self, x: torch.Tensor, start_pos: int, window_slots: torch.Tensor, tail_window_slot: int | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Compress the tokens ``[start_pos, start_pos + n)`` of one request.
|
|
|
|
``window_slots`` are the tokens' window slots (the ring block of each window page carries the
|
|
partial group); ``tail_window_slot`` is the previous token's slot, needed only when
|
|
``needs_tail_carry(start_pos)``. Returns ``(latent [G, head_dim] bf16 pre-RoPE, group_starts [G]
|
|
int64)`` for every group that completed in this call.
|
|
"""
|
|
n = x.shape[0]
|
|
device = x.device
|
|
if self.ratio == 1:
|
|
latent = self.norm.forward(self.wkv.forward(x))
|
|
return latent, torch.arange(start_pos, start_pos + n, device=device)
|
|
|
|
ratio, d = self.ratio, self.head_dim
|
|
kv, score = self._project(x)
|
|
# the partial group in progress at start_pos: its rows live in the previous token's ring block
|
|
offset = start_pos % ratio
|
|
if offset:
|
|
assert tail_window_slot is not None, "resuming mid-group needs the previous token's window slot"
|
|
carry = self.attn.read_carry(self.layer_id, tail_window_slot) # [ratio, 2d]
|
|
kv = torch.cat([carry[:offset, :d], kv], dim=0)
|
|
score = torch.cat([carry[:offset, d:], score], dim=0)
|
|
total = kv.shape[0]
|
|
groups = total // ratio
|
|
remainder = total % ratio
|
|
cut = groups * ratio
|
|
if groups:
|
|
kv_g = kv[:cut].view(groups, ratio, d)
|
|
score_g = score[:cut].view(groups, ratio, d)
|
|
pooled = (kv_g * score_g.softmax(dim=1)).sum(dim=1)
|
|
latent = self.norm.forward(pooled.to(x.dtype))
|
|
else:
|
|
latent = x.new_empty(0, d)
|
|
group_starts = (start_pos - offset) + ratio * torch.arange(groups, device=device)
|
|
# ring: the reset block at every page boundary crossed (a page holds whole groups), and the
|
|
# in-progress tail (the remainder rows) on the last page so decode / the next chunk resume
|
|
end = start_pos + n
|
|
self.attn.write_boundary_carries(self.layer_id, lo=start_pos, hi=end, window_slots=window_slots)
|
|
if end % self.window != 0:
|
|
block = self._carry_block(kv[cut:], score[cut:], remainder)
|
|
self.attn.write_carry(self.layer_id, int(window_slots[-1].item()), block)
|
|
return latent, group_starts
|
|
|
|
def _carry_block(self, kv_tail: torch.Tensor, score_tail: torch.Tensor, filled: int) -> torch.Tensor:
|
|
ratio, d = self.ratio, self.head_dim
|
|
ks = kv_tail.new_zeros(ratio, d)
|
|
ss = kv_tail.new_full((ratio, d), float("-inf"))
|
|
if filled:
|
|
ks[:filled] = kv_tail
|
|
ss[:filled] = score_tail
|
|
return torch.cat([ks, ss], dim=-1)
|
|
|
|
# ----- decode ------------------------------------------------------------------------
|
|
def forward_decode(self, x: torch.Tensor, pos: torch.Tensor, prev_window_slots: torch.Tensor, window_slots: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""One token per row. Returns ``(latent [B, head_dim] bf16 pre-RoPE, completed [B] bool)``;
|
|
a row whose group did not complete carries a meaningless latent the caller discards."""
|
|
B = x.shape[0]
|
|
if self.ratio == 1:
|
|
latent = self.norm.forward(self.wkv.forward(x))
|
|
return latent, torch.ones(B, dtype=torch.bool, device=x.device)
|
|
|
|
from freetoken.kernel.triton.dsv4.compress import gated_pool
|
|
|
|
ratio, d = self.ratio, self.head_dim
|
|
kv, score = self._project(x)
|
|
idx = (pos % ratio).view(B, 1, 1).expand(B, 1, d)
|
|
block = self.attn.read_carry_blocks(self.layer_id, prev_window_slots) # [B, ratio, 2d]
|
|
ks = block[..., :d].clone().scatter_(1, idx, kv.unsqueeze(1))
|
|
ss = block[..., d:].clone().scatter_(1, idx, score.unsqueeze(1))
|
|
self.attn.write_carry_blocks(self.layer_id, window_slots, torch.cat([ks, ss], dim=-1))
|
|
pooled = gated_pool(ks, ss, x.dtype).view(B, d)
|
|
completed = (pos + 1) % ratio == 0
|
|
return self.norm.forward(pooled), completed
|
|
|
|
|
|
__all__ = ["Compressor"]
|