Files
Xiaoze Fanandandy-yang-1 127c85665b feat(deepseek-v41): support text and vision serving (#603)
* test(deepseek-v41): add independent text and vision reference

* feat(quant): support block-32 and unrounded inference outputs

* feat(attention): add csa2 packed cache and sparse backend

* feat(deepseek-v41): serve text and images through shared runtime

* perf(mhc): tune batch-stable reduction tiles

* perf(moe): autotune fp4 decode launch geometry

* perf(moe): admit recurring misses against actual lru victims

* revert: perf(moe): admit recurring misses against actual lru victims

This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461.

* perf(mhc): derive the reduction tile from the width, not a timing

* perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch

* fix(deepseek-v41): fence engram staging against in-flight lookups

* fix(moe): return the cpu routed sum in fp32 and round it in python

* fix(deepseek-v41): keep the vision tower and aligner unquantized

* fix(scheduler): admit bounded-replay hits by the history their resume reads

* refactor(deepseek-v41): name the attention family after the model, not csa2

* refactor(kvcache): import the dsv41 pool only for dsv41 models

* refactor(deepseek-v41): name the replay switch --swa-decoder-replay

* fix(server): keep reasoning_effort a string on the wire

* refactor(models): keep the prefill replay hooks off BaseLLMModel

* refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill

* fix(moe): weight ds_fp4 routes in the down epilogue again

* docs(models): anchor the v4.1 replay paragraph on the tech report

* refactor(deepseek-v41): run the decoder source's narrowed prefill on Block

* refactor(kvcache): let the dsv41 pool own the history a resume keeps live

* refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match

* refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args

* refactor(deepseek-v41): keep the engram table locator in the model package

* refactor(deepseek-v41): drop the fp32 lm head output

- return the head's bf16 logits, as DeepSeek-V4 does
- revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used
- remove test_head_preserves_fp32_logit_margin
- cast test_vision's prefill logits to fp32 before comparing, as the harness does

* refactor(quant): carry the w4a8 activation block on MoEConfig

- drop ActDesc and QuantScheme.act; the scheme only describes stored tensors
- Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook
- the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it

* refactor(deepseek-v41): size the kv pool from the args, not a geometry copy

- delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args
- source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone
- the cost model owns the tier row formats, row bytes and the private window layers
- the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend

* refactor(kvcache): move the dsv4 pool into kvcache/dsv4

* refactor(kvcache): move the dsv41 pool into kvcache/dsv4

* chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks

* test(deepseek-v41): drop tests that repeat other coverage

* test(deepseek-v41): load the tiny model's resident experts as banks

* fix(server): parse deepseek-v4.1's space-led dsml tool calls

---------

Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
2026-10-07 00:15:11 -07:00

140 lines
5.6 KiB
Python

"""DeepSeek ViT and 3x3 pixel-unshuffle aligner, with the shared encoder weight streamer."""
from __future__ import annotations
import torch
import torch.nn.functional as F
from freetoken.layers import BaseOP, LinearReplicated, OPList
from freetoken.models.weight_stream import BlockWeightStreamer
class VisionNorm(BaseOP):
def __init__(self, dim: int):
self.weight = torch.empty(dim, dtype=torch.float32)
def forward(self, x: torch.Tensor) -> torch.Tensor:
y = x.float()
return (y * torch.rsqrt(y.square().mean(-1, keepdim=True) + 1e-6) * self.weight).to(x.dtype)
class VisionAttention(BaseOP):
def __init__(self, vc):
self.n_heads = vc.num_attention_heads
self.head_dim = vc.hidden_size // self.n_heads
self.wqkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=True)
self.wo = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=True)
def forward(self, x, cos, sin):
n = x.shape[0]
q, k, v = (t.reshape(n, self.n_heads, self.head_dim) for t in self.wqkv.forward(x).chunk(3, -1))
def rotary(t):
a, b = t.float().chunk(2, -1)
return torch.cat((a * cos - b * sin, b * cos + a * sin), -1).to(t.dtype)
q, k = rotary(q), rotary(k)
q, k, v = (t.transpose(0, 1).unsqueeze(0) for t in (q, k, v))
y = F.scaled_dot_product_attention(q, k, v)
return self.wo.forward(y[0].transpose(0, 1).reshape(n, -1))
class VisionMLP(BaseOP):
def __init__(self, vc):
self.w1 = LinearReplicated(vc.hidden_size, 2 * vc.intermediate_size, has_bias=False)
self.w2 = LinearReplicated(vc.intermediate_size, vc.hidden_size, has_bias=False)
def forward(self, x):
gate, up = self.w1.forward(x).chunk(2, -1)
return self.w2.forward(F.silu(gate) * up)
class VisionBlock(BaseOP):
def __init__(self, vc):
self.norm1 = VisionNorm(vc.hidden_size)
self.attn = VisionAttention(vc)
self.norm2 = VisionNorm(vc.hidden_size)
self.mlp = VisionMLP(vc)
def forward(self, x, cos, sin):
x = x + self.attn.forward(self.norm1.forward(x), cos, sin)
return x + self.mlp.forward(self.norm2.forward(x))
class PatchEmbed(BaseOP):
def __init__(self, vc):
self.proj = LinearReplicated(3 * vc.patch_size**2, vc.hidden_size, has_bias=True)
def forward(self, x):
return self.proj.forward(x.flatten(1))
class VisionTransformer(BaseOP):
def __init__(self, vc):
self.patch_embed = PatchEmbed(vc)
self.blocks = OPList([VisionBlock(vc) for _ in range(vc.num_hidden_layers)])
self.norm = VisionNorm(vc.hidden_size)
self._vc = vc
self._streamer = None
def place_weights(self, mode: str) -> None:
if mode == "host" and self._streamer is None:
self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)
elif mode == "gpu" and self._streamer is not None:
self._streamer.unstream()
self._streamer = None
elif mode not in ("gpu", "host"):
raise ValueError(f"unknown vision weight placement {mode!r}")
def forward(self, patches, h: int, w: int):
x = self.patch_embed.forward(patches)
dim = self._vc.hidden_size // self._vc.num_attention_heads // 2
inv = 1.0 / (self._vc.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim))
rows = torch.arange(h, device=x.device)[:, None].expand(h, w)
cols = torch.arange(w, device=x.device)[None, :].expand(h, w)
angles = (torch.stack((rows, cols), -1).reshape(-1, 2, 1).float() * inv).flatten(1)
cos, sin = angles.cos().unsqueeze(1), angles.sin().unsqueeze(1)
blocks = enumerate(self.blocks.op_list) if self._streamer is None else self._streamer.blocks(self.blocks.op_list)
for _, block in blocks:
x = block.forward(x, cos, sin)
return self.norm.forward(x)
class VisionAligner(BaseOP):
def __init__(self, vc, dim: int):
self.ratio = vc.downsample_ratio
self.w1 = LinearReplicated(vc.hidden_size * self.ratio**2, dim, has_bias=True)
self.w2 = LinearReplicated(dim, dim, has_bias=True)
def forward(self, x, h: int, w: int):
r = self.ratio
x = x.view(h, w, -1).permute(2, 0, 1)
x = F.pad(x, (0, -w % r, 0, -h % r))
x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).T
return self.w2.forward(F.gelu(self.w1.forward(x)))
class DeepseekV41Vision(BaseOP):
def __init__(self, vc, dim: int, *, num_layers: int, num_experts: int):
self.vision = VisionTransformer(vc)
self.aligner = VisionAligner(vc, dim)
self.image_start = torch.empty(dim)
self.image_end = torch.empty(dim)
self.image_newline = torch.empty(dim)
# Keep all image-only parameters under visual, including FTW/text-only filtering.
self.routing_bias = torch.empty(num_layers, num_experts, dtype=torch.float32)
def place_weights(self, mode: str) -> None:
self.vision.place_weights(mode)
@torch.inference_mode()
def forward(self, item):
_, h, w = item.grid_thw
patches = item.feature.to(device=self.image_start.device, dtype=self.image_start.dtype)
x = self.aligner.forward(self.vision.forward(patches, h, w), h, w)
r = self.aligner.ratio
nh, nw = (h + r - 1) // r, (w + r - 1) // r
out = x.new_empty(nh * (nw + 1) + 2, x.shape[-1])
out[0], out[-1] = self.image_start, self.image_end
rows = out[1:-1].view(nh, nw + 1, -1)
rows[:, :nw] = x.view(nh, nw, -1)
rows[:, nw] = self.image_newline
return out