mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
140 lines
5.6 KiB
Python
140 lines
5.6 KiB
Python
"""DeepSeek ViT and 3x3 pixel-unshuffle aligner, with the shared encoder weight streamer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from freetoken.layers import BaseOP, LinearReplicated, OPList
|
|
from freetoken.models.weight_stream import BlockWeightStreamer
|
|
|
|
|
|
class VisionNorm(BaseOP):
|
|
def __init__(self, dim: int):
|
|
self.weight = torch.empty(dim, dtype=torch.float32)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
y = x.float()
|
|
return (y * torch.rsqrt(y.square().mean(-1, keepdim=True) + 1e-6) * self.weight).to(x.dtype)
|
|
|
|
|
|
class VisionAttention(BaseOP):
|
|
def __init__(self, vc):
|
|
self.n_heads = vc.num_attention_heads
|
|
self.head_dim = vc.hidden_size // self.n_heads
|
|
self.wqkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=True)
|
|
self.wo = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=True)
|
|
|
|
def forward(self, x, cos, sin):
|
|
n = x.shape[0]
|
|
q, k, v = (t.reshape(n, self.n_heads, self.head_dim) for t in self.wqkv.forward(x).chunk(3, -1))
|
|
def rotary(t):
|
|
a, b = t.float().chunk(2, -1)
|
|
return torch.cat((a * cos - b * sin, b * cos + a * sin), -1).to(t.dtype)
|
|
q, k = rotary(q), rotary(k)
|
|
q, k, v = (t.transpose(0, 1).unsqueeze(0) for t in (q, k, v))
|
|
y = F.scaled_dot_product_attention(q, k, v)
|
|
return self.wo.forward(y[0].transpose(0, 1).reshape(n, -1))
|
|
|
|
|
|
class VisionMLP(BaseOP):
|
|
def __init__(self, vc):
|
|
self.w1 = LinearReplicated(vc.hidden_size, 2 * vc.intermediate_size, has_bias=False)
|
|
self.w2 = LinearReplicated(vc.intermediate_size, vc.hidden_size, has_bias=False)
|
|
|
|
def forward(self, x):
|
|
gate, up = self.w1.forward(x).chunk(2, -1)
|
|
return self.w2.forward(F.silu(gate) * up)
|
|
|
|
|
|
class VisionBlock(BaseOP):
|
|
def __init__(self, vc):
|
|
self.norm1 = VisionNorm(vc.hidden_size)
|
|
self.attn = VisionAttention(vc)
|
|
self.norm2 = VisionNorm(vc.hidden_size)
|
|
self.mlp = VisionMLP(vc)
|
|
|
|
def forward(self, x, cos, sin):
|
|
x = x + self.attn.forward(self.norm1.forward(x), cos, sin)
|
|
return x + self.mlp.forward(self.norm2.forward(x))
|
|
|
|
|
|
class PatchEmbed(BaseOP):
|
|
def __init__(self, vc):
|
|
self.proj = LinearReplicated(3 * vc.patch_size**2, vc.hidden_size, has_bias=True)
|
|
|
|
def forward(self, x):
|
|
return self.proj.forward(x.flatten(1))
|
|
|
|
|
|
class VisionTransformer(BaseOP):
|
|
def __init__(self, vc):
|
|
self.patch_embed = PatchEmbed(vc)
|
|
self.blocks = OPList([VisionBlock(vc) for _ in range(vc.num_hidden_layers)])
|
|
self.norm = VisionNorm(vc.hidden_size)
|
|
self._vc = vc
|
|
self._streamer = None
|
|
|
|
def place_weights(self, mode: str) -> None:
|
|
if mode == "host" and self._streamer is None:
|
|
self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)
|
|
elif mode == "gpu" and self._streamer is not None:
|
|
self._streamer.unstream()
|
|
self._streamer = None
|
|
elif mode not in ("gpu", "host"):
|
|
raise ValueError(f"unknown vision weight placement {mode!r}")
|
|
|
|
def forward(self, patches, h: int, w: int):
|
|
x = self.patch_embed.forward(patches)
|
|
dim = self._vc.hidden_size // self._vc.num_attention_heads // 2
|
|
inv = 1.0 / (self._vc.rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim))
|
|
rows = torch.arange(h, device=x.device)[:, None].expand(h, w)
|
|
cols = torch.arange(w, device=x.device)[None, :].expand(h, w)
|
|
angles = (torch.stack((rows, cols), -1).reshape(-1, 2, 1).float() * inv).flatten(1)
|
|
cos, sin = angles.cos().unsqueeze(1), angles.sin().unsqueeze(1)
|
|
blocks = enumerate(self.blocks.op_list) if self._streamer is None else self._streamer.blocks(self.blocks.op_list)
|
|
for _, block in blocks:
|
|
x = block.forward(x, cos, sin)
|
|
return self.norm.forward(x)
|
|
|
|
|
|
class VisionAligner(BaseOP):
|
|
def __init__(self, vc, dim: int):
|
|
self.ratio = vc.downsample_ratio
|
|
self.w1 = LinearReplicated(vc.hidden_size * self.ratio**2, dim, has_bias=True)
|
|
self.w2 = LinearReplicated(dim, dim, has_bias=True)
|
|
|
|
def forward(self, x, h: int, w: int):
|
|
r = self.ratio
|
|
x = x.view(h, w, -1).permute(2, 0, 1)
|
|
x = F.pad(x, (0, -w % r, 0, -h % r))
|
|
x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).T
|
|
return self.w2.forward(F.gelu(self.w1.forward(x)))
|
|
|
|
|
|
class DeepseekV41Vision(BaseOP):
|
|
def __init__(self, vc, dim: int, *, num_layers: int, num_experts: int):
|
|
self.vision = VisionTransformer(vc)
|
|
self.aligner = VisionAligner(vc, dim)
|
|
self.image_start = torch.empty(dim)
|
|
self.image_end = torch.empty(dim)
|
|
self.image_newline = torch.empty(dim)
|
|
# Keep all image-only parameters under visual, including FTW/text-only filtering.
|
|
self.routing_bias = torch.empty(num_layers, num_experts, dtype=torch.float32)
|
|
|
|
def place_weights(self, mode: str) -> None:
|
|
self.vision.place_weights(mode)
|
|
|
|
@torch.inference_mode()
|
|
def forward(self, item):
|
|
_, h, w = item.grid_thw
|
|
patches = item.feature.to(device=self.image_start.device, dtype=self.image_start.dtype)
|
|
x = self.aligner.forward(self.vision.forward(patches, h, w), h, w)
|
|
r = self.aligner.ratio
|
|
nh, nw = (h + r - 1) // r, (w + r - 1) // r
|
|
out = x.new_empty(nh * (nw + 1) + 2, x.shape[-1])
|
|
out[0], out[-1] = self.image_start, self.image_end
|
|
rows = out[1:-1].view(nh, nw + 1, -1)
|
|
rows[:, :nw] = x.view(nh, nw, -1)
|
|
rows[:, nw] = self.image_newline
|
|
return out
|