mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* test(deepseek-v41): add independent text and vision reference * feat(quant): support block-32 and unrounded inference outputs * feat(attention): add csa2 packed cache and sparse backend * feat(deepseek-v41): serve text and images through shared runtime * perf(mhc): tune batch-stable reduction tiles * perf(moe): autotune fp4 decode launch geometry * perf(moe): admit recurring misses against actual lru victims * revert: perf(moe): admit recurring misses against actual lru victims This reverts commit 1652d200f0ebb825379cf44c28baf52971e3a461. * perf(mhc): derive the reduction tile from the width, not a timing * perf(moe): replace the ds_fp4 decode autotune with a fixed two-warp launch * fix(deepseek-v41): fence engram staging against in-flight lookups * fix(moe): return the cpu routed sum in fp32 and round it in python * fix(deepseek-v41): keep the vision tower and aligner unquantized * fix(scheduler): admit bounded-replay hits by the history their resume reads * refactor(deepseek-v41): name the attention family after the model, not csa2 * refactor(kvcache): import the dsv41 pool only for dsv41 models * refactor(deepseek-v41): name the replay switch --swa-decoder-replay * fix(server): keep reasoning_effort a string on the wire * refactor(models): keep the prefill replay hooks off BaseLLMModel * refactor(deepseek-v41): drop the fp32 routed sum and batch-invariant prefill * fix(moe): weight ds_fp4 routes in the down epilogue again * docs(models): anchor the v4.1 replay paragraph on the tech report * refactor(deepseek-v41): run the decoder source's narrowed prefill on Block * refactor(kvcache): let the dsv41 pool own the history a resume keeps live * refactor(deepseek-v41): resume bounded-replay hits by capping the prefix match * refactor(deepseek-v41): describe the attention group as a dsv4 variant and derive its geometry from the args * refactor(deepseek-v41): keep the engram table locator in the model package * refactor(deepseek-v41): drop the fp32 lm head output - return the head's bf16 logits, as DeepSeek-V4 does - revert ParallelLMHead(out_dtype=) and LinearKernel.apply_out_dtype, which only this call used - remove test_head_preserves_fp32_logit_margin - cast test_vision's prefill logits to fp32 before comparing, as the harness does * refactor(quant): carry the w4a8 activation block on MoEConfig - drop ActDesc and QuantScheme.act; the scheme only describes stored tensors - Fp8BlockConfig sets MoEConfig.act_block from weight_block_size for its fp4 experts via a layer_config hook - the ds_fp4 GPU kernel and CPU executor read the block from the layer config and require it * refactor(deepseek-v41): size the kv pool from the args, not a geometry copy - delete kvcache/dsv41_geometry.py; the pool and cost model read dsv41_args by attribute, as the DSV4 pool reads dsv4_args - source / ratio aliasing reads args.roles; the duplicated source and ratio validation is gone - the cost model owns the tier row formats, row bytes and the private window layers - the pool keeps the format and window/ratio checks, and exposes the formats and is_private_window to the attention backend * refactor(kvcache): move the dsv4 pool into kvcache/dsv4 * refactor(kvcache): move the dsv41 pool into kvcache/dsv4 * chore(benchmarks): drop the dsv41 selection and fp8 linear benchmarks * test(deepseek-v41): drop tests that repeat other coverage * test(deepseek-v41): load the tiny model's resident experts as banks * fix(server): parse deepseek-v4.1's space-led dsml tool calls --------- Co-authored-by: andy-yang-1 <andy_yang@berkeley.edu>
290 lines
15 KiB
Python
290 lines
15 KiB
Python
"""DeepSeek-V4.1-Flash model (engine-native port of ``inference/model.py``).
|
|
|
|
embed -> hc_mult residual streams -> [Engram at 1, 14] -> 40 blocks (single-pass mHC around
|
|
DSV41 attention and MoE) -> collapse with the last block's pre gates -> norm -> head
|
|
|
|
Single-pass mHC (``layers/mhc.py``): each sublayer boundary applies the PREVIOUS sublayer's post /
|
|
comb to the streams, predicts this sublayer's (pre, post, comb) from the updated streams, and mixes
|
|
the sublayer input with the pre gates the previous sublayer predicted. The chain state
|
|
``(y, residual, post, comb, pre)`` threads through the layer loop; layer 0's attention starts it
|
|
with a one-hot pre on stream 0 and no pending post.
|
|
|
|
Causal Encoder-Decoder: the decoder's kv source (layer 20) compresses ITS INPUT -- the final encoder
|
|
hidden state -- into the global KV every decoder layer reads. Under Decoder SWA Bounded Replay the
|
|
decoder layers run on each request's last ``window_size`` prompt tokens only (their sliding window
|
|
truncated at the start of that window, see the backend; a prefix hit stops before it, so those tokens
|
|
are always prefilled); layer 20 still publishes global KV for every prompt token.
|
|
``--swa-decoder-replay exact`` runs the decoder on every token (the reference numerics).
|
|
|
|
KV addressing is the attention backend's; pool buffers are read off the live pool per access, so a
|
|
runtime cache rebuild needs no unbind. Decode is batched and CUDA-graph safe (the DSV4 precedent).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, List, NamedTuple
|
|
|
|
import torch
|
|
from freetoken.attention.dsv41_sparse import DSV41AttnMetadata, PrefillSegment
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding
|
|
from freetoken.layers.mhc import mhc_fused_post_pre_single_pass, mhc_mix_input, mhc_post
|
|
from freetoken.models.blocks import BaseLLMModel, embed_input_ids
|
|
|
|
from .args import DeepseekV41Args
|
|
from .attention import DSV41Attention, DecodeStepContext
|
|
from .engram import EngramLayer
|
|
from .moe import MoE
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.models.config import ModelConfig
|
|
|
|
POST_MULT = 2.0 # post gates are 2 * sigmoid (reference hc_split_sinkhorn)
|
|
|
|
|
|
class Streams(NamedTuple):
|
|
"""The single-pass mHC chain state between sublayers: the last sublayer output ``y`` with its
|
|
not-yet-applied ``post`` / ``comb``, the residual streams ``[T, hc, dim]`` they apply to, and
|
|
the ``pre`` gates ``[T, hc]`` the next sublayer mixes its input with."""
|
|
|
|
y: torch.Tensor | None
|
|
residual: torch.Tensor
|
|
post: torch.Tensor | None
|
|
comb: torch.Tensor | None
|
|
pre: torch.Tensor
|
|
|
|
def materialize(self) -> torch.Tensor:
|
|
"""The streams with the pending post applied."""
|
|
if self.post is None:
|
|
return self.residual
|
|
return mhc_post(self.y, self.residual, self.post, self.comb)
|
|
|
|
|
|
class Block(BaseOP):
|
|
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
|
|
args: DeepseekV41Args = config.dsv41_args
|
|
self.layer_id = layer_id
|
|
self.role = args.roles[layer_id]
|
|
self.norm_eps = args.norm_eps
|
|
self.hc_eps = args.hc_eps
|
|
self.sinkhorn = args.hc_sinkhorn_iters
|
|
hc, dim = args.hc_mult, args.dim
|
|
mix = (2 + hc) * hc
|
|
self.attn = DSV41Attention(args, self.role, quant_config=config.quant, prefix=f"{prefix}.attn")
|
|
self.ffn = MoE(config, layer_id, prefix=f"{prefix}.ffn")
|
|
self.attn_norm = RMSNorm(dim, args.norm_eps)
|
|
self.ffn_norm = RMSNorm(dim, args.norm_eps)
|
|
self.engram = EngramLayer(args, layer_id, quant_config=config.quant, prefix=f"{prefix}.engram") if layer_id in args.engram_layer_ids else None
|
|
self.hc_attn_fn = torch.empty(mix, hc * dim, dtype=torch.float32)
|
|
self.hc_ffn_fn = torch.empty(mix, hc * dim, dtype=torch.float32)
|
|
self.hc_attn_base = torch.empty(mix, dtype=torch.float32)
|
|
self.hc_ffn_base = torch.empty(mix, dtype=torch.float32)
|
|
self.hc_attn_scale = torch.empty(3, dtype=torch.float32)
|
|
self.hc_ffn_scale = torch.empty(3, dtype=torch.float32)
|
|
|
|
def _boundary(self, s: Streams, fn, scale, base):
|
|
residual, post, comb, pre_next, x = mhc_fused_post_pre_single_pass(
|
|
s.y if s.y is not None else s.residual[:, 0], s.residual, s.post, s.comb, s.pre, fn, scale, base,
|
|
self.norm_eps, self.hc_eps, POST_MULT, self.sinkhorn,
|
|
)
|
|
return residual, post, comb, pre_next, x
|
|
|
|
def attn_input(self, s: Streams, *, image_mask: torch.Tensor | None = None):
|
|
"""Apply the pending post, predict the attention gates, mix and norm the attention input.
|
|
Returns ``(residual, post, comb, pre_next, x)``."""
|
|
if self.engram is not None:
|
|
s = Streams(None, self.engram.forward(s.materialize(), image_mask=image_mask), None, None, s.pre)
|
|
residual, post, comb, pre_next, x = self._boundary(s, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
|
|
return residual, post, comb, pre_next, self.attn_norm.forward(x)
|
|
|
|
def ffn_step(self, y: torch.Tensor, residual, post, comb, pre, *, image_mask: torch.Tensor | None = None) -> Streams:
|
|
"""The FFN sublayer after the attention output ``y``: boundary, norm, MoE; returns the chain state."""
|
|
residual, post, comb, pre_next, x = self._boundary(Streams(y, residual, post, comb, pre), self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base)
|
|
return Streams(self.ffn.forward(self.ffn_norm.forward(x), image_mask=image_mask), residual, post, comb, pre_next)
|
|
|
|
def forward_prefill(self, s: Streams, segments: List[PrefillSegment], positions: torch.Tensor, image_mask: torch.Tensor | None = None) -> Streams:
|
|
residual, post, comb, pre_next, x = self.attn_input(s, image_mask=image_mask)
|
|
if self.attn.compressor is not None:
|
|
self.attn.publish_prefill(x, segments)
|
|
y = self.attn.forward_prefill(x, segments, positions)
|
|
return self.ffn_step(y, residual, post, comb, pre_next, image_mask=image_mask)
|
|
|
|
def forward_prefill_narrowed(
|
|
self, s: Streams, segments: List[PrefillSegment], rows: torch.Tensor, query_segments: List[PrefillSegment],
|
|
query_positions: torch.Tensor, image_mask: torch.Tensor | None = None, query_image_mask: torch.Tensor | None = None,
|
|
) -> Streams | None:
|
|
"""``forward_prefill`` with a row cut after the global KV is published: the attention input and
|
|
``publish_prefill`` cover every token of ``segments``, the attention and the FFN only ``rows``
|
|
(tiled by ``query_segments``) -- the CED decoder's kv source under bounded replay. None when no
|
|
query segment is left (every chunk in the batch ends before its prompt's last window)."""
|
|
residual, post, comb, pre_next, x = self.attn_input(s, image_mask=image_mask)
|
|
if self.attn.compressor is not None:
|
|
self.attn.publish_prefill(x, segments)
|
|
if not query_segments:
|
|
return None
|
|
y = self.attn.forward_prefill(x[rows], query_segments, query_positions)
|
|
return self.ffn_step(y, residual[rows], post[rows], comb[rows], pre_next[rows], image_mask=query_image_mask)
|
|
|
|
def forward_decode(self, s: Streams, pos: torch.Tensor, rows: torch.Tensor, dctx, cmp_stage_cap: int) -> Streams:
|
|
residual, post, comb, pre_next, x = self.attn_input(s)
|
|
y = self.attn.forward_decode(x, pos, rows, dctx, cmp_stage_cap)
|
|
return self.ffn_step(y, residual, post, comb, pre_next)
|
|
|
|
|
|
class Transformer(BaseOP):
|
|
def __init__(self, config: ModelConfig, *, prefix: str = "model"):
|
|
args: DeepseekV41Args = config.dsv41_args
|
|
self.args = args
|
|
self.embed = VocabParallelEmbedding(num_embeddings=args.vocab_size, embedding_dim=args.dim)
|
|
self.layers = OPList([Block(config, i, prefix=f"{prefix}.layers.{i}") for i in range(args.n_layers)])
|
|
self.norm = RMSNorm(args.dim, args.norm_eps)
|
|
|
|
def bind(self, device: torch.device) -> None:
|
|
tables: dict = {} # one RoPE table per distinct parameter set, shared by the layers using it
|
|
for block in self.layers.op_list:
|
|
block.attn.bind(device, tables)
|
|
|
|
def _entry(self, input_ids: torch.Tensor) -> Streams:
|
|
h = embed_input_ids(self.embed, input_ids, get_global_ctx().batch)
|
|
residual = h.unsqueeze(1).expand(-1, self.args.hc_mult, -1).contiguous()
|
|
pre = torch.zeros(h.shape[0], self.args.hc_mult, dtype=torch.float32, device=h.device)
|
|
pre[:, 0] = 1.0
|
|
return Streams(None, residual, None, None, pre)
|
|
|
|
def _exit(self, s: Streams) -> torch.Tensor:
|
|
return self.norm.forward(mhc_mix_input(s.materialize(), s.pre))
|
|
|
|
def prefill(self, input_ids: torch.Tensor, positions: torch.Tensor, md: DSV41AttnMetadata) -> torch.Tensor:
|
|
"""Ragged prefill over the flat new-token stream; returns the hidden states the head reads (the
|
|
decoder stream under bounded replay, where ``md.last_indices`` also point, plus the placeholder
|
|
row ``md.decoder_pad`` asks for)."""
|
|
s = self._entry(input_ids)
|
|
dtype = s.residual.dtype
|
|
image_mask = input_ids >= self.args.vocab_size if get_global_ctx().batch.mm_embeds is not None else None
|
|
segments, dec_segments, rows = md.segments, md.decoder_segments, md.decoder_rows
|
|
decoder_start = self.args.decoder_start_layer
|
|
if rows is not None:
|
|
dec_positions = positions[rows]
|
|
dec_image_mask = None if image_mask is None else image_mask[rows]
|
|
for block in self.layers.op_list:
|
|
if rows is None or block.layer_id < decoder_start:
|
|
s = block.forward_prefill(s, segments, positions, image_mask)
|
|
elif block.layer_id == decoder_start:
|
|
s = block.forward_prefill_narrowed(s, segments, rows, dec_segments, dec_positions, image_mask, dec_image_mask)
|
|
if s is None:
|
|
break
|
|
else:
|
|
s = block.forward_prefill(s, dec_segments, dec_positions, dec_image_mask)
|
|
pad = [torch.zeros(1, self.args.dim, dtype=dtype, device=input_ids.device)] if md.decoder_pad else []
|
|
return torch.cat([self._exit(s)] + pad) if s is not None else pad[0]
|
|
|
|
def decode(self, input_ids: torch.Tensor, pos: torch.Tensor, md: DSV41AttnMetadata, cmp_stage_cap: int) -> torch.Tensor:
|
|
B = input_ids.shape[0]
|
|
rows = torch.arange(B, device=input_ids.device)
|
|
s = self._entry(input_ids)
|
|
# the layer-invariant step context, resolved once (a capture records it once): ring slots and
|
|
# window candidates of the shared pool -- and of the private rings when bounded replay keeps
|
|
# the decoder's window KV per request -- plus each RoPE table at the step's positions
|
|
dctx = DecodeStepContext.build([b.attn for b in self.layers.op_list], md, pos, rows, self.args.window_size, self.args.index_topk)
|
|
for block in self.layers.op_list:
|
|
s = block.forward_decode(s, pos, rows, dctx, cmp_stage_cap)
|
|
return self._exit(s)
|
|
|
|
|
|
class DeepseekV41ForCausalLM(BaseLLMModel):
|
|
def __init__(self, config: ModelConfig):
|
|
self._config = config
|
|
self._args: DeepseekV41Args = config.dsv41_args
|
|
self.model = Transformer(config)
|
|
self.head = ParallelLMHead(num_embeddings=self._args.vocab_size, embedding_dim=self._args.dim, quant_config=config.quant, prefix="head")
|
|
self._bound = False
|
|
if config.is_multimodal:
|
|
from .vision import DeepseekV41Vision
|
|
|
|
self.visual = DeepseekV41Vision(config.vision_config, config.hidden_size,
|
|
num_layers=self._args.n_layers, num_experts=self._args.n_routed_experts)
|
|
|
|
def encode(self, item):
|
|
return self.visual.forward(item)
|
|
|
|
def place_encoder_weights(self, mode: str) -> None:
|
|
self.visual.place_weights(mode)
|
|
|
|
def _ensure_bound(self) -> None:
|
|
if not self._bound:
|
|
self.model.bind(get_global_ctx().kv_cache.device)
|
|
if self._config.is_multimodal:
|
|
for i, block in enumerate(self.model.layers.op_list):
|
|
block.ffn.gate._bias_vl = self.visual.routing_bias[i]
|
|
self._bound = True
|
|
|
|
def mark_for_rebind(self) -> None:
|
|
"""A runtime pool rebuild changes nothing the model caches (rope tables are pool-independent),
|
|
but the engine asks; re-bind is cheap and keeps the contract with the DSV4 sibling."""
|
|
self._bound = False
|
|
|
|
def engram_layers(self) -> List[EngramLayer]:
|
|
return [b.engram for b in self.model.layers.op_list if b.engram is not None]
|
|
|
|
def load_host_tables(self, engine_config) -> int:
|
|
"""Attach the Engram tables (streamed from the checkpoint shards; zeros for dummy weights)
|
|
and take over ``forward_host_ctx`` so every dispatch stages its rows. Returns the pinned
|
|
host bytes the engine reserves from its pin budget."""
|
|
import os
|
|
|
|
layers = self.engram_layers()
|
|
if not layers:
|
|
return 0
|
|
device = torch.device("cuda", torch.cuda.current_device())
|
|
if getattr(engine_config, "use_dummy_weight", False):
|
|
from .engram import ZeroEngramTable
|
|
|
|
for layer in layers:
|
|
layer.attach_table(ZeroEngramTable(layer.width, device))
|
|
return 0
|
|
from freetoken.utils import download_hf_weight
|
|
from transformers import AutoTokenizer
|
|
|
|
from .engram import EngramHash, build_compressed_token_map
|
|
from .engram_table import EngramDiskTable, EngramHost, engram_row_source
|
|
|
|
folder = download_hf_weight(engine_config.model_path)
|
|
token_map, compressed_vocab = build_compressed_token_map(AutoTokenizer.from_pretrained(folder, trust_remote_code=True))
|
|
hash = EngramHash(self._args, token_map, compressed_vocab)
|
|
graph_rows = max(engine_config.max_running_req, engine_config.cuda_graph_max_bs or 0, 1)
|
|
tables = [
|
|
EngramDiskTable(
|
|
engram_row_source(folder, layer.layer_id), hash.n_cols, device,
|
|
max_graph_rows=graph_rows,
|
|
max_extend_tokens=engine_config.max_extend_tokens,
|
|
use_io_uring=os.getenv("FREETOKEN_ENGRAM_IO_URING", "1") != "0",
|
|
)
|
|
for layer in layers
|
|
]
|
|
for layer, table in zip(layers, tables):
|
|
layer.attach_table(table)
|
|
self._engram_host = EngramHost(hash, tables, device, sync_mode=os.getenv("FREETOKEN_ENGRAM_SYNC", "auto"))
|
|
self.forward_host_ctx = self._engram_host.forward_host_ctx
|
|
return sum(t._graph_pinned.numel() + sum(b.numel() for b in t._eager_pinned) for t in tables)
|
|
|
|
def forward(self) -> torch.Tensor:
|
|
self._ensure_bound()
|
|
batch = get_global_ctx().batch
|
|
md = batch.attn_metadata
|
|
assert isinstance(md, DSV41AttnMetadata)
|
|
if batch.is_prefill:
|
|
hidden = self.model.prefill(batch.input_ids, batch.positions.long(), md)
|
|
else:
|
|
input_ids = batch.input_ids
|
|
B = batch.padded_size
|
|
pos = batch.positions.long().view(-1)[:B]
|
|
if torch.cuda.is_current_stream_capturing():
|
|
cmp_stage_cap = md.stage_width - 1
|
|
else:
|
|
cmp_stage_cap = int(pos.max().item())
|
|
hidden = self.model.decode(input_ids.view(B), pos, md, cmp_stage_cap)
|
|
return self.head.forward(hidden)
|
|
|
|
|
|
__all__ = ["DeepseekV41ForCausalLM", "Transformer", "Block", "Streams"]
|