Files
FreeToken/python/freetoken/layers/embedding.py
Xiaoze Fan 08cee44468 feat(spec): add DFlash 2 speculative decoding (#857)
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal

* feat(engine): Sampler.probs and sparse-q rejection

* feat(spec): let a proposer declare slot states riding the GDN slots

* feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families

* feat(attention): record the source slot of each tracked request

* feat(kernel): DFlash2 draft kernels

* feat(models): DFlash2 draft family

* feat(spec): DFlashWindow

* feat(spec): DFlashProposer

* docs(cli): add dflash to the speculative decoding section

* refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs

* fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply

* refactor(spec): share the draft path check between the MTP and DFlash proposers

* test(e2e): compare rebuilt outputs exactly only without hybrid MoE

* refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config

* perf(spec): compute the verify distribution once for sparse-q rejection

* perf(models): let the feature sink add the residual on the rows it keeps

* perf(models): share the DFlash draft K/V weights with the ring projection

* fix(kernel): keep the dflash selector off zero-probability candidates

* test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover

* fix(spec): default and cap DFlash steps at the draft's block_size - 1

* fix(models): reject DFlash draft configs the draft forward cannot serve

* test(e2e): read the MoE strategy from the rebuild test's extra args

* docs(spec): say that slot-state owners write their own prefill track snapshot

* test(models): move the DFlash draft parity tests under tests/models/dflash

* refactor(spec): drop the no-op verify handler of the DFlash window

* refactor(models): fold the DFlash candidate top-k into propose

* refactor(spec): declare the draft step limits through Proposer.num_steps only

* refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring

* refactor(models): name the DFlash draft submodules like the other model families

* refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py

* refactor(spec): hand the verify the draft distribution over the whole vocabulary

* refactor(spec): call the DFlashRing object dflash_ring and its length ring_len

* refactor(models): split the DFlash draft's propose into compute_candidates and sample_path

* refactor(spec): keep the draft distribution on its support in SpecProposal

* fix(spec): drop the step's proposal once the verify has read it

* refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink

* refactor(models): read the declared Batch.features directly instead of through getattr

* test(dflash): check the window rebuild inside the block forward test

* test(spec): share the plain run of the verify contract and run the DFlash draft once per family

* test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes

* test(spec): check the verify probs of each row against a torch top-k and top-p reference

* test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate

* refactor(scheduler): let callers take the speculative counts from the status reporter

* test(spec): keep the shared plain runs of the verify contract in their own temp dirs

* test(models): give the ricursive test's stand-in batch a features field
2026-11-07 12:29:18 -08:00

159 lines
6.2 KiB
Python

from __future__ import annotations
from typing import Dict
import torch
from freetoken.core import get_global_ctx
from freetoken.distributed import DistributedCommunicator, get_tp_info
from freetoken.utils import div_ceil, nvtx_annotate
from .base import BaseOP
from .quantization import LayerKind, QuantConfig, quant_method_for
class VocabParallelEmbedding(BaseOP):
"""The vocab table, sharded over TP: its ``quant_method`` owns the table, looks rows up, and serves a tied LM head's product."""
quant_layer_kind = LayerKind.EMBEDDING
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
embed_scale: float | None = None,
*,
quant_config: QuantConfig | None = None,
prefix: str = "",
):
super().__init__()
tp_info = get_tp_info()
tp_rank = tp_info.rank
self.tp_size = tp_info.size
self.num_embeddings = num_embeddings
self.num_embeddings_tp = div_ceil(num_embeddings, self.tp_size)
start_idx = self.num_embeddings_tp * tp_rank
finish_idx = min(start_idx + self.num_embeddings_tp, num_embeddings)
self.vocab_range = (start_idx, finish_idx - start_idx)
self.embedding_dim = embedding_dim
# the LM head subclass is a linear and builds its own method
self.quant_method = None
if self.quant_layer_kind is LayerKind.EMBEDDING:
self.quant_method = quant_method_for(quant_config, self, prefix)
self.quant_method.create_weights(self)
# Gemma scales embeddings by sqrt(hidden_size). The scale is materialized in
# the weight dtype (bf16) to match HF, which downcasts the scalar. The GPU
# scalar is built lazily (model __init__ runs on the meta device) and cached
# so it is not reallocated inside a captured CUDA graph.
self._embed_scale = embed_scale
self._embed_scale_t: torch.Tensor | None = None
self._comm = DistributedCommunicator()
@nvtx_annotate("Embedding")
def forward(self, x: torch.Tensor) -> torch.Tensor:
y = self.quant_method.embed(self, x)
if self.tp_size > 1:
y = self._comm.all_reduce(y)
if self._embed_scale is not None:
if self._embed_scale_t is None:
self._embed_scale_t = torch.tensor(
self._embed_scale, dtype=y.dtype, device=y.device
)
y = y * self._embed_scale_t
return y
class ParallelLMHead(VocabParallelEmbedding):
"""The head is a linear layer over the vocab shard: its weights come from ``quant_method``
unless they are tied to the input embedding."""
quant_layer_kind = LayerKind.LINEAR
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
bias: bool = False,
tie_word_embeddings: bool = False,
tied_embedding: VocabParallelEmbedding | None = None,
*,
quant_config: QuantConfig | None = None,
prefix: str = "",
):
super().__init__(num_embeddings, embedding_dim)
self.has_bias = bias
self.prefix = prefix
self.tied_embedding = tied_embedding
assert (tied_embedding is not None) == tie_word_embeddings
self.in_features = embedding_dim
self.out_features = self.num_embeddings_tp
self.output_sizes = (self.num_embeddings_tp,)
if tied_embedding is None:
self.quant_method = quant_method_for(quant_config, self, prefix)
self.quant_method.create_weights(self)
self.bias = torch.empty(self.num_embeddings_tp) if bias else None
def finalize(self) -> None:
if self.quant_method is not None:
self.quant_method.finalize(self)
def load_state_dict(
self,
state_dict: Dict[str, torch.Tensor],
*,
prefix: str = "",
_internal: bool = False,
) -> None:
if not self.tied_embedding:
return super().load_state_dict(state_dict, prefix=prefix, _internal=_internal)
else:
# pop the lm_head.weights and lm_head.bias if they exist
possible_weight = f"{prefix}.weight"
possible_bias = f"{prefix}.bias"
if possible_weight in state_dict:
state_dict.pop(possible_weight)
if possible_bias in state_dict:
state_dict.pop(possible_bias)
def state_dict(
self,
*,
prefix: str = "",
result: Dict[str, torch.Tensor] | None = None,
) -> Dict[str, torch.Tensor]:
if not self.tied_embedding:
return super().state_dict(prefix=prefix, result=result)
return {} if result is None else result
@nvtx_annotate("LMHead")
def forward(self, x: torch.Tensor, *, export_hidden: bool = True) -> torch.Tensor:
"""``export_hidden=False``: the model hands its own final hidden to ``batch.features``."""
ctx = get_global_ctx()
batch = ctx.batch
bs = batch.size
if export_hidden and (features := batch.features) is not None:
features.final_hidden(x)
if batch.is_prefill:
indices = batch.attn_metadata.get_last_indices(bs)
x = x[indices].contiguous()
del indices
return self.logits(x)
def logits(self, x: torch.Tensor) -> torch.Tensor:
"""Full-vocab logits of the rows of ``x``; reads no batch state."""
if self.tied_embedding is not None:
logits = self.tied_embedding.quant_method.logits(self.tied_embedding, x, self.bias)
else:
logits = self.quant_method.apply(self, x)
if self.tp_size == 1:
return logits
input_shape = logits.shape
output_tensor = self._comm.all_gather(logits)
# keyed on rows, not batch size: a verify batch has several rows per request
if input_shape[0] == 1:
return output_tensor.view(1, -1)[:, : self.num_embeddings]
output_tensor = output_tensor.view((self.tp_size,) + input_shape)
output_tensor = output_tensor.permute(1, 0, 2).contiguous()
output_tensor = output_tensor.reshape(input_shape[:1] + (self.tp_size * input_shape[1],))
return output_tensor[:, : self.num_embeddings]