mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
111 lines
4.2 KiB
Python
111 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from contextlib import contextmanager
|
|
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|
|
|
import torch
|
|
|
|
from freetoken.layers import (
|
|
BaseOP,
|
|
LinearColParallelMerged,
|
|
LinearRowParallel,
|
|
gelu_and_mul,
|
|
gelu_tanh_and_mul,
|
|
silu_and_mul,
|
|
)
|
|
from freetoken.utils import nvtx_annotate
|
|
|
|
if TYPE_CHECKING:
|
|
import torch
|
|
|
|
from freetoken.core import Batch
|
|
from freetoken.layers.quantization import QuantConfig
|
|
|
|
from .config import ModelConfig
|
|
|
|
|
|
class BaseLLMModel(ABC, BaseOP):
|
|
@abstractmethod
|
|
def forward(self) -> torch.Tensor: ...
|
|
|
|
@contextmanager
|
|
def forward_host_ctx(self, batch: Batch, use_graph: bool):
|
|
"""Around one forward dispatch: enter before it is enqueued, exit right after. A backend that feeds the forward from host memory overrides this."""
|
|
yield
|
|
|
|
|
|
@runtime_checkable
|
|
class SupportsMultimodal(Protocol):
|
|
"""What the engine asks of a model that serves multimodal items; the text forward takes its token embeddings from embed_input_ids."""
|
|
|
|
def encode(self, item: MMItem) -> torch.Tensor:
|
|
"""``[item.num_tokens, D]`` soft tokens for one item of any modality the family serves, on the model device."""
|
|
...
|
|
|
|
def place_encoder_weights(self, mode: str) -> None:
|
|
"""Every encoder tower's weights: ``gpu`` resident or ``host`` streamed from pinned banks."""
|
|
...
|
|
|
|
|
|
class FeatureSink(Protocol):
|
|
"""Where a forward hands per-row hidden states to a draft model; set on ``batch.features``.
|
|
|
|
A CUDA graph replays the sink it was captured with, never the one set at replay: write only to buffers the draft owns, keep no per-step state on the sink.
|
|
"""
|
|
|
|
wanted_layers: frozenset[int]
|
|
|
|
def final_hidden(self, x: torch.Tensor) -> None:
|
|
"""Every row's final hidden state, once per forward: the LM head's input, or what the model hands over itself before the head."""
|
|
...
|
|
|
|
def layer_output(self, layer_id: int, x: torch.Tensor, residual: torch.Tensor | None = None, num_streams: int = 1) -> None:
|
|
"""A layer's output rows, ``x + residual`` (``x`` alone when ``residual`` is None), ``[T, num_streams*H]`` stream-major; called only for layers in ``wanted_layers``. The sum is left to the sink, so it adds only the rows it keeps. The forward may overwrite ``x`` and ``residual`` afterwards, so copy what is kept."""
|
|
...
|
|
|
|
|
|
def embed_input_ids(embed_tokens, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:
|
|
"""Token embeddings of the batch; on a chunk with multimodal rows, batch.mm_embeds' leading columns replace the rows batch.mm_rows."""
|
|
if batch.mm_embeds is None:
|
|
return embed_tokens.forward(input_ids)
|
|
# multimodal rows carry content pad ids above the vocab: clamp for the lookup, the copy overwrites those rows
|
|
x = embed_tokens.forward(input_ids.clamp(max=embed_tokens.num_embeddings - 1))
|
|
x.index_copy_(0, batch.mm_rows, batch.mm_embeds[:, : x.shape[1]].to(x.dtype))
|
|
return x
|
|
|
|
|
|
class GatedMLP(BaseOP):
|
|
def __init__(self, config: ModelConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
|
|
self.gate_up_proj = LinearColParallelMerged(
|
|
config.hidden_size,
|
|
[config.intermediate_size, config.intermediate_size],
|
|
has_bias=False,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.gate_up_proj",
|
|
)
|
|
|
|
fn_map = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
|
|
act_fn = fn_map.get(config.hidden_act, None)
|
|
if act_fn is None:
|
|
raise ValueError(f"Unsupported activation function: {config.hidden_act}")
|
|
self.act_fn = act_fn
|
|
self.down_proj = LinearRowParallel(
|
|
config.intermediate_size,
|
|
config.hidden_size,
|
|
has_bias=False,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.down_proj",
|
|
)
|
|
|
|
@nvtx_annotate("MLP")
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
gate_up = self.gate_up_proj.forward(x)
|
|
del x
|
|
y = self.act_fn(gate_up)
|
|
del gate_up
|
|
return self.down_proj.forward(y)
|
|
|
|
|
|
__all__ = ["BaseLLMModel", "GatedMLP"]
|