mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
105 lines
4.4 KiB
Python
105 lines
4.4 KiB
Python
"""The draft side of speculative decoding: what every proposer declares and implements, and the engine state it may touch."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import TYPE_CHECKING, ClassVar, List, NamedTuple
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.core import Batch, Req
|
|
from freetoken.models.blocks import FeatureSink
|
|
from freetoken.models.config import SlotStateSpec
|
|
|
|
|
|
def check_draft_path(path: str) -> None:
|
|
"""Fail at config time on a local ``--speculative-draft-model-path`` that does not exist; a HF repo id is "name" or "org/name"."""
|
|
local = os.path.isabs(path) or path.startswith(".") or path.count("/") > 1 or path.endswith(".gguf")
|
|
if local and not os.path.exists(path):
|
|
raise ValueError(f"--speculative-draft-model-path {path} does not exist")
|
|
|
|
|
|
class SpecProposal(NamedTuple):
|
|
"""One step's drafts and, when they were sampled, the distribution each was drawn from, kept on its support; None when the drafts are argmaxes."""
|
|
|
|
draft_token_ids: torch.Tensor # [bs, k] int64 target-vocab ids
|
|
draft_support_ids: torch.Tensor | None = None # [bs, k, K] the token ids where q can be nonzero
|
|
draft_support_probs: torch.Tensor | None = None # [bs, k, K] fp32 q over them, summing to 1
|
|
|
|
|
|
class DraftContext(NamedTuple):
|
|
"""Built by SpecRunner.attach; the only engine state the draft side gets. A rebuild replaces it."""
|
|
|
|
page_table: torch.Tensor
|
|
num_pages: int
|
|
device: torch.device
|
|
stream: torch.cuda.Stream
|
|
# the engine's padding request: its table row points at the dummy page
|
|
dummy_req: Req
|
|
# the tensors of the proposer's slot_states, ``[num_slots, *shape]`` each, on the target's GDN slots
|
|
slot_states: dict[str, torch.Tensor]
|
|
|
|
|
|
class Proposer:
|
|
"""Draft side of one speculative method; holds no engine reference."""
|
|
|
|
# True: the draft's KV slot i depends on token i+1, so the prefix cache keys position i by (t_i, t_{i+1})
|
|
radix_lookahead: ClassVar[bool] = False
|
|
|
|
@classmethod
|
|
def adjust_config(cls, config, override) -> None:
|
|
"""Raise if this proposer cannot serve the config; the base rejects TP > 1."""
|
|
if config.tp_info.size > 1:
|
|
raise ValueError(
|
|
"speculative decoding needs --tp 1 for now: without a cross-rank agreement on the drafts, "
|
|
"near-tied draft argmaxes can differ between ranks and the ranks diverge"
|
|
)
|
|
|
|
@classmethod
|
|
def num_steps(cls, config) -> tuple[int, int]:
|
|
"""(default, max) draft tokens per step; called after ``adjust_config``."""
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def budget(cls, config) -> tuple[int, int]:
|
|
"""(fixed bytes, draft KV bytes per target KV token), from the config alone."""
|
|
return 0, 0
|
|
|
|
@classmethod
|
|
def slot_states(cls, config) -> tuple[SlotStateSpec, ...]:
|
|
"""Per-request draft state that rides the target's GDN slots and is copied and restored with them; from the config alone. The proposer writes its own prefill track snapshot."""
|
|
return ()
|
|
|
|
def __init__(self, config, target_model) -> None:
|
|
self.config = config
|
|
self.k = config.speculative_num_steps
|
|
|
|
def attach(self, ctx: DraftContext) -> None:
|
|
"""Build the draft KV pool, backends and buffers; called again with a new context after a rebuild."""
|
|
self.ctx = ctx
|
|
|
|
def detach(self, *, keep_kv: bool) -> None:
|
|
"""Free draft graphs and drop the context before a rebuild swaps the pools; draft KV on the target's slots goes too unless keep_kv says the target KV pool stays."""
|
|
self.ctx = None
|
|
|
|
def propose(self, reqs: List[Req], rows: torch.Tensor, pending: torch.Tensor) -> SpecProposal:
|
|
"""k drafts per request after its pending token; ``rows`` holds the requests' table_idx and ``pending`` their pending tokens, on the device."""
|
|
raise NotImplementedError
|
|
|
|
def feature_sink(self, batch: Batch) -> FeatureSink | None:
|
|
"""What the target forward of a prefill or verify batch hands the draft."""
|
|
return None
|
|
|
|
def post_verify(self, batch: Batch, out: torch.Tensor, emitted: torch.Tensor) -> None:
|
|
"""Advance the draft on the verify's features; ``batch`` is the padded verify batch."""
|
|
|
|
def on_prefill(self, batch: Batch, next_tokens_gpu: torch.Tensor) -> None:
|
|
pass
|
|
|
|
def capture_graphs(self, bs_list: List[int], pool) -> None:
|
|
pass
|
|
|
|
def release(self, req: Req) -> None:
|
|
pass
|