Files
FreeToken/python/freetoken/spec/proposer.py
Xiaoze Fan 08cee44468 feat(spec): add DFlash 2 speculative decoding (#857)
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal

* feat(engine): Sampler.probs and sparse-q rejection

* feat(spec): let a proposer declare slot states riding the GDN slots

* feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families

* feat(attention): record the source slot of each tracked request

* feat(kernel): DFlash2 draft kernels

* feat(models): DFlash2 draft family

* feat(spec): DFlashWindow

* feat(spec): DFlashProposer

* docs(cli): add dflash to the speculative decoding section

* refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs

* fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply

* refactor(spec): share the draft path check between the MTP and DFlash proposers

* test(e2e): compare rebuilt outputs exactly only without hybrid MoE

* refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config

* perf(spec): compute the verify distribution once for sparse-q rejection

* perf(models): let the feature sink add the residual on the rows it keeps

* perf(models): share the DFlash draft K/V weights with the ring projection

* fix(kernel): keep the dflash selector off zero-probability candidates

* test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover

* fix(spec): default and cap DFlash steps at the draft's block_size - 1

* fix(models): reject DFlash draft configs the draft forward cannot serve

* test(e2e): read the MoE strategy from the rebuild test's extra args

* docs(spec): say that slot-state owners write their own prefill track snapshot

* test(models): move the DFlash draft parity tests under tests/models/dflash

* refactor(spec): drop the no-op verify handler of the DFlash window

* refactor(models): fold the DFlash candidate top-k into propose

* refactor(spec): declare the draft step limits through Proposer.num_steps only

* refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring

* refactor(models): name the DFlash draft submodules like the other model families

* refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py

* refactor(spec): hand the verify the draft distribution over the whole vocabulary

* refactor(spec): call the DFlashRing object dflash_ring and its length ring_len

* refactor(models): split the DFlash draft's propose into compute_candidates and sample_path

* refactor(spec): keep the draft distribution on its support in SpecProposal

* fix(spec): drop the step's proposal once the verify has read it

* refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink

* refactor(models): read the declared Batch.features directly instead of through getattr

* test(dflash): check the window rebuild inside the block forward test

* test(spec): share the plain run of the verify contract and run the DFlash draft once per family

* test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes

* test(spec): check the verify probs of each row against a torch top-k and top-p reference

* test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate

* refactor(scheduler): let callers take the speculative counts from the status reporter

* test(spec): keep the shared plain runs of the verify contract in their own temp dirs

* test(models): give the ricursive test's stand-in batch a features field
2026-11-07 12:29:18 -08:00

105 lines
4.4 KiB
Python

"""The draft side of speculative decoding: what every proposer declares and implements, and the engine state it may touch."""
from __future__ import annotations
import os
from typing import TYPE_CHECKING, ClassVar, List, NamedTuple
import torch
if TYPE_CHECKING:
from freetoken.core import Batch, Req
from freetoken.models.blocks import FeatureSink
from freetoken.models.config import SlotStateSpec
def check_draft_path(path: str) -> None:
"""Fail at config time on a local ``--speculative-draft-model-path`` that does not exist; a HF repo id is "name" or "org/name"."""
local = os.path.isabs(path) or path.startswith(".") or path.count("/") > 1 or path.endswith(".gguf")
if local and not os.path.exists(path):
raise ValueError(f"--speculative-draft-model-path {path} does not exist")
class SpecProposal(NamedTuple):
"""One step's drafts and, when they were sampled, the distribution each was drawn from, kept on its support; None when the drafts are argmaxes."""
draft_token_ids: torch.Tensor # [bs, k] int64 target-vocab ids
draft_support_ids: torch.Tensor | None = None # [bs, k, K] the token ids where q can be nonzero
draft_support_probs: torch.Tensor | None = None # [bs, k, K] fp32 q over them, summing to 1
class DraftContext(NamedTuple):
"""Built by SpecRunner.attach; the only engine state the draft side gets. A rebuild replaces it."""
page_table: torch.Tensor
num_pages: int
device: torch.device
stream: torch.cuda.Stream
# the engine's padding request: its table row points at the dummy page
dummy_req: Req
# the tensors of the proposer's slot_states, ``[num_slots, *shape]`` each, on the target's GDN slots
slot_states: dict[str, torch.Tensor]
class Proposer:
"""Draft side of one speculative method; holds no engine reference."""
# True: the draft's KV slot i depends on token i+1, so the prefix cache keys position i by (t_i, t_{i+1})
radix_lookahead: ClassVar[bool] = False
@classmethod
def adjust_config(cls, config, override) -> None:
"""Raise if this proposer cannot serve the config; the base rejects TP > 1."""
if config.tp_info.size > 1:
raise ValueError(
"speculative decoding needs --tp 1 for now: without a cross-rank agreement on the drafts, "
"near-tied draft argmaxes can differ between ranks and the ranks diverge"
)
@classmethod
def num_steps(cls, config) -> tuple[int, int]:
"""(default, max) draft tokens per step; called after ``adjust_config``."""
raise NotImplementedError
@classmethod
def budget(cls, config) -> tuple[int, int]:
"""(fixed bytes, draft KV bytes per target KV token), from the config alone."""
return 0, 0
@classmethod
def slot_states(cls, config) -> tuple[SlotStateSpec, ...]:
"""Per-request draft state that rides the target's GDN slots and is copied and restored with them; from the config alone. The proposer writes its own prefill track snapshot."""
return ()
def __init__(self, config, target_model) -> None:
self.config = config
self.k = config.speculative_num_steps
def attach(self, ctx: DraftContext) -> None:
"""Build the draft KV pool, backends and buffers; called again with a new context after a rebuild."""
self.ctx = ctx
def detach(self, *, keep_kv: bool) -> None:
"""Free draft graphs and drop the context before a rebuild swaps the pools; draft KV on the target's slots goes too unless keep_kv says the target KV pool stays."""
self.ctx = None
def propose(self, reqs: List[Req], rows: torch.Tensor, pending: torch.Tensor) -> SpecProposal:
"""k drafts per request after its pending token; ``rows`` holds the requests' table_idx and ``pending`` their pending tokens, on the device."""
raise NotImplementedError
def feature_sink(self, batch: Batch) -> FeatureSink | None:
"""What the target forward of a prefill or verify batch hands the draft."""
return None
def post_verify(self, batch: Batch, out: torch.Tensor, emitted: torch.Tensor) -> None:
"""Advance the draft on the verify's features; ``batch`` is the padded verify batch."""
def on_prefill(self, batch: Batch, next_tokens_gpu: torch.Tensor) -> None:
pass
def capture_graphs(self, bs_list: List[int], pool) -> None:
pass
def release(self, req: Req) -> None:
pass