Files
Xiaoze Fan 08cee44468 feat(spec): add DFlash 2 speculative decoding (#857)
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal

* feat(engine): Sampler.probs and sparse-q rejection

* feat(spec): let a proposer declare slot states riding the GDN slots

* feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families

* feat(attention): record the source slot of each tracked request

* feat(kernel): DFlash2 draft kernels

* feat(models): DFlash2 draft family

* feat(spec): DFlashWindow

* feat(spec): DFlashProposer

* docs(cli): add dflash to the speculative decoding section

* refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs

* fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply

* refactor(spec): share the draft path check between the MTP and DFlash proposers

* test(e2e): compare rebuilt outputs exactly only without hybrid MoE

* refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config

* perf(spec): compute the verify distribution once for sparse-q rejection

* perf(models): let the feature sink add the residual on the rows it keeps

* perf(models): share the DFlash draft K/V weights with the ring projection

* fix(kernel): keep the dflash selector off zero-probability candidates

* test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover

* fix(spec): default and cap DFlash steps at the draft's block_size - 1

* fix(models): reject DFlash draft configs the draft forward cannot serve

* test(e2e): read the MoE strategy from the rebuild test's extra args

* docs(spec): say that slot-state owners write their own prefill track snapshot

* test(models): move the DFlash draft parity tests under tests/models/dflash

* refactor(spec): drop the no-op verify handler of the DFlash window

* refactor(models): fold the DFlash candidate top-k into propose

* refactor(spec): declare the draft step limits through Proposer.num_steps only

* refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring

* refactor(models): name the DFlash draft submodules like the other model families

* refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py

* refactor(spec): hand the verify the draft distribution over the whole vocabulary

* refactor(spec): call the DFlashRing object dflash_ring and its length ring_len

* refactor(models): split the DFlash draft's propose into compute_candidates and sample_path

* refactor(spec): keep the draft distribution on its support in SpecProposal

* fix(spec): drop the step's proposal once the verify has read it

* refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink

* refactor(models): read the declared Batch.features directly instead of through getattr

* test(dflash): check the window rebuild inside the block forward test

* test(spec): share the plain run of the verify contract and run the DFlash draft once per family

* test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes

* test(spec): check the verify probs of each row against a torch top-k and top-p reference

* test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate

* refactor(scheduler): let callers take the speculative counts from the status reporter

* test(spec): keep the shared plain runs of the verify contract in their own temp dirs

* test(models): give the ricursive test's stand-in batch a features field
2026-11-07 12:29:18 -08:00

171 lines
6.4 KiB
Python

from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Callable
from freetoken.core import Batch
@dataclass
class SchedulerStatusReporter:
log: Callable[[str], None]
clock: Callable[[], float] = time.perf_counter
decode_log_interval: int = 40
# --moe-collect-stats: one more line per decode log line (the stats since the previous one), or None to skip
moe_stats: Callable[[], str | None] | None = None
_last_prefill_time: float = field(init=False)
_last_decode_time: float = field(init=False)
_decode_forward_count: int = field(default=0, init=False)
_decode_generated_tokens: int = field(default=0, init=False)
# speculative steps since the last decode log: requests stepped and tokens they committed
_spec_req_steps: int = field(default=0, init=False)
_spec_tokens: int = field(default=0, init=False)
# the same since the last take_spec_counts(), which the decode log leaves alone
_spec_counts: tuple[int, int] = field(default=(0, 0), init=False)
def __post_init__(self) -> None:
now = self.clock()
self._last_prefill_time = now
self._last_decode_time = now
self.decode_log_interval = max(1, self.decode_log_interval)
def report_batch(
self,
batch: Batch,
*,
running_reqs: int,
queue_reqs: int,
kv_used_pages: int,
kv_total_pages: int,
page_size: int,
mamba_slots: tuple[int, int] | None = None,
swa_tokens: tuple[int, int] | None = None,
generated: int | None = None,
) -> None:
"""``generated``: tokens a decode step committed; None means one per request."""
if batch.is_prefill:
self._report_prefill(
batch,
running_reqs=running_reqs,
queue_reqs=queue_reqs,
kv_used_pages=kv_used_pages,
kv_total_pages=kv_total_pages,
mamba_slots=mamba_slots,
swa_tokens=swa_tokens,
)
else:
self._report_decode(
batch,
running_reqs=running_reqs,
queue_reqs=queue_reqs,
kv_used_pages=kv_used_pages,
kv_total_pages=kv_total_pages,
page_size=page_size,
mamba_slots=mamba_slots,
swa_tokens=swa_tokens,
generated=generated,
)
def take_spec_counts(self) -> tuple[int, int]:
"""Requests stepped and tokens they committed over the speculative decode steps since the last call."""
counts, self._spec_counts = self._spec_counts, (0, 0)
return counts
def _report_prefill(
self,
batch: Batch,
*,
running_reqs: int,
queue_reqs: int,
kv_used_pages: int,
kv_total_pages: int,
mamba_slots: tuple[int, int] | None = None,
swa_tokens: tuple[int, int] | None = None,
) -> None:
now = self.clock()
gap = now - self._last_prefill_time
self._last_prefill_time = now
# Read the schedule-time snapshot: by report time the forward's complete_one() has
# advanced each req's cached_len to device_len, so reading the reqs here would log
# decode-state values (#new-token == #reqs, #cached-token == full prompt).
new_tokens = batch.log_new_tokens
cached_tokens = batch.log_cached_tokens
input_throughput = new_tokens / gap if gap > 0 else 0.0
self.log(
f"Prefill batch, "
f"#new-seq: {len(batch.reqs)}, "
f"#new-token: {new_tokens}, "
f"#cached-token: {cached_tokens}, "
f"token usage: {_usage_ratio(kv_used_pages, kv_total_pages):.2f}, "
f"{_swa_msg(swa_tokens)}"
f"{_mamba_msg(mamba_slots)}"
f"#running-req: {running_reqs}, "
f"#queue-req: {queue_reqs}, "
f"input throughput (token/s): {input_throughput:.2f}"
)
def _report_decode(
self,
batch: Batch,
*,
running_reqs: int,
queue_reqs: int,
kv_used_pages: int,
kv_total_pages: int,
page_size: int,
mamba_slots: tuple[int, int] | None = None,
swa_tokens: tuple[int, int] | None = None,
generated: int | None = None,
) -> None:
self._decode_forward_count += 1
self._decode_generated_tokens += len(batch.reqs) if generated is None else generated
if generated is not None:
self._spec_req_steps += len(batch.reqs)
self._spec_tokens += generated
steps, tokens = self._spec_counts
self._spec_counts = (steps + len(batch.reqs), tokens + generated)
if self._decode_forward_count % self.decode_log_interval != 0:
return
now = self.clock()
gap = now - self._last_decode_time
self._last_decode_time = now
gen_throughput = self._decode_generated_tokens / gap if gap > 0 else 0.0
self._decode_generated_tokens = 0
accept = f"accept len: {self._spec_tokens / self._spec_req_steps:.2f}, " if self._spec_req_steps else ""
self._spec_req_steps = self._spec_tokens = 0
self.log(
f"Decode batch, "
f"#running-req: {running_reqs}, "
f"#token: {kv_used_pages * page_size}, "
f"token usage: {_usage_ratio(kv_used_pages, kv_total_pages):.2f}, "
f"{_swa_msg(swa_tokens)}"
f"{_mamba_msg(mamba_slots)}"
f"gen throughput (token/s): {gen_throughput:.2f}, "
f"{accept}"
f"#queue-req: {queue_reqs}"
)
if self.moe_stats is not None and (line := self.moe_stats()):
self.log(line)
def _usage_ratio(used: int, total: int) -> float:
return used / total if total > 0 else 0.0
def _mamba_msg(mamba_slots: tuple[int, int] | None) -> str:
"""GDN-state (mamba) pool occupancy for hybrid models; empty for the rest."""
if mamba_slots is None:
return ""
used, total = mamba_slots
return f"#mamba-slot: {used}/{total}, mamba usage: {_usage_ratio(used, total):.2f}, "
def _swa_msg(swa_tokens: tuple[int, int] | None) -> str:
"""Window (swa) pool occupancy for SWA models; empty for the rest."""
if swa_tokens is None:
return ""
used, total = swa_tokens
return f"#swa-token: {used}/{total}, swa usage: {_usage_ratio(used, total):.2f}, "