mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
171 lines
6.4 KiB
Python
171 lines
6.4 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Callable
|
|
|
|
from freetoken.core import Batch
|
|
|
|
|
|
@dataclass
|
|
class SchedulerStatusReporter:
|
|
log: Callable[[str], None]
|
|
clock: Callable[[], float] = time.perf_counter
|
|
decode_log_interval: int = 40
|
|
# --moe-collect-stats: one more line per decode log line (the stats since the previous one), or None to skip
|
|
moe_stats: Callable[[], str | None] | None = None
|
|
_last_prefill_time: float = field(init=False)
|
|
_last_decode_time: float = field(init=False)
|
|
_decode_forward_count: int = field(default=0, init=False)
|
|
_decode_generated_tokens: int = field(default=0, init=False)
|
|
# speculative steps since the last decode log: requests stepped and tokens they committed
|
|
_spec_req_steps: int = field(default=0, init=False)
|
|
_spec_tokens: int = field(default=0, init=False)
|
|
# the same since the last take_spec_counts(), which the decode log leaves alone
|
|
_spec_counts: tuple[int, int] = field(default=(0, 0), init=False)
|
|
|
|
def __post_init__(self) -> None:
|
|
now = self.clock()
|
|
self._last_prefill_time = now
|
|
self._last_decode_time = now
|
|
self.decode_log_interval = max(1, self.decode_log_interval)
|
|
|
|
def report_batch(
|
|
self,
|
|
batch: Batch,
|
|
*,
|
|
running_reqs: int,
|
|
queue_reqs: int,
|
|
kv_used_pages: int,
|
|
kv_total_pages: int,
|
|
page_size: int,
|
|
mamba_slots: tuple[int, int] | None = None,
|
|
swa_tokens: tuple[int, int] | None = None,
|
|
generated: int | None = None,
|
|
) -> None:
|
|
"""``generated``: tokens a decode step committed; None means one per request."""
|
|
if batch.is_prefill:
|
|
self._report_prefill(
|
|
batch,
|
|
running_reqs=running_reqs,
|
|
queue_reqs=queue_reqs,
|
|
kv_used_pages=kv_used_pages,
|
|
kv_total_pages=kv_total_pages,
|
|
mamba_slots=mamba_slots,
|
|
swa_tokens=swa_tokens,
|
|
)
|
|
else:
|
|
self._report_decode(
|
|
batch,
|
|
running_reqs=running_reqs,
|
|
queue_reqs=queue_reqs,
|
|
kv_used_pages=kv_used_pages,
|
|
kv_total_pages=kv_total_pages,
|
|
page_size=page_size,
|
|
mamba_slots=mamba_slots,
|
|
swa_tokens=swa_tokens,
|
|
generated=generated,
|
|
)
|
|
|
|
def take_spec_counts(self) -> tuple[int, int]:
|
|
"""Requests stepped and tokens they committed over the speculative decode steps since the last call."""
|
|
counts, self._spec_counts = self._spec_counts, (0, 0)
|
|
return counts
|
|
|
|
def _report_prefill(
|
|
self,
|
|
batch: Batch,
|
|
*,
|
|
running_reqs: int,
|
|
queue_reqs: int,
|
|
kv_used_pages: int,
|
|
kv_total_pages: int,
|
|
mamba_slots: tuple[int, int] | None = None,
|
|
swa_tokens: tuple[int, int] | None = None,
|
|
) -> None:
|
|
now = self.clock()
|
|
gap = now - self._last_prefill_time
|
|
self._last_prefill_time = now
|
|
# Read the schedule-time snapshot: by report time the forward's complete_one() has
|
|
# advanced each req's cached_len to device_len, so reading the reqs here would log
|
|
# decode-state values (#new-token == #reqs, #cached-token == full prompt).
|
|
new_tokens = batch.log_new_tokens
|
|
cached_tokens = batch.log_cached_tokens
|
|
input_throughput = new_tokens / gap if gap > 0 else 0.0
|
|
self.log(
|
|
f"Prefill batch, "
|
|
f"#new-seq: {len(batch.reqs)}, "
|
|
f"#new-token: {new_tokens}, "
|
|
f"#cached-token: {cached_tokens}, "
|
|
f"token usage: {_usage_ratio(kv_used_pages, kv_total_pages):.2f}, "
|
|
f"{_swa_msg(swa_tokens)}"
|
|
f"{_mamba_msg(mamba_slots)}"
|
|
f"#running-req: {running_reqs}, "
|
|
f"#queue-req: {queue_reqs}, "
|
|
f"input throughput (token/s): {input_throughput:.2f}"
|
|
)
|
|
|
|
def _report_decode(
|
|
self,
|
|
batch: Batch,
|
|
*,
|
|
running_reqs: int,
|
|
queue_reqs: int,
|
|
kv_used_pages: int,
|
|
kv_total_pages: int,
|
|
page_size: int,
|
|
mamba_slots: tuple[int, int] | None = None,
|
|
swa_tokens: tuple[int, int] | None = None,
|
|
generated: int | None = None,
|
|
) -> None:
|
|
self._decode_forward_count += 1
|
|
self._decode_generated_tokens += len(batch.reqs) if generated is None else generated
|
|
if generated is not None:
|
|
self._spec_req_steps += len(batch.reqs)
|
|
self._spec_tokens += generated
|
|
steps, tokens = self._spec_counts
|
|
self._spec_counts = (steps + len(batch.reqs), tokens + generated)
|
|
if self._decode_forward_count % self.decode_log_interval != 0:
|
|
return
|
|
|
|
now = self.clock()
|
|
gap = now - self._last_decode_time
|
|
self._last_decode_time = now
|
|
gen_throughput = self._decode_generated_tokens / gap if gap > 0 else 0.0
|
|
self._decode_generated_tokens = 0
|
|
accept = f"accept len: {self._spec_tokens / self._spec_req_steps:.2f}, " if self._spec_req_steps else ""
|
|
self._spec_req_steps = self._spec_tokens = 0
|
|
self.log(
|
|
f"Decode batch, "
|
|
f"#running-req: {running_reqs}, "
|
|
f"#token: {kv_used_pages * page_size}, "
|
|
f"token usage: {_usage_ratio(kv_used_pages, kv_total_pages):.2f}, "
|
|
f"{_swa_msg(swa_tokens)}"
|
|
f"{_mamba_msg(mamba_slots)}"
|
|
f"gen throughput (token/s): {gen_throughput:.2f}, "
|
|
f"{accept}"
|
|
f"#queue-req: {queue_reqs}"
|
|
)
|
|
if self.moe_stats is not None and (line := self.moe_stats()):
|
|
self.log(line)
|
|
|
|
|
|
def _usage_ratio(used: int, total: int) -> float:
|
|
return used / total if total > 0 else 0.0
|
|
|
|
|
|
def _mamba_msg(mamba_slots: tuple[int, int] | None) -> str:
|
|
"""GDN-state (mamba) pool occupancy for hybrid models; empty for the rest."""
|
|
if mamba_slots is None:
|
|
return ""
|
|
used, total = mamba_slots
|
|
return f"#mamba-slot: {used}/{total}, mamba usage: {_usage_ratio(used, total):.2f}, "
|
|
|
|
|
|
def _swa_msg(swa_tokens: tuple[int, int] | None) -> str:
|
|
"""Window (swa) pool occupancy for SWA models; empty for the rest."""
|
|
if swa_tokens is None:
|
|
return ""
|
|
used, total = swa_tokens
|
|
return f"#swa-token: {used}/{total}, swa usage: {_usage_ratio(used, total):.2f}, "
|