mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
72 lines
3.5 KiB
Python
72 lines
3.5 KiB
Python
"""Rejection sampling for a speculative verify.
|
|
|
|
With argmax drafts the draft distribution q is one-hot at the drafted id. Chain rejection with a one-hot q accepts d with probability p(d) and on a reject samples p with d removed.
|
|
Drawing y ~ p at every row and accepting drafts while y == d gives exactly that: P(accept) = p(d), and a mismatching y is p conditioned on y != d.
|
|
So the verify samples each row as decode would and compares prefixes; greedy rows reduce to the argmax prefix match.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Tuple
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.engine.sample import BatchSamplingArgs, Sampler
|
|
|
|
|
|
def reject(
|
|
logits: torch.Tensor, # [bs*(k+1), V], k+1 rows per request
|
|
drafts: torch.Tensor, # [bs, k] int64
|
|
args: BatchSamplingArgs,
|
|
sampler: Sampler,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""(out [bs, k+1], emitted [bs]): emitted counts the accepted drafts (0..k); a request commits ``out[i, : emitted[i] + 1]``."""
|
|
bs, k = drafts.shape
|
|
target = sampler.sample_verify(logits, args, drafts).view(bs, k + 1).to(torch.int64)
|
|
emitted = (drafts == target[:, :k]).to(torch.int64).cumprod(dim=1).sum(dim=1)
|
|
return target, emitted
|
|
|
|
|
|
def reject_with_draft_probs(
|
|
logits: torch.Tensor, # [bs*(k+1), V]
|
|
drafts: torch.Tensor, # [bs, k] int64, slot j drawn from q[:, j]
|
|
support: torch.Tensor, # [bs, k, K] the token ids where q can be nonzero
|
|
q: torch.Tensor, # [bs, k, K] q over support
|
|
args: BatchSamplingArgs,
|
|
sampler: Sampler,
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""``reject`` for drafts sampled from q: slot j accepts d with probability min(1, p(d)/q(d)) and on a reject samples max(0, p - q); greedy rows and rows whose q is not a distribution take the one-hot result."""
|
|
from freetoken.engine.sample import sampling_module
|
|
|
|
p = sampler.probs(logits, args, drafts)
|
|
if p is None:
|
|
return reject(logits, drafts, args, sampler)
|
|
bs, k = drafts.shape
|
|
p = p.view(bs, k + 1, -1)
|
|
# a NaN or a wrong total in q would reach the target's output through the residual
|
|
ok = torch.isfinite(q).flatten(1).all(dim=1) & ((q.sum(dim=-1) - 1.0).abs() < 1e-3).all(dim=1)
|
|
if args.greedy_mask is not None:
|
|
ok &= ~args.greedy_mask
|
|
q = torch.where(ok.view(-1, 1, 1), q, torch.zeros_like(q))
|
|
d = drafts.unsqueeze(-1)
|
|
p_d = p[:, :k].gather(2, d).squeeze(-1)
|
|
# a row without a usable q verifies with q one-hot at its draft, which is ``reject``: accept with p(d), else sample p without d; a greedy row's p is one-hot at its argmax
|
|
q_d = torch.where(ok.view(-1, 1), (q * (support == d)).sum(dim=-1), 1.0)
|
|
accept = torch.rand_like(p_d) * q_d < p_d
|
|
emitted = accept.to(torch.int64).cumprod(dim=1).sum(dim=1)
|
|
rows = torch.arange(bs, device=p.device)
|
|
slot = emitted.clamp(max=k - 1)
|
|
rejected = (emitted < k).view(-1, 1)
|
|
# the residual max(0, p - q) at the first rejected slot; the bonus row is p itself
|
|
pick = p[rows, emitted]
|
|
pick.scatter_add_(1, support[rows, slot], -q[rows, slot] * rejected)
|
|
pick.scatter_add_(1, drafts[rows, slot].view(-1, 1), -(rejected & ~ok.view(-1, 1)).float())
|
|
pick.clamp_(min=0.0)
|
|
total = pick.sum(dim=-1, keepdim=True)
|
|
pick = torch.where(total > 0, pick / total.clamp_min(1e-30), p[rows, emitted])
|
|
recovered = sampling_module().sampling_from_probs(pick).to(torch.int64)
|
|
out = torch.cat([drafts, drafts[:, -1:]], dim=1)
|
|
out.scatter_(1, emitted.view(-1, 1), recovered.view(-1, 1))
|
|
return out, emitted
|