Files
FreeToken/python/freetoken/models/dflash/config.py
Xiaoze Fan 08cee44468 feat(spec): add DFlash 2 speculative decoding (#857)
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal

* feat(engine): Sampler.probs and sparse-q rejection

* feat(spec): let a proposer declare slot states riding the GDN slots

* feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families

* feat(attention): record the source slot of each tracked request

* feat(kernel): DFlash2 draft kernels

* feat(models): DFlash2 draft family

* feat(spec): DFlashWindow

* feat(spec): DFlashProposer

* docs(cli): add dflash to the speculative decoding section

* refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs

* fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply

* refactor(spec): share the draft path check between the MTP and DFlash proposers

* test(e2e): compare rebuilt outputs exactly only without hybrid MoE

* refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config

* perf(spec): compute the verify distribution once for sparse-q rejection

* perf(models): let the feature sink add the residual on the rows it keeps

* perf(models): share the DFlash draft K/V weights with the ring projection

* fix(kernel): keep the dflash selector off zero-probability candidates

* test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover

* fix(spec): default and cap DFlash steps at the draft's block_size - 1

* fix(models): reject DFlash draft configs the draft forward cannot serve

* test(e2e): read the MoE strategy from the rebuild test's extra args

* docs(spec): say that slot-state owners write their own prefill track snapshot

* test(models): move the DFlash draft parity tests under tests/models/dflash

* refactor(spec): drop the no-op verify handler of the DFlash window

* refactor(models): fold the DFlash candidate top-k into propose

* refactor(spec): declare the draft step limits through Proposer.num_steps only

* refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring

* refactor(models): name the DFlash draft submodules like the other model families

* refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py

* refactor(spec): hand the verify the draft distribution over the whole vocabulary

* refactor(spec): call the DFlashRing object dflash_ring and its length ring_len

* refactor(models): split the DFlash draft's propose into compute_candidates and sample_path

* refactor(spec): keep the draft distribution on its support in SpecProposal

* fix(spec): drop the step's proposal once the verify has read it

* refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink

* refactor(models): read the declared Batch.features directly instead of through getattr

* test(dflash): check the window rebuild inside the block forward test

* test(spec): share the plain run of the verify contract and run the DFlash draft once per family

* test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes

* test(spec): check the verify probs of each row against a torch top-k and top-p reference

* test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate

* refactor(scheduler): let callers take the speculative counts from the status reporter

* test(spec): keep the shared plain runs of the verify contract in their own temp dirs

* test(models): give the ricursive test's stand-in batch a features field
2026-11-07 12:29:18 -08:00

98 lines
5.0 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class DFlashDraftConfig:
hidden_size: int
num_layers: int
num_heads: int
num_kv_heads: int
head_dim: int
intermediate_size: int
hidden_act: str
vocab_size: int
rms_norm_eps: float
rope_theta: float
window: int
block_size: int
# the output of target layer i (0-based) feeds tap i, for every supported target
target_layer_ids: tuple[int, ...]
mask_token_id: int
selector_top_k: int
selector_rank: int
conv_kernel_size: int
conv_group_size: int
num_target_layers: int
def parse_config(hf_config: Any) -> DFlashDraftConfig:
"""The draft's ``config.json``; only the DFlash 2 shape (non-causal, all sliding, conv + selector, default rope) is served."""
archs = list(getattr(hf_config, "architectures", None) or [])
if archs != ["DFlash2DraftModel"]:
raise ValueError(f"expected a DFlash2DraftModel checkpoint, got architectures {archs}")
dc = hf_config.dflash_config
if not dc.get("selector_rank") or not dc.get("conv_kernel_size"):
raise ValueError("only DFlash 2 drafts (dflash_config with selector_rank and conv_kernel_size) are supported")
missing = [key for key in ("block_size", "target_layer_ids", "mask_token_id", "selector_top_k", "conv_group_size") if key not in dc]
if missing:
raise ValueError(f"dflash_config lacks {missing}")
if getattr(hf_config, "is_causal", False):
raise ValueError("only non-causal (is_causal false) DFlash drafts are supported")
types = set(getattr(hf_config, "layer_types", None) or ["full_attention"])
# the ring holds one window; a full-attention draft has no bound
if types != {"sliding_attention"}:
raise ValueError(f"only sliding_attention DFlash drafts are supported, got layer_types {sorted(types)}")
if getattr(hf_config, "sliding_window", None) is None:
raise ValueError("DFlash drafts need a sliding_window (use_sliding_window true)")
hidden, head_dim = int(hf_config.hidden_size), int(hf_config.head_dim)
rope = getattr(hf_config, "rope_parameters", None) or {"rope_theta": getattr(hf_config, "rope_theta", 10000.0)}
if rope.get("rope_type", "default") != "default":
raise ValueError(f"rope_type {rope.get('rope_type')!r} is not supported for DFlash drafts")
rope_dim = rope.get("rope_dim") or head_dim
factor = rope.get("partial_rotary_factor") or getattr(hf_config, "partial_rotary_factor", None) or 1.0
neox = bool(getattr(hf_config, "rope_is_neox_style", getattr(hf_config, "is_neox_style", True)))
# norm_rope_store rotates the whole head in NeoX halves
if (int(rope_dim), float(factor), neox) != (head_dim, 1.0, True):
raise ValueError(f"DFlash drafts need full NeoX rotary, got rope_dim {rope_dim}, partial_rotary_factor {factor}, neox style {neox}")
# the draft forward does not apply these; ignoring them would only show up as lower acceptance
identity = {"output_multiplier": (None, 1.0), "final_logit_softcapping": (None, 0.0), "attention_value_scale": (None, 1.0)}
scaled = {key: dc[key] for key, values in identity.items() if dc.get(key) not in values}
if scaled:
raise ValueError(f"DFlash drafts with {scaled} in dflash_config are not supported")
# tl.arange in the draft kernels needs powers of two
sizes = {"selector_top_k": int(dc["selector_top_k"]), "selector_rank": int(dc["selector_rank"]), "head_dim": head_dim}
odd = {key: v for key, v in sizes.items() if v < 1 or v & (v - 1)}
if odd:
raise ValueError(f"DFlash drafts need powers of two for {odd}")
group = int(dc["conv_group_size"])
if group < 1 or hidden % group:
raise ValueError(f"conv_group_size {group} does not divide hidden_size {hidden}")
taps = tuple(int(i) for i in dc["target_layer_ids"])
# each tap fills its own column block of the feature buffer
if not taps or len(set(taps)) != len(taps) or min(taps) < 0:
raise ValueError(f"target_layer_ids {list(taps)} must be non-empty, distinct and >= 0")
return DFlashDraftConfig(
hidden_size=hidden,
num_layers=int(hf_config.num_hidden_layers),
num_heads=int(hf_config.num_attention_heads),
num_kv_heads=int(hf_config.num_key_value_heads),
head_dim=head_dim,
intermediate_size=int(hf_config.intermediate_size),
hidden_act=str(getattr(hf_config, "hidden_act", "silu")),
vocab_size=int(hf_config.vocab_size),
rms_norm_eps=float(getattr(hf_config, "rms_norm_eps", 1e-6)),
rope_theta=float(rope["rope_theta"]),
window=int(hf_config.sliding_window),
block_size=int(dc["block_size"]),
target_layer_ids=taps,
mask_token_id=int(dc["mask_token_id"]),
selector_top_k=sizes["selector_top_k"],
selector_rank=sizes["selector_rank"],
conv_kernel_size=int(dc["conv_kernel_size"]),
conv_group_size=group,
num_target_layers=int(hf_config.num_target_layers),
)