mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 05:49:20 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
98 lines
5.0 KiB
Python
98 lines
5.0 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DFlashDraftConfig:
|
|
hidden_size: int
|
|
num_layers: int
|
|
num_heads: int
|
|
num_kv_heads: int
|
|
head_dim: int
|
|
intermediate_size: int
|
|
hidden_act: str
|
|
vocab_size: int
|
|
rms_norm_eps: float
|
|
rope_theta: float
|
|
window: int
|
|
block_size: int
|
|
# the output of target layer i (0-based) feeds tap i, for every supported target
|
|
target_layer_ids: tuple[int, ...]
|
|
mask_token_id: int
|
|
selector_top_k: int
|
|
selector_rank: int
|
|
conv_kernel_size: int
|
|
conv_group_size: int
|
|
num_target_layers: int
|
|
|
|
|
|
def parse_config(hf_config: Any) -> DFlashDraftConfig:
|
|
"""The draft's ``config.json``; only the DFlash 2 shape (non-causal, all sliding, conv + selector, default rope) is served."""
|
|
archs = list(getattr(hf_config, "architectures", None) or [])
|
|
if archs != ["DFlash2DraftModel"]:
|
|
raise ValueError(f"expected a DFlash2DraftModel checkpoint, got architectures {archs}")
|
|
dc = hf_config.dflash_config
|
|
if not dc.get("selector_rank") or not dc.get("conv_kernel_size"):
|
|
raise ValueError("only DFlash 2 drafts (dflash_config with selector_rank and conv_kernel_size) are supported")
|
|
missing = [key for key in ("block_size", "target_layer_ids", "mask_token_id", "selector_top_k", "conv_group_size") if key not in dc]
|
|
if missing:
|
|
raise ValueError(f"dflash_config lacks {missing}")
|
|
if getattr(hf_config, "is_causal", False):
|
|
raise ValueError("only non-causal (is_causal false) DFlash drafts are supported")
|
|
types = set(getattr(hf_config, "layer_types", None) or ["full_attention"])
|
|
# the ring holds one window; a full-attention draft has no bound
|
|
if types != {"sliding_attention"}:
|
|
raise ValueError(f"only sliding_attention DFlash drafts are supported, got layer_types {sorted(types)}")
|
|
if getattr(hf_config, "sliding_window", None) is None:
|
|
raise ValueError("DFlash drafts need a sliding_window (use_sliding_window true)")
|
|
hidden, head_dim = int(hf_config.hidden_size), int(hf_config.head_dim)
|
|
rope = getattr(hf_config, "rope_parameters", None) or {"rope_theta": getattr(hf_config, "rope_theta", 10000.0)}
|
|
if rope.get("rope_type", "default") != "default":
|
|
raise ValueError(f"rope_type {rope.get('rope_type')!r} is not supported for DFlash drafts")
|
|
rope_dim = rope.get("rope_dim") or head_dim
|
|
factor = rope.get("partial_rotary_factor") or getattr(hf_config, "partial_rotary_factor", None) or 1.0
|
|
neox = bool(getattr(hf_config, "rope_is_neox_style", getattr(hf_config, "is_neox_style", True)))
|
|
# norm_rope_store rotates the whole head in NeoX halves
|
|
if (int(rope_dim), float(factor), neox) != (head_dim, 1.0, True):
|
|
raise ValueError(f"DFlash drafts need full NeoX rotary, got rope_dim {rope_dim}, partial_rotary_factor {factor}, neox style {neox}")
|
|
# the draft forward does not apply these; ignoring them would only show up as lower acceptance
|
|
identity = {"output_multiplier": (None, 1.0), "final_logit_softcapping": (None, 0.0), "attention_value_scale": (None, 1.0)}
|
|
scaled = {key: dc[key] for key, values in identity.items() if dc.get(key) not in values}
|
|
if scaled:
|
|
raise ValueError(f"DFlash drafts with {scaled} in dflash_config are not supported")
|
|
# tl.arange in the draft kernels needs powers of two
|
|
sizes = {"selector_top_k": int(dc["selector_top_k"]), "selector_rank": int(dc["selector_rank"]), "head_dim": head_dim}
|
|
odd = {key: v for key, v in sizes.items() if v < 1 or v & (v - 1)}
|
|
if odd:
|
|
raise ValueError(f"DFlash drafts need powers of two for {odd}")
|
|
group = int(dc["conv_group_size"])
|
|
if group < 1 or hidden % group:
|
|
raise ValueError(f"conv_group_size {group} does not divide hidden_size {hidden}")
|
|
taps = tuple(int(i) for i in dc["target_layer_ids"])
|
|
# each tap fills its own column block of the feature buffer
|
|
if not taps or len(set(taps)) != len(taps) or min(taps) < 0:
|
|
raise ValueError(f"target_layer_ids {list(taps)} must be non-empty, distinct and >= 0")
|
|
return DFlashDraftConfig(
|
|
hidden_size=hidden,
|
|
num_layers=int(hf_config.num_hidden_layers),
|
|
num_heads=int(hf_config.num_attention_heads),
|
|
num_kv_heads=int(hf_config.num_key_value_heads),
|
|
head_dim=head_dim,
|
|
intermediate_size=int(hf_config.intermediate_size),
|
|
hidden_act=str(getattr(hf_config, "hidden_act", "silu")),
|
|
vocab_size=int(hf_config.vocab_size),
|
|
rms_norm_eps=float(getattr(hf_config, "rms_norm_eps", 1e-6)),
|
|
rope_theta=float(rope["rope_theta"]),
|
|
window=int(hf_config.sliding_window),
|
|
block_size=int(dc["block_size"]),
|
|
target_layer_ids=taps,
|
|
mask_token_id=int(dc["mask_token_id"]),
|
|
selector_top_k=sizes["selector_top_k"],
|
|
selector_rank=sizes["selector_rank"],
|
|
conv_kernel_size=int(dc["conv_kernel_size"]),
|
|
conv_group_size=group,
|
|
num_target_layers=int(hf_config.num_target_layers),
|
|
)
|