mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 04:15:36 +08:00
* feat(spec): add the MTP proposer for the qwen3_5 family * test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints * feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers * feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack * docs(cli): document the speculative decoding flags * fix(spec): refuse MTP draft sources the draft cannot read before the target loads * docs(qwen4_exp): describe the verify path in the PLE docstrings * fix(spec): refuse a missing relative MTP draft path before the target loads * fix(moe): read MTP head experts on the host and require both stacked tensors * refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq * refactor(spec): let each MTP draft family check its own weight source * fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows * perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr * refactor(moe): let each family name its MTP expert tensors * refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load * fix(spec): refuse MTP with dummy weights * refactor(spec): move HiddenRelay under spec/mtp, its only user * refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens * refactor(models): note FeatureSink's graph binding, drop its runtime_checkable * fix(models): read quantized MTP fc projections under the quant config
242 lines
11 KiB
Python
242 lines
11 KiB
Python
"""Qwen3.8-Flash-Next decoder stack (text-only).
|
|
|
|
The residual state is ``R [T, hc_count*hidden]`` end to end: the embedding is repeated over the
|
|
``hc_count`` streams, every layer mixes them down to one ``[T, hidden]`` block input and injects
|
|
its output back, and the top-level mixer collapses them once before ``lm_head``. There is no
|
|
input/post layernorm and no final ``model.norm`` -- the hyper-connection norms are the only ones.
|
|
|
|
Layer contract (frozen): ``forward(R [T, hc*hidden], batch) -> R' [T, hc*hidden]`` with an
|
|
immediate combine::
|
|
|
|
R = R + ple(R, batch) # zero-based layer 1 only
|
|
x, s = attn_hc.mix(R); y = (GDN | QSA)(x); R = attn_hc.combine(R, y, s)
|
|
x, s = mlp_hc.mix(R); y = MoE(x); R = mlp_hc.combine(R, y, s)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, List
|
|
|
|
import torch
|
|
from freetoken.core import get_global_ctx
|
|
from freetoken.layers import BaseOP, OPList, ParallelLMHead, VocabParallelEmbedding
|
|
from freetoken.models.blocks import BaseLLMModel
|
|
from freetoken.utils import nvtx_annotate
|
|
|
|
from .attention import Qwen4ExpAttention
|
|
from .config import PLE_NGRAM_STATE
|
|
from .hc import GatedResidual
|
|
from .moe import Qwen4ExpMoE
|
|
from .ple import PLELayer
|
|
from freetoken.models.blocks import embed_input_ids
|
|
from freetoken.models.qwen3_vl.vision import Qwen3VLVisionModel, QwenVLVisionMixin
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.core import Batch
|
|
from freetoken.models.config import ModelConfig
|
|
|
|
|
|
def build_linear_mixer(config: ModelConfig, layer_id: int, prefix: str) -> BaseOP:
|
|
"""GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate)."""
|
|
from freetoken.layers.linear_attn import GatedDeltaNet
|
|
|
|
g = config.linear_attention_group()
|
|
return GatedDeltaNet(
|
|
hidden_size=config.hidden_size,
|
|
num_k_heads=g.num_key_heads,
|
|
num_v_heads=g.num_value_heads,
|
|
head_k_dim=g.key_head_dim,
|
|
head_v_dim=g.value_head_dim,
|
|
conv_kernel_size=g.conv_kernel_dim,
|
|
rms_norm_eps=config.rms_norm_eps,
|
|
layer_id=layer_id,
|
|
output_gate=g.output_gate,
|
|
quant_config=config.quant,
|
|
prefix=prefix,
|
|
out_heads_tiled=config.qwen4_args.gdn_out_heads_tiled,
|
|
)
|
|
|
|
|
|
class Qwen4ExpDecoderLayer(BaseOP):
|
|
"""One decoder layer over the hyper-connection streams (see the module docstring for the flow)."""
|
|
|
|
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = "") -> None:
|
|
self._layer_id = layer_id
|
|
self._is_linear = config.is_linear_layer(layer_id)
|
|
if self._is_linear:
|
|
self.linear_attn = build_linear_mixer(config, layer_id, f"{prefix}.linear_attn")
|
|
else:
|
|
self.self_attn = Qwen4ExpAttention(config, layer_id, prefix=f"{prefix}.self_attn")
|
|
self.mlp = Qwen4ExpMoE(config, layer_id, prefix=f"{prefix}.mlp")
|
|
self.attn_hyper_connection = GatedResidual(config, prefix=f"{prefix}.attn_hyper_connection")
|
|
self.mlp_hyper_connection = GatedResidual(config, prefix=f"{prefix}.mlp_hyper_connection")
|
|
self.ple = (
|
|
PLELayer(config, layer_id, prefix=f"{prefix}.ple") if layer_id in config.qwen4_args.ple_layer_ids else None
|
|
)
|
|
|
|
@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
|
|
def forward(self, hidden: torch.Tensor, batch: Batch) -> torch.Tensor:
|
|
if self.ple is not None:
|
|
hidden = hidden + self.ple.forward(hidden, batch)
|
|
block_input, inject = self.attn_hyper_connection.mix(hidden)
|
|
if self._is_linear:
|
|
block_output = self.linear_attn.forward(block_input)
|
|
else:
|
|
block_output = self.self_attn.forward(block_input, batch)
|
|
hidden = self.attn_hyper_connection.combine(hidden, block_output, inject)
|
|
block_input, inject = self.mlp_hyper_connection.mix(hidden)
|
|
return self.mlp_hyper_connection.combine(hidden, self.mlp.forward(block_input), inject)
|
|
|
|
|
|
class Qwen4ExpModel(BaseOP):
|
|
def __init__(self, config: ModelConfig, *, prefix: str = "model") -> None:
|
|
self.hc_count = config.qwen4_args.hc_count
|
|
self.embed_tokens = VocabParallelEmbedding(
|
|
num_embeddings=config.vocab_size,
|
|
embedding_dim=config.hidden_size,
|
|
quant_config=config.quant,
|
|
prefix=f"{prefix}.embed_tokens",
|
|
)
|
|
self.layers = OPList(
|
|
[
|
|
Qwen4ExpDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
|
|
for layer_id in range(config.num_layers)
|
|
]
|
|
)
|
|
self.hyper_connection_mixer = GatedResidual(config, use_combine=False, prefix=f"{prefix}.hyper_connection_mixer")
|
|
# plain tuple (not an OP child), so it never shows up in the state dict
|
|
self._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None)
|
|
|
|
@property
|
|
def ple_layers(self) -> List[PLELayer]:
|
|
"""The PLE layers in decoder order -- the seam the loader attaches table backends to."""
|
|
return list(self._ple)
|
|
|
|
def forward(self, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:
|
|
hidden = embed_input_ids(self.embed_tokens, input_ids, batch)
|
|
hidden = hidden.repeat(1, self.hc_count)
|
|
meta = None
|
|
if self._ple:
|
|
from .ple import build_ple_metadata, commit_ngram_context
|
|
|
|
meta = build_ple_metadata(batch, self._ple[0].args, input_ids.device)
|
|
for ple in self._ple: # gather the pinned-host PLE rows while the early layers run
|
|
ple.start_prefetch(batch, meta)
|
|
for layer in self.layers.op_list:
|
|
hidden = layer.forward(hidden, batch)
|
|
if meta is not None and meta.is_verify:
|
|
# the commit rolls the context by the accepted rows only
|
|
get_global_ctx().linear_state_pool.verify_handler(PLE_NGRAM_STATE).stash(meta)
|
|
elif meta is not None:
|
|
# single writer: the layers only read the context, so a second PLE layer's
|
|
# prefetch sees the un-rolled window
|
|
commit_ngram_context(meta, getattr(batch, "fla_metadata", None))
|
|
if (features := getattr(batch, "features", None)) is not None:
|
|
# a draft reads the residual streams before the final mix
|
|
features.final_hidden(hidden)
|
|
return self.hyper_connection_mixer.mix(hidden)[0]
|
|
|
|
|
|
class Qwen4ExpForCausalLM(BaseLLMModel):
|
|
def __init__(self, config: ModelConfig) -> None:
|
|
self._config = config
|
|
self.model = Qwen4ExpModel(config)
|
|
self.lm_head = ParallelLMHead(
|
|
num_embeddings=config.vocab_size,
|
|
embedding_dim=config.hidden_size,
|
|
tie_word_embeddings=config.tie_word_embeddings,
|
|
tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
|
|
quant_config=config.quant,
|
|
prefix="lm_head",
|
|
)
|
|
super().__init__()
|
|
|
|
def load_host_tables(self, engine_config) -> int:
|
|
"""Attach the PLE n-gram table (pinned checkpoint bank, or zeros for dummy weights); returns the pinned host bytes the engine reserves from its pin budget."""
|
|
ple_layers = self.model.ple_layers
|
|
if not ple_layers:
|
|
return 0
|
|
from .ple import PinnedUVATable, ZeroTable, derive_ngram_hash_constants
|
|
|
|
if getattr(engine_config, "use_dummy_weight", False):
|
|
# Dummy fill leaves the int64 hash buffers garbage (a zero vocab size divides by
|
|
# zero in the hash), so re-derive the real constants and read a zero table.
|
|
for ple in ple_layers:
|
|
args = ple.args
|
|
mult, sizes, offsets = derive_ngram_hash_constants(
|
|
vocab_size=self._config.vocab_size,
|
|
ngram_size=args.ngram_size,
|
|
num_ngram_heads=args.num_ngram_heads,
|
|
ngram_vocab_size_base=args.ngram_vocab_size_base,
|
|
ple_layer_index=ple.ple_index,
|
|
)
|
|
emb = ple.ple_embedding
|
|
emb.layer_multipliers.copy_(torch.tensor(mult, dtype=torch.int64))
|
|
emb.ngram_heads_vocab_sizes.copy_(torch.tensor(sizes, dtype=torch.int64))
|
|
emb.ngram_heads_offsets.copy_(torch.tensor(offsets, dtype=torch.int64))
|
|
emb.attach_table(ZeroTable(offsets[-1] + sizes[-1], args.ngram_head_dim))
|
|
return 0
|
|
|
|
if engine_config.ple_backend == "disk":
|
|
from .ple_disk import DiskRowTable, resolve_row_source
|
|
|
|
# one WAIT node per captured graph: the flag protocol supports a single consume
|
|
assert len(ple_layers) == 1, "disk PLE backend expects exactly one PLE layer"
|
|
emb, args = ple_layers[0].ple_embedding, ple_layers[0].args
|
|
# hash with the state-dict-loaded constants, the same source the pinned path reads
|
|
constants = {
|
|
"num_ngram_heads": args.num_ngram_heads,
|
|
"layer_multipliers": emb.layer_multipliers.tolist(),
|
|
"per_head_vocab_sizes": emb.ngram_heads_vocab_sizes.tolist(),
|
|
"per_head_offsets": emb.ngram_heads_offsets.tolist(),
|
|
"eos_token_id": args.ngram_boundary_token_id,
|
|
"image_token_id": args.image_token_id,
|
|
}
|
|
rows = engine_config.decode_rows_per_req
|
|
disk_table = DiskRowTable(
|
|
resolve_row_source(engine_config.model_path),
|
|
constants,
|
|
# verify graphs stop at max_running_req requests, decode graphs at cuda_graph_max_bs
|
|
max_graph_rows=engine_config.max_running_req * rows if rows > 1 else max(256, engine_config.cuda_graph_max_bs or 0),
|
|
# an eager decode or verify step stages max_running_req * rows tokens, which can exceed a prefill chunk
|
|
max_eager_tokens=max(engine_config.max_extend_tokens, engine_config.max_running_req * rows),
|
|
)
|
|
self._ple_table = disk_table
|
|
for ple in ple_layers:
|
|
ple.ple_embedding.attach_table(disk_table)
|
|
# engine enters this around every dispatch; the graph itself never waits on the disk
|
|
self.forward_host_ctx = disk_table.forward_host_ctx
|
|
return 0
|
|
|
|
from .weight import load_ple_table
|
|
|
|
table = load_ple_table(engine_config.model_path, self._config.qwen4_args)
|
|
self._ple_table = table # owns the pinned HostBank; keep it alive
|
|
for ple in ple_layers:
|
|
ple.ple_embedding.attach_table(
|
|
PinnedUVATable(table.bank.tensor, float(table.weight_scale))
|
|
)
|
|
return table.bank.nbytes
|
|
|
|
def forward(self) -> torch.Tensor:
|
|
batch = get_global_ctx().batch
|
|
# the model hands its pre-mix streams to batch.features itself
|
|
return self.lm_head.forward(self.model.forward(batch.input_ids, batch), export_hidden=False)
|
|
|
|
|
|
class Qwen4ExpForConditionalGeneration(QwenVLVisionMixin, Qwen4ExpForCausalLM):
|
|
def __init__(self, config: ModelConfig) -> None:
|
|
super().__init__(config)
|
|
if config.is_multimodal:
|
|
assert not config.vision_config.deepstack_visual_indexes, "Qwen3.8 consumes no DeepStack features"
|
|
self.visual = Qwen3VLVisionModel(config.vision_config, quant_config=config.quant, prefix="visual")
|
|
|
|
|
|
__all__ = [
|
|
"Qwen4ExpDecoderLayer",
|
|
"Qwen4ExpForCausalLM",
|
|
"Qwen4ExpForConditionalGeneration",
|
|
"Qwen4ExpModel",
|
|
"build_linear_mixer",
|
|
]
|