Files
Xiaoze Fan dacac5f5fa feat(spec): add MTP drafts for the Qwen3.5 family and Qwen3.8-Flash-Next (#844)
* feat(spec): add the MTP proposer for the qwen3_5 family

* test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints

* feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers

* feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack

* docs(cli): document the speculative decoding flags

* fix(spec): refuse MTP draft sources the draft cannot read before the target loads

* docs(qwen4_exp): describe the verify path in the PLE docstrings

* fix(spec): refuse a missing relative MTP draft path before the target loads

* fix(moe): read MTP head experts on the host and require both stacked tensors

* refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq

* refactor(spec): let each MTP draft family check its own weight source

* fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows

* perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr

* refactor(moe): let each family name its MTP expert tensors

* refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load

* fix(spec): refuse MTP with dummy weights

* refactor(spec): move HiddenRelay under spec/mtp, its only user

* refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens

* refactor(models): note FeatureSink's graph binding, drop its runtime_checkable

* fix(models): read quantized MTP fc projections under the quant config
2026-11-02 11:37:09 -08:00

111 lines
3.9 KiB
Python

from __future__ import annotations
from abc import ABC, abstractmethod
from contextlib import contextmanager
from typing import TYPE_CHECKING, Protocol, runtime_checkable
import torch
from freetoken.layers import (
BaseOP,
LinearColParallelMerged,
LinearRowParallel,
gelu_and_mul,
gelu_tanh_and_mul,
silu_and_mul,
)
from freetoken.utils import nvtx_annotate
if TYPE_CHECKING:
import torch
from freetoken.core import Batch
from freetoken.layers.quantization import QuantConfig
from .config import ModelConfig
class BaseLLMModel(ABC, BaseOP):
@abstractmethod
def forward(self) -> torch.Tensor: ...
@contextmanager
def forward_host_ctx(self, batch: Batch, use_graph: bool):
"""Around one forward dispatch: enter before it is enqueued, exit right after. A backend that feeds the forward from host memory overrides this."""
yield
@runtime_checkable
class SupportsMultimodal(Protocol):
"""What the engine asks of a model that serves multimodal items; the text forward takes its token embeddings from embed_input_ids."""
def encode(self, item: MMItem) -> torch.Tensor:
"""``[item.num_tokens, D]`` soft tokens for one item of any modality the family serves, on the model device."""
...
def place_encoder_weights(self, mode: str) -> None:
"""Every encoder tower's weights: ``gpu`` resident or ``host`` streamed from pinned banks."""
...
class FeatureSink(Protocol):
"""Where a forward hands per-row hidden states to a draft model; set on ``batch.features``.
A CUDA graph replays the sink it was captured with, never the one set at replay: write only to buffers the draft owns, keep no per-step state on the sink.
"""
wanted_layers: frozenset[int]
def final_hidden(self, x: torch.Tensor) -> None:
"""Every row's final hidden state, once per forward: the LM head's input, or what the model hands over itself before the head."""
...
def layer_output(self, layer_id: int, x: torch.Tensor, num_streams: int = 1) -> None:
"""A layer's output rows; called only for layers in ``wanted_layers``."""
...
def embed_input_ids(embed_tokens, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:
"""Token embeddings of the batch; on a chunk with multimodal rows, batch.mm_embeds' leading columns replace the rows batch.mm_rows."""
if batch.mm_embeds is None:
return embed_tokens.forward(input_ids)
# multimodal rows carry content pad ids above the vocab: clamp for the lookup, the copy overwrites those rows
x = embed_tokens.forward(input_ids.clamp(max=embed_tokens.num_embeddings - 1))
x.index_copy_(0, batch.mm_rows, batch.mm_embeds[:, : x.shape[1]].to(x.dtype))
return x
class GatedMLP(BaseOP):
def __init__(self, config: ModelConfig, *, quant_config: QuantConfig | None = None, prefix: str = ""):
self.gate_up_proj = LinearColParallelMerged(
config.hidden_size,
[config.intermediate_size, config.intermediate_size],
has_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
fn_map = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
act_fn = fn_map.get(config.hidden_act, None)
if act_fn is None:
raise ValueError(f"Unsupported activation function: {config.hidden_act}")
self.act_fn = act_fn
self.down_proj = LinearRowParallel(
config.intermediate_size,
config.hidden_size,
has_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
@nvtx_annotate("MLP")
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up = self.gate_up_proj.forward(x)
del x
y = self.act_fn(gate_up)
del gate_up
return self.down_proj.forward(y)
__all__ = ["BaseLLMModel", "GatedMLP"]