Files
Xiaoze Fan dacac5f5fa feat(spec): add MTP drafts for the Qwen3.5 family and Qwen3.8-Flash-Next (#844)
* feat(spec): add the MTP proposer for the qwen3_5 family

* test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints

* feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers

* feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack

* docs(cli): document the speculative decoding flags

* fix(spec): refuse MTP draft sources the draft cannot read before the target loads

* docs(qwen4_exp): describe the verify path in the PLE docstrings

* fix(spec): refuse a missing relative MTP draft path before the target loads

* fix(moe): read MTP head experts on the host and require both stacked tensors

* refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq

* refactor(spec): let each MTP draft family check its own weight source

* fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows

* perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr

* refactor(moe): let each family name its MTP expert tensors

* refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load

* fix(spec): refuse MTP with dummy weights

* refactor(spec): move HiddenRelay under spec/mtp, its only user

* refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens

* refactor(models): note FeatureSink's graph binding, drop its runtime_checkable

* fix(models): read quantized MTP fc projections under the quant config
2026-11-02 11:37:09 -08:00

216 lines
6.6 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Literal, Protocol
from freetoken.utils import Registry, init_logger
from .base import AttentionSpec, AttnType, BaseAttnBackend, BaseAttnMetadata, HybridBackend
if TYPE_CHECKING:
from freetoken.models import ModelConfig
logger = init_logger(__name__)
class BackendCreator(Protocol):
def __call__(self, config: ModelConfig) -> BaseAttnBackend: ...
@dataclass(frozen=True)
class BackendInfo:
"""Declarative capability matrix entry for one backend. The engine's config-time
validation interprets the requirement flags through its own (monkeypatch-able)
availability probes; this stays pure data so registration never imports kernels."""
supported_types: frozenset[AttnType]
requires_flashinfer: bool = False
requires_sgl_kernel: bool = False
requires_sm100: bool = False
# Allowed page sizes (None -> any). Config-time resolution coerces to the last
# entry when the resolved page_size is not in the list.
page_sizes: tuple[int, ...] | None = None
# Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks).
# Non-consumers raise on a non-None spec instead of silently dropping it.
consumes_attn_spec: bool = False
# How the backend captures a graph of `rows_per_req` query rows per request: "device" keeps all metadata on device, "host_plan" plans on the host before each replay, "none" cannot.
uniform_extend_graph: Literal["none", "host_plan", "device"] = "none"
SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend")
@SUPPORTED_ATTENTION_BACKENDS.register(
"trtllm",
BackendInfo(
supported_types=frozenset({AttnType.FULL}),
requires_flashinfer=True,
requires_sm100=True,
page_sizes=(16, 32, 64),
),
)
def create_trtllm_backend(config: ModelConfig):
from .trtllm import TensorRTLLMBackend
return TensorRTLLMBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"fi",
BackendInfo(
supported_types=frozenset({AttnType.FULL}),
requires_flashinfer=True,
uniform_extend_graph="host_plan",
),
)
def create_fi_backend(config: ModelConfig):
from .fi import FlashInferBackend
return FlashInferBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"fa",
BackendInfo(
supported_types=frozenset({AttnType.FULL}),
requires_sgl_kernel=True,
uniform_extend_graph="device",
),
)
def create_fa_backend(config: ModelConfig):
from .fa import FlashAttentionBackend
return FlashAttentionBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"triton",
BackendInfo(
supported_types=frozenset({AttnType.FULL, AttnType.SWA}),
consumes_attn_spec=True,
uniform_extend_graph="device",
),
)
def create_triton_backend(config: ModelConfig):
from .triton import TritonAttentionBackend
return TritonAttentionBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"dsv4_sparse",
BackendInfo(supported_types=frozenset({AttnType.DSV4})),
)
def create_dsv4_sparse_backend(config: ModelConfig):
from .dsv4_sparse import DSV4SparseAttnBackend
return DSV4SparseAttnBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"dsv41_sparse",
BackendInfo(supported_types=frozenset({AttnType.DSV41})),
)
def create_dsv41_sparse_backend(config: ModelConfig):
from .dsv41_sparse import DSV41SparseAttnBackend
return DSV41SparseAttnBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"dsa",
BackendInfo(supported_types=frozenset({AttnType.MLA, AttnType.DSA})),
)
def create_dsa_backend(config: ModelConfig):
# MLA with a grouped index (index_ratio > 1) is the kpool indexer layout.
if any(s.mla and s.index_ratio > 1 for s in config.kv_cache_group_specs()):
from .dsa_indexer_kpool import Glm5NextDSABackend
return Glm5NextDSABackend(config)
from .dsa import DSAAttnBackend
return DSAAttnBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"m3_sparse",
BackendInfo(
supported_types=frozenset({AttnType.BSA}),
# One KV page == one 128-token sparse block: the top-k block ids ARE page
# indices and the block-base-row addressing needs page-aligned 128-row runs.
# Config-time resolution coerces any other page size here.
page_sizes=(128,),
),
)
def create_m3_sparse_backend(config: ModelConfig):
from .m3_sparse import M3SparseAttnBackend
return M3SparseAttnBackend(config)
@SUPPORTED_ATTENTION_BACKENDS.register(
"qsa_sparse",
BackendInfo(
supported_types=frozenset({AttnType.QSA}),
# 64-token pages: a 4-token compress group never straddles a page, so the
# compressed row of a group is page_base // 4 + block-in-page.
page_sizes=(64,),
uniform_extend_graph="device",
),
)
def create_qsa_sparse_backend(config: ModelConfig):
from .qsa_sparse import QSASparseAttnBackend
return QSASparseAttnBackend(config)
def attention_backend_info(name: str) -> BackendInfo:
return SUPPORTED_ATTENTION_BACKENDS.info(name)
def validate_attn_backend(backend: str, allow_auto: bool = True):
if backend != "auto":
parts = backend.split(",")
if len(parts) > 2:
from argparse import ArgumentTypeError
raise ArgumentTypeError(
f"At most two comma-separated attention backends are allowed "
f"(prefill,decode), got {backend!r}"
)
SUPPORTED_ATTENTION_BACKENDS.assert_supported(parts)
else:
assert allow_auto, "auto is not allowed here"
return backend
def create_attention_backend(
backend: str,
config: ModelConfig,
) -> BaseAttnBackend:
validate_attn_backend(backend, allow_auto=False)
if "," in backend:
p_backend, d_backend = backend.split(",", 1)
if p_backend != d_backend:
logger.info(f"Using hybrid attention backend: prefill={p_backend}, decode={d_backend}")
p_backend = create_attention_backend(p_backend, config)
d_backend = create_attention_backend(d_backend, config)
return HybridBackend(p_backend, d_backend)
backend = p_backend # both are the same, fall through to single backend
logger.warning(f"P/D attention backends are the same: {backend}, using single backend.")
return SUPPORTED_ATTENTION_BACKENDS[backend](config)
__all__ = [
"AttnType",
"BackendInfo",
"BaseAttnMetadata",
"BaseAttnBackend",
"AttentionSpec",
"attention_backend_info",
"create_attention_backend",
"SUPPORTED_ATTENTION_BACKENDS",
"validate_attn_backend",
]