mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): add the MTP proposer for the qwen3_5 family * test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints * feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers * feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack * docs(cli): document the speculative decoding flags * fix(spec): refuse MTP draft sources the draft cannot read before the target loads * docs(qwen4_exp): describe the verify path in the PLE docstrings * fix(spec): refuse a missing relative MTP draft path before the target loads * fix(moe): read MTP head experts on the host and require both stacked tensors * refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq * refactor(spec): let each MTP draft family check its own weight source * fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows * perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr * refactor(moe): let each family name its MTP expert tensors * refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load * fix(spec): refuse MTP with dummy weights * refactor(spec): move HiddenRelay under spec/mtp, its only user * refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens * refactor(models): note FeatureSink's graph binding, drop its runtime_checkable * fix(models): read quantized MTP fc projections under the quant config
216 lines
6.6 KiB
Python
216 lines
6.6 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Literal, Protocol
|
|
|
|
from freetoken.utils import Registry, init_logger
|
|
|
|
from .base import AttentionSpec, AttnType, BaseAttnBackend, BaseAttnMetadata, HybridBackend
|
|
|
|
if TYPE_CHECKING:
|
|
from freetoken.models import ModelConfig
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class BackendCreator(Protocol):
|
|
def __call__(self, config: ModelConfig) -> BaseAttnBackend: ...
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BackendInfo:
|
|
"""Declarative capability matrix entry for one backend. The engine's config-time
|
|
validation interprets the requirement flags through its own (monkeypatch-able)
|
|
availability probes; this stays pure data so registration never imports kernels."""
|
|
|
|
supported_types: frozenset[AttnType]
|
|
requires_flashinfer: bool = False
|
|
requires_sgl_kernel: bool = False
|
|
requires_sm100: bool = False
|
|
# Allowed page sizes (None -> any). Config-time resolution coerces to the last
|
|
# entry when the resolved page_size is not in the list.
|
|
page_sizes: tuple[int, ...] | None = None
|
|
# Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks).
|
|
# Non-consumers raise on a non-None spec instead of silently dropping it.
|
|
consumes_attn_spec: bool = False
|
|
# How the backend captures a graph of `rows_per_req` query rows per request: "device" keeps all metadata on device, "host_plan" plans on the host before each replay, "none" cannot.
|
|
uniform_extend_graph: Literal["none", "host_plan", "device"] = "none"
|
|
|
|
|
|
SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend")
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"trtllm",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.FULL}),
|
|
requires_flashinfer=True,
|
|
requires_sm100=True,
|
|
page_sizes=(16, 32, 64),
|
|
),
|
|
)
|
|
def create_trtllm_backend(config: ModelConfig):
|
|
from .trtllm import TensorRTLLMBackend
|
|
|
|
return TensorRTLLMBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"fi",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.FULL}),
|
|
requires_flashinfer=True,
|
|
uniform_extend_graph="host_plan",
|
|
),
|
|
)
|
|
def create_fi_backend(config: ModelConfig):
|
|
from .fi import FlashInferBackend
|
|
|
|
return FlashInferBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"fa",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.FULL}),
|
|
requires_sgl_kernel=True,
|
|
uniform_extend_graph="device",
|
|
),
|
|
)
|
|
def create_fa_backend(config: ModelConfig):
|
|
from .fa import FlashAttentionBackend
|
|
|
|
return FlashAttentionBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"triton",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.FULL, AttnType.SWA}),
|
|
consumes_attn_spec=True,
|
|
uniform_extend_graph="device",
|
|
),
|
|
)
|
|
def create_triton_backend(config: ModelConfig):
|
|
from .triton import TritonAttentionBackend
|
|
|
|
return TritonAttentionBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"dsv4_sparse",
|
|
BackendInfo(supported_types=frozenset({AttnType.DSV4})),
|
|
)
|
|
def create_dsv4_sparse_backend(config: ModelConfig):
|
|
from .dsv4_sparse import DSV4SparseAttnBackend
|
|
|
|
return DSV4SparseAttnBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"dsv41_sparse",
|
|
BackendInfo(supported_types=frozenset({AttnType.DSV41})),
|
|
)
|
|
def create_dsv41_sparse_backend(config: ModelConfig):
|
|
from .dsv41_sparse import DSV41SparseAttnBackend
|
|
|
|
return DSV41SparseAttnBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"dsa",
|
|
BackendInfo(supported_types=frozenset({AttnType.MLA, AttnType.DSA})),
|
|
)
|
|
def create_dsa_backend(config: ModelConfig):
|
|
# MLA with a grouped index (index_ratio > 1) is the kpool indexer layout.
|
|
if any(s.mla and s.index_ratio > 1 for s in config.kv_cache_group_specs()):
|
|
from .dsa_indexer_kpool import Glm5NextDSABackend
|
|
|
|
return Glm5NextDSABackend(config)
|
|
from .dsa import DSAAttnBackend
|
|
|
|
return DSAAttnBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"m3_sparse",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.BSA}),
|
|
# One KV page == one 128-token sparse block: the top-k block ids ARE page
|
|
# indices and the block-base-row addressing needs page-aligned 128-row runs.
|
|
# Config-time resolution coerces any other page size here.
|
|
page_sizes=(128,),
|
|
),
|
|
)
|
|
def create_m3_sparse_backend(config: ModelConfig):
|
|
from .m3_sparse import M3SparseAttnBackend
|
|
|
|
return M3SparseAttnBackend(config)
|
|
|
|
|
|
@SUPPORTED_ATTENTION_BACKENDS.register(
|
|
"qsa_sparse",
|
|
BackendInfo(
|
|
supported_types=frozenset({AttnType.QSA}),
|
|
# 64-token pages: a 4-token compress group never straddles a page, so the
|
|
# compressed row of a group is page_base // 4 + block-in-page.
|
|
page_sizes=(64,),
|
|
uniform_extend_graph="device",
|
|
),
|
|
)
|
|
def create_qsa_sparse_backend(config: ModelConfig):
|
|
from .qsa_sparse import QSASparseAttnBackend
|
|
|
|
return QSASparseAttnBackend(config)
|
|
|
|
|
|
def attention_backend_info(name: str) -> BackendInfo:
|
|
return SUPPORTED_ATTENTION_BACKENDS.info(name)
|
|
|
|
|
|
def validate_attn_backend(backend: str, allow_auto: bool = True):
|
|
if backend != "auto":
|
|
parts = backend.split(",")
|
|
if len(parts) > 2:
|
|
from argparse import ArgumentTypeError
|
|
|
|
raise ArgumentTypeError(
|
|
f"At most two comma-separated attention backends are allowed "
|
|
f"(prefill,decode), got {backend!r}"
|
|
)
|
|
SUPPORTED_ATTENTION_BACKENDS.assert_supported(parts)
|
|
else:
|
|
assert allow_auto, "auto is not allowed here"
|
|
return backend
|
|
|
|
|
|
def create_attention_backend(
|
|
backend: str,
|
|
config: ModelConfig,
|
|
) -> BaseAttnBackend:
|
|
validate_attn_backend(backend, allow_auto=False)
|
|
if "," in backend:
|
|
p_backend, d_backend = backend.split(",", 1)
|
|
if p_backend != d_backend:
|
|
logger.info(f"Using hybrid attention backend: prefill={p_backend}, decode={d_backend}")
|
|
p_backend = create_attention_backend(p_backend, config)
|
|
d_backend = create_attention_backend(d_backend, config)
|
|
return HybridBackend(p_backend, d_backend)
|
|
backend = p_backend # both are the same, fall through to single backend
|
|
logger.warning(f"P/D attention backends are the same: {backend}, using single backend.")
|
|
|
|
return SUPPORTED_ATTENTION_BACKENDS[backend](config)
|
|
|
|
|
|
__all__ = [
|
|
"AttnType",
|
|
"BackendInfo",
|
|
"BaseAttnMetadata",
|
|
"BaseAttnBackend",
|
|
"AttentionSpec",
|
|
"attention_backend_info",
|
|
"create_attention_backend",
|
|
"SUPPORTED_ATTENTION_BACKENDS",
|
|
"validate_attn_backend",
|
|
]
|