Files
FreeToken/python/freetoken/moe/expert_pieces.py
Xiaoze Fan dacac5f5fa feat(spec): add MTP drafts for the Qwen3.5 family and Qwen3.8-Flash-Next (#844)
* feat(spec): add the MTP proposer for the qwen3_5 family

* test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints

* feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers

* feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack

* docs(cli): document the speculative decoding flags

* fix(spec): refuse MTP draft sources the draft cannot read before the target loads

* docs(qwen4_exp): describe the verify path in the PLE docstrings

* fix(spec): refuse a missing relative MTP draft path before the target loads

* fix(moe): read MTP head experts on the host and require both stacked tensors

* refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq

* refactor(spec): let each MTP draft family check its own weight source

* fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows

* perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr

* refactor(moe): let each family name its MTP expert tensors

* refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load

* fix(spec): refuse MTP with dummy weights

* refactor(spec): move HiddenRelay under spec/mtp, its only user

* refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens

* refactor(models): note FeatureSink's graph binding, drop its runtime_checkable

* fix(models): read quantized MTP fc projections under the quant config
2026-11-02 11:37:09 -08:00

223 lines
9.9 KiB
Python

"""Expert pieces: the checkpoint-side stream ``build_expert_banks`` packs into host banks.
A piece is ``(bank_layer, e0, e1, {role: tensor[e1 - e0, ...]})``: rows ``e0:e1`` of one MoE
layer's experts, in the checkpoint's own form (per-projection ``gate`` / ``up`` / ``down`` with
their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). The expert kernel's
``pack`` turns pieces into its bank layout, so readers never know the layout.
"""
from __future__ import annotations
from typing import Callable, Collection, Iterable, Iterator
import torch
from freetoken.layers.quantization import QuantKind
from freetoken.models.register import _load_attr, get_model_spec
Piece = tuple[int, int, int, dict[str, torch.Tensor]]
def num_moe_layers(config) -> int:
value = getattr(config, "num_moe_layers", None)
if value is not None:
return int(value)
return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0) or 0)
def bank_layer_of(config, layer: int) -> int | None:
"""MoE-layer index of checkpoint layer ``layer``; None for the leading dense layers."""
bank_layer = layer - int(getattr(config, "first_k_dense_replace", 0) or 0)
return bank_layer if 0 <= bank_layer < num_moe_layers(config) else None
def _model_hook(spec, name: str):
try:
return _load_attr(spec.module, name)
except AttributeError:
return None
def iter_expert_pieces(
model_path: str,
config,
kind: QuantKind,
*,
layers: Collection[int] | None = None,
parallel: bool = False,
workers: int = 8,
chunk: int = 8 << 20,
) -> Iterator[Piece]:
"""The pieces of ``model_path``'s routed experts, stored as ``kind``.
A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, layers, parallel,
workers, chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16
experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its
``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so a
missing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.
``layers`` (bank layer ids, default all) limits the pieces to those layers, for experts of several
formats: a reader skips the other layers' tensors and checks only these layers for completeness.
"""
spec = get_model_spec(config.architectures[0])
hook = _model_hook(spec, "iter_expert_pieces")
if hook is not None:
pieces = hook(model_path, config, kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
if pieces is not None:
return pieces
if kind is QuantKind.NONE:
return _bf16_pieces(model_path, config, spec, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
if kind is QuantKind.NVFP4:
spec_hook = _model_hook(spec, "nvfp4_expert_spec")
if spec_hook is None:
raise NotImplementedError(f"{spec.module} provides no nvfp4_expert_spec")
from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces
return iter_nvfp4_expert_pieces(
model_path, config, spec_hook(model_path, config), layers=layers, parallel=parallel, workers=workers, chunk=chunk
)
raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")
def packed_expert_source_info(key: str) -> tuple[int, str] | None:
"""``model.layers.N....experts.{gate_up_proj,down_proj}`` -> (N, leaf); None otherwise."""
parts = key.split(".")
if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":
return None
if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:
return None
try:
return int(parts[2]), parts[-1]
except ValueError:
return None
def _source_layer(key: str) -> int | None:
"""Checkpoint layer of a ``model.layers.N....`` key."""
parts = key.split(".")
return int(parts[2]) if len(parts) > 2 and parts[:2] == ["model", "layers"] and parts[2].isdigit() else None
def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config, *, layers: Collection[int] | None = None) -> Iterator[Piece]:
"""Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer, for ``layers`` (bank layer ids, default all)."""
keep = None if layers is None else set(layers)
pending: dict[int, dict[str, torch.Tensor]] = {}
num_experts = config.num_experts
for name, tensor in tensors:
info = packed_expert_source_info(name)
if info is None:
# another format's layer stores its experts in another form
layer = _source_layer(name)
if keep is not None and layer is not None and bank_layer_of(config, layer) not in keep:
continue
raise ValueError(f"Unexpected expert weight key: {name}")
layer, packed_name = info
bank_layer = bank_layer_of(config, layer)
if bank_layer is None:
raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")
if keep is not None and bank_layer not in keep:
continue
if tensor.size(0) != num_experts:
raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")
piece = pending.setdefault(bank_layer, {})
piece["gate_up" if packed_name == "gate_up_proj" else "down"] = tensor
if len(piece) == 2:
del pending[bank_layer]
yield bank_layer, 0, num_experts, piece
if pending:
raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")
def _bf16_pieces(model_path: str, config, spec, *, layers: Collection[int] | None, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:
device = torch.device("cpu")
if parallel:
iter_weights = _model_hook(spec, "iter_weights_parallel")
if iter_weights is None:
raise NotImplementedError(f"{spec.module} provides no iter_weights_parallel")
tensors = iter_weights(
model_path, device, include_moe_experts=True, include_non_moe=False, workers=workers, chunk=chunk
)
else:
iter_weights = _load_attr(spec.module, spec.iter_weights)
tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)
return stacked_expert_pieces(tensors, config, layers=layers)
def per_expert_pieces(
tensors: Iterable[tuple[str, torch.Tensor]],
locate: Callable[[str], tuple[int, int, str] | None],
*,
tensors_per_expert: int,
) -> Iterator[Piece]:
"""Group per-expert tensors into one piece per expert, in whatever order they arrive.
``locate(name)`` -> ``(bank_layer, expert, role)`` or None to skip; a role tensor is one
expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals and activation scales may be scalars.
"""
pending: dict[tuple[int, int], dict[str, torch.Tensor]] = {}
for name, tensor in tensors:
where = locate(name)
if where is None:
continue
bank_layer, expert, role = where
piece = pending.setdefault((bank_layer, expert), {})
piece[role] = tensor.reshape(1, -1) if role.endswith(("_global", "_input")) else tensor.unsqueeze(0)
if len(piece) == tensors_per_expert:
del pending[(bank_layer, expert)]
yield bank_layer, expert, expert + 1, piece
if pending:
missing = sorted(pending)[:8]
raise ValueError(f"Incomplete expert tensors in checkpoint (layer, expert): {missing}")
def iter_extra_layer_expert_pieces(model_path: str, prefix: str, locate: Callable[[str], tuple[int | None, str] | None]) -> Iterator[Piece]:
"""The routed experts of one MoE layer outside the model's stack (an MTP head), as pieces of bank layer 0.
The family names the tensors: ``locate(name)`` -> ``(expert, role)`` of a per-expert tensor, ``(None, "gate_up" | "down")`` of a stacked one, or None. A name under ``prefix`` that ``locate`` does not read is refused.
"""
import safetensors
from freetoken.models.loader import iter_weight_files
found: dict[str, tuple[str, int | None, str]] = {}
unknown: list[str] = []
for file in iter_weight_files(model_path):
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
for name in f.keys():
if (where := locate(name)) is not None:
found[name] = (file, *where)
elif name.startswith(prefix):
unknown.append(name)
if unknown:
raise ValueError(f"{model_path}: the experts under {prefix}* use a format this reader does not support, e.g. {sorted(unknown)[:3]}")
if not found:
raise ValueError(f"{model_path} has no routed experts under {prefix}*")
stacked = {role: (file, name) for name, (file, expert, role) in found.items() if expert is None}
if stacked:
if missing := {"gate_up", "down"} - set(stacked):
raise ValueError(f"{model_path}: stacked experts under {prefix}* lack {sorted(missing)}")
piece = {}
for role, (file, name) in stacked.items():
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
piece[role] = f.get_tensor(name)
return iter([(0, 0, piece["down"].shape[0], piece)])
def tensors():
for file in dict.fromkeys(file for file, _, _ in found.values()):
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
yield from ((name, f.get_tensor(name)) for name, where in found.items() if where[0] == file)
roles = {role for _, _, role in found.values()}
return per_expert_pieces(tensors(), lambda name: (0, *found[name][1:]), tensors_per_expert=len(roles))
__all__ = [
"Piece",
"bank_layer_of",
"iter_expert_pieces",
"iter_extra_layer_expert_pieces",
"num_moe_layers",
"packed_expert_source_info",
"per_expert_pieces",
"stacked_expert_pieces",
]