mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-22 17:53:30 +08:00
* feat(spec): add the MTP proposer for the qwen3_5 family * test(models): read the qwen3_5 MTP head of bf16 and block-fp8 checkpoints * feat(spec): verify qwen4_exp targets on qsa_sparse graphs with PLE verify handlers * feat(spec): add the qwen4_exp MTP draft from the checkpoint or a Strata pack * docs(cli): document the speculative decoding flags * fix(spec): refuse MTP draft sources the draft cannot read before the target loads * docs(qwen4_exp): describe the verify path in the PLE docstrings * fix(spec): refuse a missing relative MTP draft path before the target loads * fix(moe): read MTP head experts on the host and require both stacked tensors * refactor(spec): tell the final prefill chunk from prompt_len instead of the scheduler's ChunkedReq * refactor(spec): let each MTP draft family check its own weight source * fix(qwen4_exp): size the disk PLE eager staging for decode and verify rows * perf(kernel): read the KV-index gather's chunk count from the grid, not a constexpr * refactor(moe): let each family name its MTP expert tensors * refactor(spec): force the MTP draft's MoE resident in the proposer and check it after load * fix(spec): refuse MTP with dummy weights * refactor(spec): move HiddenRelay under spec/mtp, its only user * refactor(spec): drop MTPDraft.config, fold its embed hooks into embed_tokens * refactor(models): note FeatureSink's graph binding, drop its runtime_checkable * fix(models): read quantized MTP fc projections under the quant config
223 lines
9.9 KiB
Python
223 lines
9.9 KiB
Python
"""Expert pieces: the checkpoint-side stream ``build_expert_banks`` packs into host banks.
|
|
|
|
A piece is ``(bank_layer, e0, e1, {role: tensor[e1 - e0, ...]})``: rows ``e0:e1`` of one MoE
|
|
layer's experts, in the checkpoint's own form (per-projection ``gate`` / ``up`` / ``down`` with
|
|
their ``_scale`` / ``_global`` companions, or an already fused ``gate_up``). The expert kernel's
|
|
``pack`` turns pieces into its bank layout, so readers never know the layout.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Callable, Collection, Iterable, Iterator
|
|
|
|
import torch
|
|
from freetoken.layers.quantization import QuantKind
|
|
from freetoken.models.register import _load_attr, get_model_spec
|
|
|
|
Piece = tuple[int, int, int, dict[str, torch.Tensor]]
|
|
|
|
|
|
def num_moe_layers(config) -> int:
|
|
value = getattr(config, "num_moe_layers", None)
|
|
if value is not None:
|
|
return int(value)
|
|
return int(config.num_layers) - int(getattr(config, "first_k_dense_replace", 0) or 0)
|
|
|
|
|
|
def bank_layer_of(config, layer: int) -> int | None:
|
|
"""MoE-layer index of checkpoint layer ``layer``; None for the leading dense layers."""
|
|
bank_layer = layer - int(getattr(config, "first_k_dense_replace", 0) or 0)
|
|
return bank_layer if 0 <= bank_layer < num_moe_layers(config) else None
|
|
|
|
|
|
def _model_hook(spec, name: str):
|
|
try:
|
|
return _load_attr(spec.module, name)
|
|
except AttributeError:
|
|
return None
|
|
|
|
|
|
def iter_expert_pieces(
|
|
model_path: str,
|
|
config,
|
|
kind: QuantKind,
|
|
*,
|
|
layers: Collection[int] | None = None,
|
|
parallel: bool = False,
|
|
workers: int = 8,
|
|
chunk: int = 8 << 20,
|
|
) -> Iterator[Piece]:
|
|
"""The pieces of ``model_path``'s routed experts, stored as ``kind``.
|
|
|
|
A family may ship its own ``iter_expert_pieces(model_path, config, kind, *, layers, parallel,
|
|
workers, chunk)`` for the storage forms only it uses (it returns None for the rest); otherwise bf16
|
|
experts come from the family's stacked ``iter_weights`` and NVFP4 experts from its
|
|
``nvfp4_expert_spec``. The reader is resolved here, before any bank is allocated, so a
|
|
missing parallel reader raises ``NotImplementedError`` while a serial fallback is still cheap.
|
|
|
|
``layers`` (bank layer ids, default all) limits the pieces to those layers, for experts of several
|
|
formats: a reader skips the other layers' tensors and checks only these layers for completeness.
|
|
"""
|
|
spec = get_model_spec(config.architectures[0])
|
|
hook = _model_hook(spec, "iter_expert_pieces")
|
|
if hook is not None:
|
|
pieces = hook(model_path, config, kind, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
|
|
if pieces is not None:
|
|
return pieces
|
|
if kind is QuantKind.NONE:
|
|
return _bf16_pieces(model_path, config, spec, layers=layers, parallel=parallel, workers=workers, chunk=chunk)
|
|
if kind is QuantKind.NVFP4:
|
|
spec_hook = _model_hook(spec, "nvfp4_expert_spec")
|
|
if spec_hook is None:
|
|
raise NotImplementedError(f"{spec.module} provides no nvfp4_expert_spec")
|
|
from freetoken.models.nvfp4_banks import iter_nvfp4_expert_pieces
|
|
|
|
return iter_nvfp4_expert_pieces(
|
|
model_path, config, spec_hook(model_path, config), layers=layers, parallel=parallel, workers=workers, chunk=chunk
|
|
)
|
|
raise NotImplementedError(f"{spec.module} provides no expert reader for {kind!r} experts")
|
|
|
|
|
|
def packed_expert_source_info(key: str) -> tuple[int, str] | None:
|
|
"""``model.layers.N....experts.{gate_up_proj,down_proj}`` -> (N, leaf); None otherwise."""
|
|
parts = key.split(".")
|
|
if len(parts) < 5 or parts[0] != "model" or parts[1] != "layers":
|
|
return None
|
|
if parts[-2] != "experts" or parts[-1] not in {"gate_up_proj", "down_proj"}:
|
|
return None
|
|
try:
|
|
return int(parts[2]), parts[-1]
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _source_layer(key: str) -> int | None:
|
|
"""Checkpoint layer of a ``model.layers.N....`` key."""
|
|
parts = key.split(".")
|
|
return int(parts[2]) if len(parts) > 2 and parts[:2] == ["model", "layers"] and parts[2].isdigit() else None
|
|
|
|
|
|
def stacked_expert_pieces(tensors: Iterable[tuple[str, torch.Tensor]], config, *, layers: Collection[int] | None = None) -> Iterator[Piece]:
|
|
"""Whole-layer ``experts.gate_up_proj`` / ``experts.down_proj`` tensors -> one piece per layer, for ``layers`` (bank layer ids, default all)."""
|
|
keep = None if layers is None else set(layers)
|
|
pending: dict[int, dict[str, torch.Tensor]] = {}
|
|
num_experts = config.num_experts
|
|
for name, tensor in tensors:
|
|
info = packed_expert_source_info(name)
|
|
if info is None:
|
|
# another format's layer stores its experts in another form
|
|
layer = _source_layer(name)
|
|
if keep is not None and layer is not None and bank_layer_of(config, layer) not in keep:
|
|
continue
|
|
raise ValueError(f"Unexpected expert weight key: {name}")
|
|
layer, packed_name = info
|
|
bank_layer = bank_layer_of(config, layer)
|
|
if bank_layer is None:
|
|
raise ValueError(f"Unexpected MoE expert layer {layer}; expected a routed-expert layer")
|
|
if keep is not None and bank_layer not in keep:
|
|
continue
|
|
if tensor.size(0) != num_experts:
|
|
raise ValueError(f"Unexpected {packed_name} expert count {tensor.size(0)}; expected {num_experts}")
|
|
piece = pending.setdefault(bank_layer, {})
|
|
piece["gate_up" if packed_name == "gate_up_proj" else "down"] = tensor
|
|
if len(piece) == 2:
|
|
del pending[bank_layer]
|
|
yield bank_layer, 0, num_experts, piece
|
|
if pending:
|
|
raise ValueError(f"Missing MoE expert source layers: {sorted(pending)}")
|
|
|
|
|
|
def _bf16_pieces(model_path: str, config, spec, *, layers: Collection[int] | None, parallel: bool, workers: int, chunk: int) -> Iterator[Piece]:
|
|
device = torch.device("cpu")
|
|
if parallel:
|
|
iter_weights = _model_hook(spec, "iter_weights_parallel")
|
|
if iter_weights is None:
|
|
raise NotImplementedError(f"{spec.module} provides no iter_weights_parallel")
|
|
tensors = iter_weights(
|
|
model_path, device, include_moe_experts=True, include_non_moe=False, workers=workers, chunk=chunk
|
|
)
|
|
else:
|
|
iter_weights = _load_attr(spec.module, spec.iter_weights)
|
|
tensors = iter_weights(model_path, device, include_moe_experts=True, include_non_moe=False)
|
|
return stacked_expert_pieces(tensors, config, layers=layers)
|
|
|
|
|
|
def per_expert_pieces(
|
|
tensors: Iterable[tuple[str, torch.Tensor]],
|
|
locate: Callable[[str], tuple[int, int, str] | None],
|
|
*,
|
|
tensors_per_expert: int,
|
|
) -> Iterator[Piece]:
|
|
"""Group per-expert tensors into one piece per expert, in whatever order they arrive.
|
|
|
|
``locate(name)`` -> ``(bank_layer, expert, role)`` or None to skip; a role tensor is one
|
|
expert's ``[...]`` row and becomes the piece's ``[1, ...]`` entry. Globals and activation scales may be scalars.
|
|
"""
|
|
pending: dict[tuple[int, int], dict[str, torch.Tensor]] = {}
|
|
for name, tensor in tensors:
|
|
where = locate(name)
|
|
if where is None:
|
|
continue
|
|
bank_layer, expert, role = where
|
|
piece = pending.setdefault((bank_layer, expert), {})
|
|
piece[role] = tensor.reshape(1, -1) if role.endswith(("_global", "_input")) else tensor.unsqueeze(0)
|
|
if len(piece) == tensors_per_expert:
|
|
del pending[(bank_layer, expert)]
|
|
yield bank_layer, expert, expert + 1, piece
|
|
if pending:
|
|
missing = sorted(pending)[:8]
|
|
raise ValueError(f"Incomplete expert tensors in checkpoint (layer, expert): {missing}")
|
|
|
|
|
|
def iter_extra_layer_expert_pieces(model_path: str, prefix: str, locate: Callable[[str], tuple[int | None, str] | None]) -> Iterator[Piece]:
|
|
"""The routed experts of one MoE layer outside the model's stack (an MTP head), as pieces of bank layer 0.
|
|
|
|
The family names the tensors: ``locate(name)`` -> ``(expert, role)`` of a per-expert tensor, ``(None, "gate_up" | "down")`` of a stacked one, or None. A name under ``prefix`` that ``locate`` does not read is refused.
|
|
"""
|
|
import safetensors
|
|
|
|
from freetoken.models.loader import iter_weight_files
|
|
|
|
found: dict[str, tuple[str, int | None, str]] = {}
|
|
unknown: list[str] = []
|
|
for file in iter_weight_files(model_path):
|
|
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
|
|
for name in f.keys():
|
|
if (where := locate(name)) is not None:
|
|
found[name] = (file, *where)
|
|
elif name.startswith(prefix):
|
|
unknown.append(name)
|
|
if unknown:
|
|
raise ValueError(f"{model_path}: the experts under {prefix}* use a format this reader does not support, e.g. {sorted(unknown)[:3]}")
|
|
if not found:
|
|
raise ValueError(f"{model_path} has no routed experts under {prefix}*")
|
|
stacked = {role: (file, name) for name, (file, expert, role) in found.items() if expert is None}
|
|
if stacked:
|
|
if missing := {"gate_up", "down"} - set(stacked):
|
|
raise ValueError(f"{model_path}: stacked experts under {prefix}* lack {sorted(missing)}")
|
|
piece = {}
|
|
for role, (file, name) in stacked.items():
|
|
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
|
|
piece[role] = f.get_tensor(name)
|
|
return iter([(0, 0, piece["down"].shape[0], piece)])
|
|
|
|
def tensors():
|
|
for file in dict.fromkeys(file for file, _, _ in found.values()):
|
|
with safetensors.safe_open(file, framework="pt", device="cpu") as f:
|
|
yield from ((name, f.get_tensor(name)) for name, where in found.items() if where[0] == file)
|
|
|
|
roles = {role for _, _, role in found.values()}
|
|
return per_expert_pieces(tensors(), lambda name: (0, *found[name][1:]), tensors_per_expert=len(roles))
|
|
|
|
|
|
__all__ = [
|
|
"Piece",
|
|
"bank_layer_of",
|
|
"iter_expert_pieces",
|
|
"iter_extra_layer_expert_pieces",
|
|
"num_moe_layers",
|
|
"packed_expert_source_info",
|
|
"per_expert_pieces",
|
|
"stacked_expert_pieces",
|
|
]
|