mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(gguf): serve gpt-oss gguf checkpoints on gpt_oss * fix(gguf): refuse missing gpt-oss bias tensors, honor swa pattern, scale nvfp4 before bias * fix(gguf): honor gpt-oss yarn truncate, gate the bf16 float cast, scale nvfp4 before the down bias
219 lines
8.5 KiB
Python
219 lines
8.5 KiB
Python
"""The MoE side: per-layer config, bank descriptions, the kernel ABC and the Method base."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import Any, ClassVar
|
|
|
|
import torch
|
|
|
|
from ..method import QuantMethod
|
|
from ..scheme import QuantScheme
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MoEConfig:
|
|
"""Everything an expert kernel may look at to accept or decline a layer and to size its banks."""
|
|
|
|
num_experts: int
|
|
hidden: int
|
|
intermediate: int
|
|
top_k: int
|
|
tp_rank: int = 0
|
|
tp_size: int = 1
|
|
scheme: QuantScheme | None = None
|
|
activation: str = "silu"
|
|
alpha: float = 1.0
|
|
beta: float = 0.0
|
|
limit: float | None = None
|
|
interleaved: bool = False
|
|
# dynamic fp8 activation quant block of a W4A8 expert kernel, set by the quant dialect; None = no activation quant
|
|
act_block: int | None = None
|
|
has_bias: bool = False
|
|
apply_router_weight_on_input: bool = False
|
|
strategy: str = "resident"
|
|
decode_target: str = "gpu"
|
|
dtype: torch.dtype = torch.bfloat16
|
|
|
|
@classmethod
|
|
def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "MoEConfig":
|
|
return cls(
|
|
num_experts=layer.num_experts,
|
|
hidden=layer.hidden_size,
|
|
intermediate=layer.intermediate_size,
|
|
top_k=layer.top_k,
|
|
tp_rank=layer.tp_rank,
|
|
tp_size=layer.tp_size,
|
|
scheme=scheme,
|
|
activation=layer.activation,
|
|
alpha=float(layer.alpha),
|
|
beta=float(layer.beta),
|
|
limit=layer.limit,
|
|
interleaved=bool(layer.interleaved),
|
|
has_bias=bool(layer.has_bias),
|
|
apply_router_weight_on_input=bool(layer.apply_router_weight_on_input),
|
|
strategy=layer.strategy,
|
|
decode_target=layer.decode_target,
|
|
dtype=torch.get_default_dtype(),
|
|
)
|
|
|
|
@property
|
|
def local_intermediate(self) -> int:
|
|
return self.intermediate // self.tp_size
|
|
|
|
@property
|
|
def plain_silu(self) -> bool:
|
|
return (self.activation, self.alpha, self.beta, self.limit) == ("silu", 1.0, 0.0, None)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BankSpec:
|
|
"""One bank of an expert layout, per expert (the E dim is prepended by the allocator)."""
|
|
|
|
shape: tuple[int, ...]
|
|
dtype: torch.dtype
|
|
# GPU-resident per-expert vector (marlin / b12x alphas) instead of a host bank
|
|
resident: bool = False
|
|
|
|
|
|
@dataclass
|
|
class ExpertView:
|
|
"""The weights an apply() call reads: layout roles -> [E, ...] (resident, materialized layer) or [S, ...] (slot cache)."""
|
|
|
|
tensors: dict[str, torch.Tensor]
|
|
slots: torch.Tensor | None = None
|
|
n: int | None = None
|
|
alphas: tuple[torch.Tensor, torch.Tensor] | None = None
|
|
|
|
|
|
def fused_piece(pieces: dict[str, torch.Tensor], role: str) -> torch.Tensor:
|
|
"""A gate_up-family piece, concatenating separate gate / up pieces along the row axis."""
|
|
if role in pieces:
|
|
return pieces[role]
|
|
suffix = role[len("gate_up"):]
|
|
return torch.cat([pieces["gate" + suffix], pieces["up" + suffix]], dim=1)
|
|
|
|
|
|
def global_rows(piece: torch.Tensor, rows: int) -> torch.Tensor:
|
|
"""Per-expert global scale as one fp32 value per output row (a scalar per expert is broadcast)."""
|
|
flat = piece.reshape(piece.shape[0], -1).to(torch.float32)
|
|
if flat.shape[1] == rows:
|
|
return flat
|
|
assert flat.shape[1] == 1, (flat.shape, rows)
|
|
return flat.expand(-1, rows)
|
|
|
|
|
|
def fused_global(pieces: dict[str, torch.Tensor], half_rows: int) -> torch.Tensor:
|
|
if "gate_up_global" in pieces:
|
|
return global_rows(pieces["gate_up_global"], 2 * half_rows)
|
|
return torch.cat([global_rows(pieces["gate_global"], half_rows), global_rows(pieces["up_global"], half_rows)], dim=1)
|
|
|
|
|
|
def limit_or_inf(layer: Any) -> float:
|
|
limit = getattr(layer, "limit", None)
|
|
return float("inf") if limit is None else float(limit)
|
|
|
|
|
|
def gated_epilogue_reason(cfg: "MoEConfig") -> str | None:
|
|
"""Why ``gated_act_and_mul`` cannot run ``cfg``'s activation quadruple, or None; the plain kinds have no alpha / limit and nothing takes beta or interleaved rows."""
|
|
from freetoken.layers import GATED_ACTIVATIONS
|
|
|
|
if cfg.activation not in GATED_ACTIVATIONS:
|
|
return f"no {cfg.activation!r} epilogue"
|
|
if cfg.interleaved:
|
|
return "the epilogue reads uninterleaved [gate; up] halves"
|
|
if cfg.beta != 0.0:
|
|
return f"the epilogue has no beta (got {cfg.beta})"
|
|
if cfg.activation in ("silu", "gelu", "gelu_tanh") and (cfg.alpha != 1.0 or cfg.limit is not None):
|
|
return f"{cfg.activation} takes no alpha / limit (got alpha={cfg.alpha}, limit={cfg.limit})"
|
|
return None
|
|
|
|
|
|
def is_resident(layer: Any) -> bool:
|
|
return layer.quant_method.cfg.strategy == "resident"
|
|
|
|
|
|
class MoEKernel(ABC):
|
|
"""One backend for one expert kind: bank layout, packing from checkpoint pieces, and the fused forward."""
|
|
|
|
name: ClassVar[str]
|
|
cpu_format: ClassVar[str | None] = None
|
|
max_slots: ClassVar[int | None] = None
|
|
# Whether ``apply`` treats a slot of -1 as an inactive route (zero output, no read of any slot).
|
|
# Hybrid decode hands the GPU a route split with such holes; kernels without this get the holes
|
|
# pointed at slot 0 with weight 0 (slot storage is zero-filled at allocation, so that is exact).
|
|
supports_inactive_slots: ClassVar[bool] = False
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
return None
|
|
|
|
def worth_it(self, cfg: MoEConfig) -> bool:
|
|
return True
|
|
|
|
def cpu_options(self, cfg: MoEConfig) -> dict[str, int]:
|
|
"""Integer options of ``cpu_format`` for these experts."""
|
|
return {}
|
|
|
|
def slot_limit(self, cfg: MoEConfig) -> int | None:
|
|
"""Most GPU cache slots the kernel can address for ``cfg``; None for no limit."""
|
|
return self.max_slots
|
|
|
|
def _common_reject(self, cfg: MoEConfig, *, tp_ok: bool, cpu_ok: bool, plain_silu_only: bool, bias_ok: bool = False) -> str | None:
|
|
if not tp_ok and cfg.tp_size > 1:
|
|
return "TP > 1 is not supported for this expert format"
|
|
if not cpu_ok and cfg.decode_target != "gpu":
|
|
return "has no CPU executor format; decode must run on the GPU"
|
|
if cfg.has_bias and not bias_ok:
|
|
return "kernel has no bias epilogue"
|
|
if plain_silu_only and not cfg.plain_silu:
|
|
return f"kernel is plain-silu only, experts use ({cfg.activation}, alpha={cfg.alpha}, beta={cfg.beta}, limit={cfg.limit})"
|
|
return None
|
|
|
|
@abstractmethod
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]: ...
|
|
|
|
@abstractmethod
|
|
def pack(self, pieces: dict[str, torch.Tensor], cfg: MoEConfig, out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
|
|
"""Write one batch of expert pieces into ``out`` rows; returns the GPU-resident per-expert values, if any."""
|
|
|
|
@abstractmethod
|
|
def apply(self, layer: Any, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, is_prefill: bool) -> torch.Tensor: ...
|
|
|
|
|
|
class MoEMethod(QuantMethod):
|
|
"""Declares resident experts for its kind; layout, pack and apply go to the kernel."""
|
|
|
|
def layout(self) -> dict[str, BankSpec]:
|
|
return self.kernel.layout(self.cfg)
|
|
|
|
def pack(self, pieces: dict[str, torch.Tensor], out: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
|
|
return self.kernel.pack(pieces, self.cfg, out)
|
|
|
|
def slot_limit(self) -> int | None:
|
|
return self.kernel.slot_limit(self.cfg)
|
|
|
|
def apply(self, x: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, view: ExpertView, *, layer: Any, is_prefill: bool) -> torch.Tensor:
|
|
return self.kernel.apply(layer, x, topk_weights, topk_ids, view, is_prefill=is_prefill)
|
|
|
|
@property
|
|
def cpu_format(self) -> str | None:
|
|
return self.kernel.cpu_format
|
|
|
|
def cpu_spec(self) -> tuple[str | None, dict[str, int]]:
|
|
"""The CPU executor format and options of this layer's experts."""
|
|
return self.cpu_format, self.kernel.cpu_options(self.cfg)
|
|
|
|
def create_weights(self, layer: Any) -> None:
|
|
# resident experts stay out of the state dict; attach_resident_banks fills these after the dense weights load
|
|
layer._expert_banks = None
|
|
layer._expert_alphas = None
|
|
|
|
def finalize(self, layer: Any) -> None:
|
|
pass
|
|
|
|
def resident_view(self, layer: Any) -> ExpertView:
|
|
"""Layout roles -> the resident layer's tensors, for apply()."""
|
|
assert layer._expert_banks is not None, "resident expert banks were never loaded"
|
|
return ExpertView(layer._expert_banks, n=self.cfg.num_experts, alphas=layer._expert_alphas)
|