Files
Xiaoze Fan 189c8ad0c3 feat(moe): serve NVFP4 and MXFP4 experts fused through the expert banks (#601)
* feat(moe): serve NVFP4 and MXFP4 experts fused through the expert banks

Resident experts skip the state dict and load through the offload expert-bank loader (HF pieces or FTW banks) into per-layer GPU banks, so fused serves bf16, fp8_block, NVFP4 and MXFP4 alike. Unified-memory GPUs auto-select fused for these formats and ignore offload-only flags with a warning.

BREAKING CHANGE: ft checkpoint always writes MoE experts as banks; an FTW converted with --moe-backend fused or triton (dense experts) no longer loads and must be reconverted.

* docs(moe): name only GB10 / DGX Spark for unified memory
2026-10-04 13:51:55 -07:00

42 lines
1.8 KiB
Python

"""bf16 experts: the fused Triton MoE over gate_up / down banks."""
from __future__ import annotations
import torch
from ..registry import LayerKind, register_method
from ..scheme import QuantKind
from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
class FusedMoEKernel(MoEKernel):
name = "fused"
cpu_format = "bf16"
def unusable_reason(self, cfg: MoEConfig) -> str | None:
reason = gated_epilogue_reason(cfg)
return f"bf16 fused MoE: {reason}" if reason else None
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
i = cfg.local_intermediate
# resident experts compute in the model dtype; offload banks stay bf16, the FTW and CPU executor format
dt = cfg.dtype if cfg.strategy == "resident" else torch.bfloat16
return {"gate_up": BankSpec((2 * i, cfg.hidden), dt), "down": BankSpec((cfg.hidden, i), dt)}
def pack(self, pieces, cfg: MoEConfig, out):
out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
out["down"].copy_(pieces["down"])
return {}
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl
# the resident layer runs the in-place prefill kernel for both phases
impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl
return impl(x, view.tensors["gate_up"], view.tensors["down"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))
@register_method(QuantKind.NONE, LayerKind.MOE)
class UnquantizedMoEMethod(MoEMethod):
candidates = (FusedMoEKernel,)