mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(moe): serve NVFP4 and MXFP4 experts fused through the expert banks Resident experts skip the state dict and load through the offload expert-bank loader (HF pieces or FTW banks) into per-layer GPU banks, so fused serves bf16, fp8_block, NVFP4 and MXFP4 alike. Unified-memory GPUs auto-select fused for these formats and ignore offload-only flags with a warning. BREAKING CHANGE: ft checkpoint always writes MoE experts as banks; an FTW converted with --moe-backend fused or triton (dense experts) no longer loads and must be reconverted. * docs(moe): name only GB10 / DGX Spark for unified memory
42 lines
1.8 KiB
Python
42 lines
1.8 KiB
Python
"""bf16 experts: the fused Triton MoE over gate_up / down banks."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import QuantKind
|
|
from .base import BankSpec, ExpertView, fused_piece, gated_epilogue_reason, is_resident, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
|
|
|
|
|
|
class FusedMoEKernel(MoEKernel):
|
|
name = "fused"
|
|
cpu_format = "bf16"
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
reason = gated_epilogue_reason(cfg)
|
|
return f"bf16 fused MoE: {reason}" if reason else None
|
|
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
|
|
i = cfg.local_intermediate
|
|
# resident experts compute in the model dtype; offload banks stay bf16, the FTW and CPU executor format
|
|
dt = cfg.dtype if cfg.strategy == "resident" else torch.bfloat16
|
|
return {"gate_up": BankSpec((2 * i, cfg.hidden), dt), "down": BankSpec((cfg.hidden, i), dt)}
|
|
|
|
def pack(self, pieces, cfg: MoEConfig, out):
|
|
out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
|
|
out["down"].copy_(pieces["down"])
|
|
return {}
|
|
|
|
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
|
|
from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl
|
|
|
|
# the resident layer runs the in-place prefill kernel for both phases
|
|
impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl
|
|
return impl(x, view.tensors["gate_up"], view.tensors["down"], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))
|
|
|
|
|
|
@register_method(QuantKind.NONE, LayerKind.MOE)
|
|
class UnquantizedMoEMethod(MoEMethod):
|
|
candidates = (FusedMoEKernel,)
|