mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
debug_once / info_once / warning_once log on rank 0, once per distinct message and args. select_kernel, which logged its choice once per layer, and the hand-rolled log-once guards now use them.
624 lines
28 KiB
Python
624 lines
28 KiB
Python
"""NVFP4 experts: served resident or from the offload cache through the Triton inline-dequant kernels, vLLM's Marlin or flashinfer's b12x.
|
|
|
|
FreeToken stores MoE experts as ModelOpt NVFP4 (packed e2m1 codes + fp8-e4m3 per-16
|
|
block scales + per-tensor global scale) in pinned host banks and gathers the routed
|
|
experts into a GPU slot cache. Rules every kernel here follows:
|
|
|
|
* A kernel owns ``{load-time pack, forward}`` as a unit. Prefill and decode consume
|
|
the *same* bank layout -- there is never a second copy of the experts.
|
|
* Every pack keeps the expert dimension outermost with per-expert blocks contiguous
|
|
and byte-identical in size to the native layout, so the banks are repacked *in
|
|
place* and the offload cache's slot gather (``copy_missing``) works unchanged.
|
|
* The per-tensor global scales never enter the offload banks: they are tiny
|
|
``[L*E]`` vectors kept resident on the GPU and gathered per forward call
|
|
(device-side, CUDA-graph safe).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import torch
|
|
|
|
from freetoken.kernel import backend
|
|
from freetoken.utils import init_logger
|
|
|
|
from ..registry import LayerKind, register_method
|
|
from ..scheme import NVFP4_GROUP as GROUP, QuantKind
|
|
from .base import BankSpec, ExpertView, fused_global, fused_piece, gated_epilogue_reason, global_rows, limit_or_inf, MoEConfig, MoEKernel, MoEMethod
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
FP8 = torch.float8_e4m3fn
|
|
MARLIN_MAX_SLOTS = 992
|
|
B12X_MIN_INTERMEDIATE = 1024
|
|
|
|
|
|
class TritonNvfp4MoEKernel(MoEKernel):
|
|
"""FreeToken's inline-dequant kernels over the native ModelOpt rows."""
|
|
|
|
name = "triton"
|
|
cpu_format = "nvfp4"
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
reason = self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)
|
|
if reason:
|
|
return reason
|
|
reason = gated_epilogue_reason(cfg)
|
|
return f"triton nvfp4 MoE kernel: {reason}" if reason else None
|
|
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
|
|
i, h = cfg.intermediate, cfg.hidden
|
|
return {
|
|
"gate_up": BankSpec((2 * i, h // 2), torch.uint8),
|
|
"gate_up_scale": BankSpec((2 * i, h // GROUP), FP8),
|
|
"gate_up_global": BankSpec((2 * i,), torch.float32),
|
|
"down": BankSpec((h, i // 2), torch.uint8),
|
|
"down_scale": BankSpec((h, i // GROUP), FP8),
|
|
"down_global": BankSpec((h,), torch.float32),
|
|
}
|
|
|
|
def pack(self, pieces, cfg: MoEConfig, out):
|
|
out["gate_up"].copy_(fused_piece(pieces, "gate_up"))
|
|
out["gate_up_scale"].copy_(fused_piece(pieces, "gate_up_scale"))
|
|
out["gate_up_global"].copy_(fused_global(pieces, cfg.intermediate))
|
|
out["down"].copy_(pieces["down"])
|
|
out["down_scale"].copy_(pieces["down_scale"])
|
|
out["down_global"].copy_(global_rows(pieces["down_global"], cfg.hidden))
|
|
return {}
|
|
|
|
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
|
|
from freetoken.moe.fused_nvfp4 import fused_experts_decode_nvfp4_marlin, fused_experts_nvfp4
|
|
|
|
t = view.tensors
|
|
banks = (t["gate_up"], t["gate_up_scale"], t["gate_up_global"], t["down"], t["down_scale"], t["down_global"])
|
|
alpha, limit = float(layer.alpha), limit_or_inf(layer)
|
|
if is_prefill:
|
|
return fused_experts_nvfp4(x, *banks, topk_weights, topk_ids, view.n, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
|
|
return fused_experts_decode_nvfp4_marlin(x, *banks, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, alpha, limit)
|
|
|
|
|
|
class W4a4Nvfp4MoEKernel(TritonNvfp4MoEKernel):
|
|
"""The Triton kernels' banks and decode, with the prefill GEMMs on NVFP4 activations (W4A4) where the GPU and the batch favour it.
|
|
|
|
Needs a checkpoint whose experts declare static activation scales; the GPU-resident alphas carry them per expert (gate_up_alpha: the larger of gate's and up's input_scale).
|
|
"""
|
|
|
|
name = "w4a4"
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
reason = super().unusable_reason(cfg)
|
|
if reason:
|
|
return reason
|
|
if cfg.scheme is None or not cfg.scheme.has("input_scale"):
|
|
return "the experts declare no static activation scale (W4A16 checkpoint)"
|
|
if cfg.apply_router_weight_on_input:
|
|
return "the routing weight scales the input, which is quantized once per token"
|
|
return None
|
|
|
|
def worth_it(self, cfg: MoEConfig) -> bool:
|
|
from freetoken.kernel import nvfp4_moe
|
|
|
|
return not backend.is_rocm() and torch.cuda.is_available() and nvfp4_moe.w4a4_available(torch.cuda.current_device())
|
|
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
|
|
return {
|
|
**super().layout(cfg),
|
|
"gate_up_alpha": BankSpec((), torch.float32, resident=True),
|
|
"down_alpha": BankSpec((), torch.float32, resident=True),
|
|
}
|
|
|
|
def pack(self, pieces, cfg: MoEConfig, out):
|
|
super().pack(pieces, cfg, out)
|
|
if not all(f"{p}_input" in pieces for p in ("gate", "up", "down")):
|
|
raise ValueError("w4a4 nvfp4 MoE kernel: the checkpoint has no expert input_scale tensors")
|
|
gate, up, down = (pieces[f"{p}_input"].reshape(len(pieces[f"{p}_input"]), -1)[:, 0].float() for p in ("gate", "up", "down"))
|
|
return {"gate_up_alpha": torch.maximum(gate, up), "down_alpha": down}
|
|
|
|
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
|
|
if not is_prefill:
|
|
return super().apply(layer, x, topk_weights, topk_ids, view, is_prefill=False)
|
|
from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4_w4a4
|
|
|
|
t = view.tensors
|
|
assert view.alphas is not None
|
|
return fused_experts_nvfp4_w4a4(
|
|
x, t["gate_up"], t["gate_up_scale"], t["gate_up_global"], t["down"], t["down_scale"], t["down_global"],
|
|
view.alphas[0], view.alphas[1], topk_weights, topk_ids, view.n, layer.activation, float(layer.alpha), limit_or_inf(layer),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Marlin: vLLM's fused Marlin MoE (W4A16, dequant in kernel) over Marlin-tiled banks.
|
|
# The Marlin kernels come from vLLM (not sgl-kernel) deliberately: vLLM ships the
|
|
# NVFP4 instantiations in its AOT wheel together with the matching host-side layout
|
|
# transforms, so the pair stays consistent by construction. sgl-kernel's AOT wheel
|
|
# excludes the NVFP4 MoE Marlin kernels to cut wheel size (its python transforms pair
|
|
# with sglang's *JIT* kernel tree, whose scale encoding has since diverged).
|
|
#
|
|
# The small layout-transform helpers are ported from sglang
|
|
# (``srt/layers/quantization/marlin_utils{,_fp4}.py``, Apache-2.0); the kernels
|
|
# themselves are imported, not vendored.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _marlin_symbols_ok() -> bool:
|
|
"""Probe the exact vLLM symbols the pack/forward paths below use.
|
|
|
|
``find_spec`` only proves the top-level package exists; an incompatible donor
|
|
version would otherwise crash mid-init, after minutes of bank loading. Probing
|
|
at selection time degrades to triton before any bank is touched.
|
|
"""
|
|
try:
|
|
from vllm import _custom_ops # noqa: F401
|
|
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( # noqa: F401
|
|
fused_marlin_moe,
|
|
)
|
|
from vllm.model_executor.layers.quantization.utils.marlin_utils import ( # noqa: F401
|
|
marlin_permute_scales,
|
|
)
|
|
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401
|
|
nvfp4_marlin_process_global_scale,
|
|
nvfp4_marlin_process_scales,
|
|
)
|
|
from vllm.scalar_type import scalar_types # noqa: F401
|
|
except Exception as exc:
|
|
logger.warning_once(
|
|
f"NVFP4 marlin backend is installed but unusable ({exc!r}); "
|
|
"falling back to the Triton inline-dequant backend"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
# Marlin pack: mirrors vLLM's prepare_moe_fp4_layer_for_marlin so the host prep always matches the AOT op.
|
|
def _marlin_pack_proj(
|
|
packed: torch.Tensor,
|
|
scale: torch.Tensor,
|
|
row_global: torch.Tensor,
|
|
*,
|
|
size_k: int,
|
|
size_n: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Pack one expert projection (on GPU) into the Marlin layout.
|
|
|
|
``packed`` ``[N, K//2]`` uint8 e2m1 pairs, ``scale`` ``[N, K//16]`` e4m3,
|
|
``row_global`` ``[N]`` fp32 per-row global (rows of a merged gate_up may carry
|
|
w1's and w3's different globals). Returns ``(qweight [K//16, 2N] int32,
|
|
scales [K//16, N] e4m3-coded, global scalar bf16)``.
|
|
|
|
Marlin takes one global scalar per projection, so when the row globals differ
|
|
the ratio is folded into the fp16 block scales before they are re-encoded
|
|
(exact when all rows share one global, <=1 scale ulp otherwise).
|
|
"""
|
|
from vllm import _custom_ops as ops
|
|
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
|
marlin_permute_scales,
|
|
)
|
|
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
|
nvfp4_marlin_process_global_scale,
|
|
nvfp4_marlin_process_scales,
|
|
)
|
|
|
|
assert size_n % 64 == 0, f"Marlin requires N % 64 == 0, got {size_n}"
|
|
qweight = ops.gptq_marlin_repack(
|
|
b_q_weight=packed.view(torch.int32).T.contiguous(),
|
|
perm=torch.empty(0, dtype=torch.int, device=packed.device),
|
|
size_k=size_k,
|
|
size_n=size_n,
|
|
num_bits=4,
|
|
)
|
|
|
|
g = row_global.to(torch.float32)
|
|
g_max = g.max()
|
|
s = scale.to(torch.bfloat16)
|
|
if not torch.all(g == g_max):
|
|
s = (s.float() * (g / g_max).unsqueeze(1)).to(torch.bfloat16)
|
|
s = marlin_permute_scales(s.T, size_k=size_k, size_n=size_n, group_size=16)
|
|
s = nvfp4_marlin_process_scales(s)
|
|
if isinstance(s, tuple): # newer vLLM returns (scales, scale_factor)
|
|
s, factor = s
|
|
g_max = g_max / factor
|
|
g_out = nvfp4_marlin_process_global_scale(g_max.to(torch.bfloat16).reshape(1))
|
|
return qweight, s, g_out
|
|
|
|
|
|
@torch.no_grad()
|
|
def marlin_fused_experts(
|
|
hidden_states: torch.Tensor,
|
|
gate_up_q: torch.Tensor,
|
|
gate_up_s: torch.Tensor,
|
|
gate_up_alpha: torch.Tensor,
|
|
down_q: torch.Tensor,
|
|
down_s: torch.Tensor,
|
|
down_alpha: torch.Tensor,
|
|
topk_weights: torch.Tensor,
|
|
topk_ids: torch.Tensor,
|
|
activation: str,
|
|
apply_router_weight_on_input: bool,
|
|
) -> torch.Tensor:
|
|
"""Marlin W4A16 fused MoE (two grouped GEMMs + activation + reduce).
|
|
|
|
Two calling regimes share this entry: decode passes the full ``[S]`` slot cache
|
|
with ``topk_ids`` rewritten to slot ids; full-layer prefill passes banks whose
|
|
position == expert id (the materialized ``[:E]`` slot view or the overlap double
|
|
buffer views), so the raw routing ids arrive unmapped. The ``*_alpha`` vectors
|
|
are the matching per-row global scales in both regimes.
|
|
vLLM's implementation is device-side only (no host syncs), so the decode call is
|
|
CUDA-graph capturable.
|
|
"""
|
|
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe
|
|
from vllm.scalar_type import scalar_types
|
|
|
|
assert activation == "silu", "Marlin NVFP4 backend supports gated silu only"
|
|
return fused_marlin_moe(
|
|
hidden_states,
|
|
gate_up_q,
|
|
down_q,
|
|
None, # bias1
|
|
None, # bias2
|
|
gate_up_s,
|
|
down_s,
|
|
gating_output=None,
|
|
topk_weights=topk_weights,
|
|
topk_ids=topk_ids,
|
|
quant_type_id=scalar_types.float4_e2m1f.id,
|
|
apply_router_weight_on_input=apply_router_weight_on_input,
|
|
global_num_experts=gate_up_q.size(0),
|
|
activation=activation,
|
|
global_scale1=gate_up_alpha,
|
|
global_scale2=down_alpha,
|
|
)
|
|
|
|
|
|
class MarlinNvfp4MoEKernel(MoEKernel):
|
|
"""vLLM's fused Marlin MoE over pre-tiled banks; the globals fold into GPU-resident alphas."""
|
|
|
|
name = "marlin"
|
|
max_slots = MARLIN_MAX_SLOTS
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
if backend.is_rocm():
|
|
return "Marlin is CUDA-only and unavailable on ROCm; use triton"
|
|
if not backend.is_vllm_installed():
|
|
return "vLLM is not installed"
|
|
reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=True)
|
|
if reason:
|
|
return reason
|
|
if not _marlin_symbols_ok():
|
|
return "vLLM Marlin donor symbols are unusable"
|
|
return None
|
|
|
|
def worth_it(self, cfg: MoEConfig) -> bool:
|
|
return (8, 0) <= backend.device_capability() < (10, 0)
|
|
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
|
|
i, h = cfg.intermediate, cfg.hidden
|
|
return {
|
|
"gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),
|
|
"gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),
|
|
"down": BankSpec((i // GROUP, 2 * h), torch.int32),
|
|
"down_scale": BankSpec((i // GROUP, h), FP8),
|
|
"gate_up_alpha": BankSpec((), torch.bfloat16, resident=True),
|
|
"down_alpha": BankSpec((), torch.bfloat16, resident=True),
|
|
}
|
|
|
|
def pack(self, pieces, cfg: MoEConfig, out):
|
|
i, h = cfg.intermediate, cfg.hidden
|
|
device = torch.device("cuda")
|
|
gu, gus, gug = fused_piece(pieces, "gate_up"), fused_piece(pieces, "gate_up_scale"), fused_global(pieces, i)
|
|
dn, dns, dng = pieces["down"], pieces["down_scale"], global_rows(pieces["down_global"], h)
|
|
e = gu.shape[0]
|
|
gate_up_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)
|
|
down_alpha = torch.empty(e, dtype=torch.bfloat16, device=device)
|
|
for k in range(e):
|
|
qw, sc, al = _marlin_pack_proj(gu[k].to(device), gus[k].to(device), gug[k].to(device), size_k=h, size_n=2 * i)
|
|
out["gate_up"][k].copy_(qw)
|
|
out["gate_up_scale"][k].copy_(sc.view(FP8))
|
|
gate_up_alpha[k] = al[0]
|
|
qw, sc, al = _marlin_pack_proj(dn[k].to(device), dns[k].to(device), dng[k].to(device), size_k=i, size_n=h)
|
|
out["down"][k].copy_(qw)
|
|
out["down_scale"][k].copy_(sc.view(FP8))
|
|
down_alpha[k] = al[0]
|
|
torch.cuda.synchronize(device)
|
|
return {"gate_up_alpha": gate_up_alpha, "down_alpha": down_alpha}
|
|
|
|
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
|
|
t = view.tensors
|
|
assert view.alphas is not None
|
|
return marlin_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# b12x: flashinfer's SM12x CuTe-DSL fused MoE (W4A16) over b12x-packed banks.
|
|
# The pack (``prepare_w4a16_packed_weights``) runs on any CUDA build, but the fused-MoE
|
|
# kernel needs sm_120/121 AND a CUDA>=13 *driver* (it JIT-compiles PTX at runtime). The
|
|
# kernel object gates on that, so the forward is only reached on hardware that can run it.
|
|
#
|
|
# Layout contract (verified against flashinfer 0.6.12 prepare_w4a16_packed_weights):
|
|
# * prepare() takes the block scales already *swizzled* (it calls unswizzle_expert_
|
|
# scales internally), so the native row-major fp8 scales are run through
|
|
# nvfp4_block_scale_interleave first.
|
|
# * prepare() reorders the w13 rows with reorder_w13_to_gate_up == cat([second, first]),
|
|
# i.e. it expects the merged proj as [up, gate] and emits [gate, up]. FreeToken stores
|
|
# [gate, up], so the halves are swapped before prepare() to come out [gate, up] again
|
|
# (the kernel computes silu(first) * second == silu(gate) * up).
|
|
# * modelopt globals are passed straight through (no 1/g inversion).
|
|
# The fused forward reconstructs the W4A16PackedWeights from the banks and uses the
|
|
# prepared-weights launch (_launch_sm120_w4a16_moe) directly, so weights are prepared
|
|
# exactly once at load time (b12x_fused_moe would re-prepare + ptr-cache every call).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _b12x_symbols_ok() -> bool:
|
|
"""Probe the exact flashinfer symbols the pack/forward paths below use.
|
|
|
|
``find_spec`` only proves the top-level package exists; an incompatible donor
|
|
version would otherwise crash mid-init, after minutes of bank loading. Probing
|
|
at selection time degrades to triton before any bank is touched.
|
|
"""
|
|
try:
|
|
# Probe exactly what the pack/forward use: the row-major->swizzled scale
|
|
# transform, the W4A16 prepare, and the prepared-weights launch (which also
|
|
# transitively imports the CuTe-DSL kernel + cutlass).
|
|
from flashinfer import nvfp4_block_scale_interleave # noqa: F401
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import ( # noqa: F401
|
|
_launch_sm120_w4a16_moe,
|
|
)
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import ( # noqa: F401
|
|
W4A16PackedWeights,
|
|
_make_workspace,
|
|
prepare_w4a16_packed_weights,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning_once(
|
|
f"NVFP4 b12x backend is installed but unusable ({exc!r}); "
|
|
"falling back to the Triton inline-dequant backend"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def _flashinfer_cuda_major() -> int | None:
|
|
"""flashinfer's own toolkit-CUDA major; None if it can't be determined. Only a
|
|
fallback proxy when the driver version is unavailable -- the b12x kernel is JIT
|
|
PTX-compiled through the driver, so the toolkit major is *not* what gates it."""
|
|
try:
|
|
from flashinfer.jit.cpp_ext import get_cuda_version
|
|
|
|
return int(get_cuda_version().major)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _b12x_unusable_reason(cc: tuple[int, int]) -> str | None:
|
|
"""None if the flashinfer b12x decode/prefill kernel can actually run on this device,
|
|
else a human-readable reason. The b12x *pack* works anywhere, but the SM12x CuTe-DSL
|
|
kernel itself requires sm_120+ and a CUDA>=13 *driver* (it JIT-compiles PTX through the
|
|
driver), so selection must check the runtime here -- not just that flashinfer imports --
|
|
or the model loads in the b12x layout and then crashes on the first decode."""
|
|
import importlib.util
|
|
|
|
if cc < (12, 0):
|
|
return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
|
|
if importlib.util.find_spec("flashinfer") is None:
|
|
return "flashinfer is not installed"
|
|
from freetoken.kernel.backend import driver_cuda_version
|
|
|
|
drv = driver_cuda_version()
|
|
if drv is not None:
|
|
if drv < 13000:
|
|
return (
|
|
"b12x fused MoE requires a CUDA>=13 driver "
|
|
f"(driver supports CUDA {drv // 1000}.{(drv % 1000) // 10})"
|
|
)
|
|
else:
|
|
# Driver version undetermined: fall back to flashinfer's toolkit major as a
|
|
# conservative proxy rather than risk loading the b12x layout and crashing.
|
|
major = _flashinfer_cuda_major()
|
|
if major is not None and major < 13:
|
|
return (
|
|
"b12x fused MoE requires CUDA>=13 (driver undetermined; "
|
|
f"flashinfer toolkit is CUDA {major}.x)"
|
|
)
|
|
if not _b12x_symbols_ok():
|
|
return "flashinfer b12x donor symbols are unusable"
|
|
return None
|
|
|
|
|
|
def _b12x_swizzle_block_scales(scale: torch.Tensor) -> torch.Tensor:
|
|
"""Row-major per-16 fp8 block scales ``[n, N, K//16]`` -> flashinfer's expert-leading
|
|
*swizzled* storage, which is what :func:`prepare_w4a16_packed_weights` expects on input
|
|
(it unswizzles internally). The 128x4 interleave is per-expert; ``N`` is a multiple of
|
|
128 and ``K//16`` a multiple of 4 for the supported models, so the shape is preserved."""
|
|
from flashinfer import nvfp4_block_scale_interleave
|
|
|
|
pieces = [
|
|
nvfp4_block_scale_interleave(scale[e].view(torch.uint8).contiguous())
|
|
for e in range(scale.shape[0])
|
|
]
|
|
return torch.stack(pieces, dim=0).view(torch.float8_e4m3fn)
|
|
|
|
|
|
# Per-device scratch (sized sms*4+2) that run_w4a16_moe reads off the prepared object;
|
|
# tiny and shape-independent, so one per device is reused across all layers/decodes.
|
|
_B12X_WORKSPACE: dict[torch.device, torch.Tensor] = {}
|
|
|
|
|
|
@torch.no_grad()
|
|
def _b12x_small_workspace(device: torch.device) -> torch.Tensor:
|
|
ws = _B12X_WORKSPACE.get(device)
|
|
if ws is None:
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
|
|
_make_workspace,
|
|
)
|
|
|
|
ws = _make_workspace(device, max_blocks_per_sm=4)
|
|
_B12X_WORKSPACE[device] = ws
|
|
return ws
|
|
|
|
|
|
def b12x_fused_experts(
|
|
hidden_states: torch.Tensor,
|
|
gate_up_q: torch.Tensor,
|
|
gate_up_s: torch.Tensor,
|
|
gate_up_alpha: torch.Tensor,
|
|
down_q: torch.Tensor,
|
|
down_s: torch.Tensor,
|
|
down_alpha: torch.Tensor,
|
|
topk_weights: torch.Tensor,
|
|
topk_ids: torch.Tensor,
|
|
activation: str,
|
|
apply_router_weight_on_input: bool,
|
|
) -> torch.Tensor:
|
|
"""SM12x W4A16 fused MoE over cache slots (same calling convention as
|
|
:func:`marlin_fused_experts`). Requires sm_120/121 + a CUDA>=13 driver; reached only
|
|
when the b12x kernel object has confirmed the runtime supports it.
|
|
|
|
The banks already hold flashinfer's prepared (tiled) layout from
|
|
the b12x kernel object's pack, so this wraps them back into a
|
|
``W4A16PackedWeights`` and calls the prepared-weights launch directly -- the public
|
|
``b12x_fused_moe`` would re-prepare the raw modelopt weights (and ptr-cache them) on
|
|
every call, which both costs a prepare per step and breaks for the offload cache,
|
|
whose slot contents move between calls.
|
|
|
|
CUDA-graph note: the launch resolves a (shape-keyed, module-cached) scratch workspace
|
|
on first use and flashinfer raises if that happens *during* capture, so the decode
|
|
path must be warmed once eagerly before graph capture (FreeToken already does)."""
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_dispatch import (
|
|
_launch_sm120_w4a16_moe,
|
|
)
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import (
|
|
W4A16PackedWeights,
|
|
)
|
|
|
|
assert activation == "silu", "b12x backend supports gated silu only"
|
|
assert not apply_router_weight_on_input
|
|
num_experts = gate_up_q.size(0)
|
|
hidden_size = hidden_states.size(-1)
|
|
# down bank is the prepared w2 == [E, K_tiles, ...] with K_tiles == intermediate//16.
|
|
intermediate_size = down_q.size(1) * 16
|
|
prepared = W4A16PackedWeights(
|
|
w13=gate_up_q,
|
|
w13_scale=gate_up_s,
|
|
w13_global_scale=gate_up_alpha,
|
|
w2=down_q,
|
|
w2_scale=down_s,
|
|
w2_global_scale=down_alpha,
|
|
workspace=_b12x_small_workspace(hidden_states.device),
|
|
hidden_size=hidden_size,
|
|
intermediate_size=intermediate_size,
|
|
num_experts=num_experts,
|
|
is_gated=True,
|
|
params_dtype=hidden_states.dtype,
|
|
source_format="modelopt",
|
|
)
|
|
out = torch.empty(
|
|
hidden_states.size(0), hidden_size, dtype=hidden_states.dtype, device=hidden_states.device
|
|
)
|
|
_launch_sm120_w4a16_moe(
|
|
a=hidden_states,
|
|
topk_ids=topk_ids,
|
|
topk_weights=topk_weights,
|
|
w1_weight=gate_up_q,
|
|
w1_weight_sf=gate_up_s,
|
|
w1_alpha=gate_up_alpha,
|
|
w2_weight=down_q,
|
|
w2_weight_sf=down_s,
|
|
w2_alpha=down_alpha,
|
|
num_experts=num_experts,
|
|
top_k=topk_ids.size(1),
|
|
num_local_experts=num_experts,
|
|
scatter_output=out,
|
|
activation="silu",
|
|
source_format="modelopt",
|
|
_prepared_weights=prepared,
|
|
)
|
|
return out
|
|
|
|
|
|
class B12xNvfp4MoEKernel(MoEKernel):
|
|
"""flashinfer's SM12x CuTe-DSL W4A16 MoE; the packed blocks reuse the native bank bytes."""
|
|
|
|
name = "b12x"
|
|
|
|
def unusable_reason(self, cfg: MoEConfig) -> str | None:
|
|
if backend.is_rocm():
|
|
return "b12x is CUDA-only and unavailable on ROCm; use triton"
|
|
cc = backend.device_capability()
|
|
if cc < (12, 0):
|
|
return f"b12x requires sm_120+, got sm_{cc[0]}{cc[1]}"
|
|
if not backend.is_flashinfer_installed():
|
|
return "flashinfer is not installed"
|
|
reason = self._common_reject(cfg, tp_ok=False, cpu_ok=False, plain_silu_only=True)
|
|
if reason:
|
|
return reason
|
|
return _b12x_unusable_reason(cc)
|
|
|
|
def worth_it(self, cfg: MoEConfig) -> bool:
|
|
# NOTE: never auto-selected. flashinfer's cute launcher indexes each bank with int32 element offsets, so the GPU slot cache is capped at (2^31 - 1) / elements-per-slot (about 1000 slots for GLM-5.3-Flash's 12960 experts); until flashinfer lifts that, b12x stays behind triton in the table; the selection does not weigh the cap, cache-auto clamps to slot_limit() and OffloadMoeCache refuses a larger cache.
|
|
return cfg.intermediate >= B12X_MIN_INTERMEDIATE
|
|
|
|
def slot_limit(self, cfg: MoEConfig) -> int | None:
|
|
# the cute launcher indexes each bank with int32 element offsets
|
|
per_slot = max(math.prod(spec.shape) for spec in self.layout(cfg).values() if not spec.resident)
|
|
return (2**31 - 1) // per_slot
|
|
|
|
def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:
|
|
# flashinfer's prepared tiles: the Marlin shapes, byte-identical to the native rows
|
|
i, h = cfg.intermediate, cfg.hidden
|
|
return {
|
|
"gate_up": BankSpec((h // GROUP, 4 * i), torch.int32),
|
|
"gate_up_scale": BankSpec((h // GROUP, 2 * i), FP8),
|
|
"down": BankSpec((i // GROUP, 2 * h), torch.int32),
|
|
"down_scale": BankSpec((i // GROUP, h), FP8),
|
|
"gate_up_alpha": BankSpec((), torch.float32, resident=True),
|
|
"down_alpha": BankSpec((), torch.float32, resident=True),
|
|
}
|
|
|
|
def pack(self, pieces, cfg: MoEConfig, out):
|
|
from flashinfer.fused_moe.cute_dsl.blackwell_sm12x.moe_w4a16_prepare import prepare_w4a16_packed_weights
|
|
|
|
i, h = cfg.intermediate, cfg.hidden
|
|
device = torch.device("cuda")
|
|
gu = fused_piece(pieces, "gate_up").to(device)
|
|
gug = fused_global(pieces, i).to(device).float()
|
|
g_max = gug.max(dim=1, keepdim=True).values
|
|
gus = fused_piece(pieces, "gate_up_scale").to(device).to(torch.float16)
|
|
ratio = gug / g_max
|
|
if not torch.all(ratio == 1.0):
|
|
gus = (gus.float() * ratio.unsqueeze(-1)).to(torch.float16)
|
|
dng = global_rows(pieces["down_global"], h).to(device)
|
|
if not torch.all(dng == dng[:, :1]):
|
|
raise ValueError("b12x pack requires a row-constant per-expert down_proj global scale")
|
|
# b12x wants up|gate row order
|
|
gu = torch.cat([gu[:, i:], gu[:, :i]], dim=1).contiguous()
|
|
gus = torch.cat([gus[:, i:], gus[:, :i]], dim=1).contiguous()
|
|
prepared = prepare_w4a16_packed_weights(
|
|
gu,
|
|
_b12x_swizzle_block_scales(gus.to(FP8)),
|
|
g_max.squeeze(1),
|
|
pieces["down"].to(device),
|
|
_b12x_swizzle_block_scales(pieces["down_scale"].to(device)),
|
|
dng[:, 0].float(),
|
|
activation="silu",
|
|
params_dtype=torch.bfloat16,
|
|
source_format="modelopt",
|
|
)
|
|
e = gu.shape[0]
|
|
for role, t in (("gate_up", prepared.w13), ("gate_up_scale", prepared.w13_scale), ("down", prepared.w2), ("down_scale", prepared.w2_scale)):
|
|
bank = out[role]
|
|
assert t.shape[1:] == bank.shape[1:] and t.dtype == bank.dtype, (role, t.shape, t.dtype, bank.shape, bank.dtype)
|
|
bank.copy_(t[:e])
|
|
return {"gate_up_alpha": prepared.w13_global_scale[:e].float(), "down_alpha": prepared.w2_global_scale[:e].float()}
|
|
|
|
def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):
|
|
t = view.tensors
|
|
assert view.alphas is not None
|
|
return b12x_fused_experts(x, t["gate_up"], t["gate_up_scale"], view.alphas[0], t["down"], t["down_scale"], view.alphas[1], topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input)
|
|
|
|
|
|
@register_method(QuantKind.NVFP4, LayerKind.MOE)
|
|
class Nvfp4MoEMethod(MoEMethod):
|
|
candidates = (W4a4Nvfp4MoEKernel, TritonNvfp4MoEKernel, MarlinNvfp4MoEKernel, B12xNvfp4MoEKernel)
|