perf(moe): run Kimi-K3 NVFP4 SiTU experts on FlashInfer's CuTe DSL MoE

Register flashinfer_cutedsl_nvfp4_situ_routed_moe_apply
(--moe-backend flashinfer_cutedsl) for NVFP4 SiTU experts on precomputed
top-k. Its preprocessor converts NVFP4 checkpoint experts to the CuTe DSL
layout: W13 rows [gate | up] become [up | gate] interleaved in 64-row groups,
and the linear E4M3 block scales are swizzled per expert. The storage stays
expert-major, so DWDP prefill streams these experts like the TRT-LLM-Gen
layout. The kernel quantizes the GEMM2 input with one global scale, so the
preprocessor rejects checkpoints whose per-expert GEMM2 input scales differ.
The apply uses one fixed tactic and the fused finalize. auto still selects
TRT-LLM-Gen.

Kimi-K3 takes the backend through Kimi3MoEExecutionPlan.use_cutedsl (an
explicit field), and the peer-weight check admits it.

On GB300 at K3 prefill shapes (896 experts, top-16) the routed MoE layer is
5-6% faster than TRT-LLM-Gen at 16K rows and 11% faster at 24K rows beside a
peer weight copy. GSM8K strict on a K3 PD deployment with this backend for
prefill: 96.66% (TRT-LLM-Gen on the same deployment: 96.89-97.19%).
End-to-end serving throughput was not measured for this configuration.

Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
This commit is contained in:
Enwei Zhu
2026-11-16 16:04:10 +00:00
parent 970f00f9c5
commit 40f0605d06
12 changed files with 663 additions and 12 deletions
+4 -1
View File
@@ -969,7 +969,10 @@ long-context forwards.
The initial K3 NVFP4 configuration requires `--attn-tp-size 1 --moe-tp-size 1
--ep-size 1 --all2all-backend none`, `--disaggregation-mode prefill`,
`--enforce-eager --prefill-graph-max-tokens 0`, and a fixed prefill token budget.
The weight-sharing size must divide the instance size. Shared-expert and
The routed experts run on FlashInfer's TRT-LLM-Gen kernels
(`--moe-backend flashinfer_trtllm`) or its CuTe DSL kernels
(`--moe-backend flashinfer_cutedsl`, 5-11% faster MoE layers at K3 prefill
shapes on GB300). The weight-sharing size must divide the instance size. Shared-expert and
attention-projection TP overrides must also remain local. The additional
staging memory is allocated before KV cache sizing. Live checkpoint replacement
requires restarting these workers.
@@ -570,10 +570,11 @@ class MoELayer(torch.nn.Module):
if not (
self._quant_kind == "nvfp4"
and self.activation == "situ"
and self.plan["solution"] == "flashinfer_trtllm"
and self.plan["solution"] in ("flashinfer_trtllm", "flashinfer_cutedsl")
):
raise ValueError(
"Peer weights currently require FlashInfer TRTLLM NVFP4 SiTU"
"Peer weights currently require FlashInfer TRTLLM or CuTe DSL "
"NVFP4 SiTU"
)
self._storage_spec = replace(
self._spec,
@@ -139,6 +139,8 @@ class Kimi3MoEExecutionPlan:
owns_ep_communication: bool
overlap_shared_experts: bool
joint_moe_reduce: bool
# FlashInfer's CuTe DSL NVFP4 SiTU experts on precomputed top-k.
use_cutedsl: bool
use_marlin: bool = False
@classmethod
@@ -175,12 +177,19 @@ class Kimi3MoEExecutionPlan:
and not use_marlin
and (moe_backend.is_auto() or moe_backend.is_flashinfer_trtllm())
)
use_cutedsl = (
not owns_ep_communication
and not use_native
and not use_marlin
and moe_backend.is_flashinfer_cutedsl()
)
return cls(
use_native=use_native,
use_trtllm=use_trtllm,
use_mega_moe=use_mega_moe,
owns_ep_communication=owns_ep_communication,
use_marlin=use_marlin,
use_cutedsl=use_cutedsl,
overlap_shared_experts=(
use_native and alt_stream is not None and mapping.moe.tp_ep_size == 1
),
+10 -9
View File
@@ -2057,10 +2057,11 @@ class KimiLinearMoE(nn.Module):
and not plan.use_native
and not plan.use_marlin
):
if not plan.use_trtllm:
if not (plan.use_trtllm or plan.use_cutedsl):
raise RuntimeError(
"Kimi-K3 MXFP4 SiTU MoE requires the native, FlashInfer "
"TRT-LLM, or Marlin (Hopper W4A16) backend; no portable SiTU "
"Kimi-K3 SiTU MoE requires the native, FlashInfer TRT-LLM, "
"FlashInfer CuTe DSL (NVFP4), or Marlin (Hopper W4A16) "
"backend; no portable SiTU "
f"Triton fallback exists (selected MoE backend: "
f"{moe_backend.value!r})."
)
@@ -2112,7 +2113,7 @@ class KimiLinearMoE(nn.Module):
else (
_k3_trtllm_situ_internal_activation_dtype(quant_config, prefix)
if self.execution_plan.use_trtllm
else None
else "input" if self.execution_plan.use_cutedsl else None
)
),
)
@@ -2424,10 +2425,9 @@ class KimiLinearMoE(nn.Module):
def process_weights_after_loading(self, module) -> None:
"""Configure the latent projection from the processed expert input scale."""
if (
self.experts.plan["weight_dtype"] == "nvfp4"
and self.experts.plan["solution"] == "flashinfer_trtllm"
):
if self.experts.plan["weight_dtype"] == "nvfp4" and self.experts.plan[
"solution"
] in ("flashinfer_trtllm", "flashinfer_cutedsl"):
# The loader visits this parent before its expert child.
self.experts.process_weights_after_loading(self.experts)
self.routed_expert_down_proj.prepare_nvfp4_output(
@@ -2448,11 +2448,12 @@ class KimiLinearMoE(nn.Module):
plan.owns_ep_communication
or plan.use_native
or plan.use_trtllm
or plan.use_cutedsl
or plan.use_marlin
):
raise RuntimeError(
"Kimi-K3 has no portable SiTU Triton fallback; use the native, "
"FlashInfer TRT-LLM, or Marlin SiTU MoE path."
"FlashInfer TRT-LLM, FlashInfer CuTe DSL, or Marlin SiTU MoE path."
)
out = self.experts(
hidden_states=routed_in,
+16
View File
@@ -291,6 +291,22 @@ def test_kimi3_moe_execution_policy_honours_a_forced_marlin_backend() -> None:
assert not plan.use_native
def test_kimi3_moe_execution_policy_honours_a_forced_cutedsl_backend() -> None:
"""Asked for FlashInfer's CuTe DSL experts, the plan routes to them alone."""
with mock.patch.object(
latent_module, "native_latent_moe_available", return_value=False
):
plan = Kimi3MoEExecutionPlan.build(
_plan_mapping(), MoeBackend.FLASHINFER_CUTEDSL, alt_stream=None
)
assert plan.use_cutedsl
assert not plan.use_trtllm
assert not plan.use_marlin
assert not plan.use_native
assert not plan.owns_ep_communication
def test_kimi3_moe_execution_policy_takes_marlin_when_its_probe_says_yes() -> None:
"""The probe decides AUTO, and the contraction shard's gate rides on it."""
with (
+8
View File
@@ -300,6 +300,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=False,
use_trtllm=True,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
)
@@ -326,6 +327,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=False,
use_trtllm=True,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
)
@@ -370,6 +372,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=False,
use_trtllm=True,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
),
@@ -389,6 +392,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=False,
use_trtllm=False,
use_cutedsl=False,
use_marlin=True,
overlap_shared_experts=False,
joint_moe_reduce=False,
@@ -413,6 +417,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=False,
use_trtllm=True,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
),
@@ -1002,6 +1007,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=True,
use_trtllm=False,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=True,
),
@@ -1092,6 +1098,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=True,
use_trtllm=False,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
),
@@ -1134,6 +1141,7 @@ class TestKimiK3RegistrationTests:
owns_ep_communication=False,
use_native=True,
use_trtllm=False,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=True,
),
@@ -136,6 +136,7 @@ def test_k3_accepts_a_placement_only_on_the_attention_dp_megamoe_path(
trtllm = Kimi3MoEExecutionPlan(
use_native=False,
use_trtllm=True,
use_cutedsl=False,
use_mega_moe=False,
owns_ep_communication=False,
overlap_shared_experts=False,
+1
View File
@@ -99,6 +99,7 @@ def test_attn_dp_replicates_dense_weights_and_selects_transport(
owns_ep_communication=mega_moe,
use_native=False,
use_trtllm=not mega_moe,
use_cutedsl=False,
overlap_shared_experts=False,
joint_moe_reduce=False,
)
@@ -264,6 +264,25 @@ expert ids. With FP32 route weights the routed kernel stops before its
finalize and `moe_finalize_fuse_shared` combines the per-route outputs with the
FP32 weights, accumulating in FP32 and rounding once to BF16.
## FlashInfer CuTe DSL NVFP4 SiTU (`flashinfer_cutedsl.py`)
`flashinfer_cutedsl_nvfp4_situ_routed_moe_apply` runs Kimi-K3 NVFP4 SiTU
experts on FlashInfer's CuTe DSL fused MoE (`--moe-backend
flashinfer_cutedsl`); `auto` keeps the TRT-LLM-Gen kernel. The preprocessor
converts the checkpoint layout: W13 rows `[gate | up]` become `[up | gate]`
interleaved in 64-row groups, and the linear E4M3 block scales are swizzled
per expert. The scale storage stays expert-major and the kernels read it
through an MMA-layout view, so DWDP streams these experts like the TRT-LLM
layout. FC1 alpha is `a13 * w13_scale_2` and FC2 alpha `a2 * w2_scale_2`. The
kernel quantizes the GEMM2 input with one global scale (`1 / a2`), so the
preprocessor rejects checkpoints whose per-expert `a2` differ (Kimi-K3-NVFP4
ships 1.0 everywhere).
The apply uses one fixed tactic (256-row routing tiles, 2-CTA clusters) and
the fused, atomic finalize; it always finalizes. On GB300 at K3 prefill shapes
(896 experts, top-16, uniform routing) the routed MoE layer is 5-6% faster than
TRT-LLM-Gen at 16K rows and 11% faster at 24K rows beside a peer weight copy.
## FP8 per-channel experts
Per-channel FP8 experts store weights in FP8 E4M3 with FP32 scales per output
@@ -2107,6 +2107,7 @@ import tokenspeed_kernel.ops.moe.cute_dsl # noqa: E402,F401
import tokenspeed_kernel.ops.moe.deep_ep # noqa: E402,F401
import tokenspeed_kernel.ops.moe.deep_gemm # noqa: E402,F401
import tokenspeed_kernel.ops.moe.flashinfer # noqa: E402,F401
import tokenspeed_kernel.ops.moe.flashinfer_cutedsl # noqa: E402,F401
import tokenspeed_kernel.ops.moe.gluon # noqa: E402,F401
import tokenspeed_kernel.ops.moe.marlin # noqa: E402,F401
import tokenspeed_kernel.ops.moe.native # noqa: E402,F401
@@ -0,0 +1,267 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""FlashInfer CuTe DSL NVFP4 routed MoE with the SiTU activation (precomputed top-k).
The preprocessor turns NVFP4 checkpoint experts (``[gate | up]`` W13 rows,
linear E4M3 block scales) into the CuTe DSL layout: W13 rows become
``[up | gate]`` interleaved in 64-row groups and every block-scale matrix is
swizzled. The kernels read the scales through an MMA-layout view of that
expert-major storage, so the peer-weight store can stream experts as it does
the TRT-LLM layout.
"""
from __future__ import annotations
import torch
from tokenspeed_kernel.platform import (
ArchVersion,
CapabilityRequirement,
current_platform,
)
from tokenspeed_kernel.registry import Priority, register_kernel
from tokenspeed_kernel.signature import format_signatures
platform = current_platform()
_SF_VEC = 16
_W13_ROW_GROUP = 64
# One fixed tactic for both GEMMs (256-row routing tiles, 2-CTA clusters), so
# each kernel compiles once, during the startup forward, for any row count.
_TACTIC = (256, ((256, 256), (2, 1), False), ((256, 256), (2, 1), False))
_STREAMED = (
"cutedsl_w1_weight",
"cutedsl_w1_weight_sf",
"cutedsl_w2_weight",
"cutedsl_w2_weight_sf",
)
_REPLICATED = ("cutedsl_w1_alpha", "cutedsl_w2_alpha")
if platform.is_nvidia:
from tokenspeed_kernel.ops.moe.flashinfer import _positive_situ_value
def _w13_to_cute_rows(t: torch.Tensor) -> torch.Tensor:
"""``[E, 2I, X]`` rows ``[gate | up]`` -> ``[up | gate]`` interleaved by 64 rows."""
experts, rows, cols = t.shape
halves = t.view(experts, 2, rows // (2 * _W13_ROW_GROUP), _W13_ROW_GROUP, cols)
return halves.flip(1).transpose(1, 2).reshape(experts, rows, cols).contiguous()
def _swizzle_block_scales(linear: torch.Tensor) -> torch.Tensor:
"""Per-expert 128x4 swizzle of ``[E, rows, cols]`` E4M3 block scales (bytes)."""
from flashinfer import nvfp4_block_scale_interleave
experts, rows, cols = linear.shape
return torch.stack(
[
nvfp4_block_scale_interleave(linear[e].contiguous()).view(rows, cols)
for e in range(experts)
]
)
def flashinfer_cutedsl_nvfp4_situ_moe_weights(plan: dict, w: torch.nn.Module):
"""Convert NVFP4 checkpoint experts to the CuTe DSL layout.
Args:
plan: The selected MoE plan.
w: Expert module holding ``w13_*`` / ``w2_*`` NVFP4 parameters.
Returns:
The expert-major processed weights for the peer-weight store.
"""
from tokenspeed_kernel.ops.communication.peer_weights import (
ExpertWeightShard,
PreparedExpertWeights,
)
ws2 = w.w13_weight_scale_2
gate_ws2, up_ws2 = (ws2[:, 0], ws2[:, 1]) if ws2.dim() == 2 else (ws2, ws2)
if not torch.equal(gate_ws2, up_ws2):
raise RuntimeError(
"NVFP4 SiTU requires equal gate/up weight_scale_2 per expert."
)
w2_input_scale = w.w2_input_scale.to(torch.float32)
if not torch.all(w2_input_scale == w2_input_scale[0]):
raise RuntimeError(
"CuTe DSL NVFP4 MoE quantizes the GEMM2 input with one global "
"scale; the checkpoint has per-expert values."
)
w13_input_scale = w.w13_input_scale.to(torch.float32)
w1 = _w13_to_cute_rows(w.w13_weight.data)
w1_sf = _swizzle_block_scales(
_w13_to_cute_rows(w.w13_weight_scale.data.view(torch.uint8))
)
w2 = w.w2_weight.data.contiguous()
w2_sf = _swizzle_block_scales(w.w2_weight_scale.data.view(torch.uint8))
def param(t: torch.Tensor) -> torch.nn.Parameter:
return torch.nn.Parameter(t, requires_grad=False)
w.cutedsl_w1_weight = param(w1)
w.cutedsl_w1_weight_sf = param(w1_sf)
w.cutedsl_w2_weight = param(w2)
w.cutedsl_w2_weight_sf = param(w2_sf)
w.cutedsl_w1_alpha = param((w13_input_scale * gate_ws2).contiguous())
w.cutedsl_w2_alpha = param(
(w2_input_scale * w.w2_weight_scale_2.to(torch.float32)).contiguous()
)
w.cutedsl_fc2_input_scale = param((1.0 / w2_input_scale[:1]).contiguous())
w.w13_input_scale_quant = param((1.0 / w13_input_scale).contiguous())
w.cutedsl_situ_beta = _positive_situ_value(
w, "activation_situ_beta", "situ_beta"
)
w.cutedsl_situ_linear_beta = _positive_situ_value(
w, "activation_situ_linear_beta", "situ_linear_beta"
)
for name in (
"w13_weight",
"w2_weight",
"w13_weight_scale",
"w2_weight_scale",
"w13_weight_scale_2",
"w2_weight_scale_2",
"w13_input_scale",
"w2_input_scale",
):
delattr(w, name)
parameters = dict(w.named_parameters(recurse=False))
return PreparedExpertWeights(
layout="flashinfer-cutedsl-nvfp4-situ-v1",
shards=tuple(
ExpertWeightShard(name, parameters[name], name in _REPLICATED)
for name in (*_STREAMED, *_REPLICATED)
),
)
@register_kernel(
"moe",
"apply",
name="flashinfer_cutedsl_nvfp4_situ_routed_moe_apply",
solution="flashinfer_cutedsl",
weight_preprocessor=flashinfer_cutedsl_nvfp4_situ_moe_weights,
capability=CapabilityRequirement(
vendors=frozenset({"nvidia"}),
min_arch_version=ArchVersion(10, 0),
max_arch_version=ArchVersion(10, 7),
),
signatures=format_signatures("x", "dense", {torch.bfloat16, torch.uint8}),
traits={
"weight_dtype": frozenset({"nvfp4"}),
"activation": frozenset({"situ"}),
"routing_mode": frozenset({"precomputed_topk"}),
"supports_deferred_finalize": frozenset({False}),
"supports_ep": frozenset({False}),
"supports_all_to_all_ep": frozenset({False}),
"ispp_alignment": frozenset({128}),
"internal_activation_dtype": frozenset({"input"}),
"supports_bias": frozenset({False}),
},
priority=Priority.SPECIALIZED - 1,
)
def flashinfer_cutedsl_nvfp4_situ_routed_moe_apply(
plan: dict,
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
w: torch.nn.Module,
router_logits: torch.Tensor,
topk_weights: torch.Tensor | None = None,
topk_ids: torch.Tensor | None = None,
num_tokens_global: int | None = None,
max_num_tokens_per_gpu: int | None = None,
do_finalize: bool = True,
enable_pdl: bool = False,
) -> torch.Tensor:
"""Routed NVFP4 SiTU experts on precomputed top-k, finalized in the kernel.
Args:
plan: The selected MoE plan.
x: BF16 ``[tokens, hidden]`` activations, or an NVFP4 pair of packed
``[tokens, hidden / 2]`` codes and linear ``[tokens, hidden / 16]``
block scales quantized with ``w.w13_input_scale_quant``.
w: Expert module prepared by the preprocessor.
router_logits: Unused; routing is precomputed.
topk_weights: ``[tokens, top_k]`` FP32 route weights.
topk_ids: ``[tokens, top_k]`` expert ids.
num_tokens_global: Unused.
max_num_tokens_per_gpu: Unused.
do_finalize: Must be True.
enable_pdl: Whether the kernels use programmatic dependent launch.
Returns:
The BF16 ``[tokens, hidden]`` weighted sum over each token's experts.
"""
from flashinfer import fp4_quantize
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
from flashinfer.fused_moe.cute_dsl.fused_moe import cute_dsl_fused_moe
from flashinfer.tllm_enums import ActivationType
if topk_weights is None or topk_ids is None:
raise ValueError("precomputed_topk plan requires topk_weights and topk_ids")
if not do_finalize:
raise ValueError("CuTe DSL NVFP4 SiTU MoE always finalizes")
if isinstance(x, tuple):
x_fp4, x_sf = x
else:
x_fp4, x_sf = fp4_quantize(
x,
w.w13_input_scale_quant,
sf_vec_size=_SF_VEC,
is_sf_swizzled_layout=False,
)
experts, w13_rows, half_hidden = w.cutedsl_w1_weight.shape
hidden = half_hidden * 2
w1_sf = convert_sf_to_mma_layout(
w.cutedsl_w1_weight_sf.view(experts * w13_rows, -1),
m=w13_rows,
k=hidden,
num_groups=experts,
sf_vec_size=_SF_VEC,
)
w2_sf = convert_sf_to_mma_layout(
w.cutedsl_w2_weight_sf.view(experts * hidden, -1),
m=hidden,
k=w13_rows // 2,
num_groups=experts,
sf_vec_size=_SF_VEC,
)
return cute_dsl_fused_moe(
x_fp4,
x_sf.view(torch.uint8).unsqueeze(-1),
topk_ids.to(torch.int32),
topk_weights,
w.cutedsl_w1_weight,
w1_sf,
w.cutedsl_w1_alpha,
w.cutedsl_fc2_input_scale,
w.cutedsl_w2_weight,
w2_sf,
w.cutedsl_w2_alpha,
num_experts=experts,
top_k=topk_ids.shape[1],
use_fused_finalize=True,
enable_pdl=enable_pdl,
activation_type=ActivationType.Swiglu.value,
situ_beta=w.cutedsl_situ_beta,
situ_linear_beta=w.cutedsl_situ_linear_beta,
tactic=_TACTIC,
)
@@ -0,0 +1,324 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""flashinfer CuTe DSL NVFP4 SiTU MoE (w4a4) on NVFP4 checkpoint experts.
The preprocessor's layout is checked bytewise against FlashInfer's own
``prepare_cute_dsl_weights``; the registered apply against an fp32 SiTU MoE
over the exactly-dequantized weights (both activation quantizations modeled),
anchored to the TRT-LLM-Gen SiTU kernel on the same checkpoint.
"""
from __future__ import annotations
import pytest
import torch
from tokenspeed_kernel.numerics.torch.activation import torch_situ_and_mul
from tokenspeed_kernel.numerics.torch.moe import DequantizedExperts, torch_moe_experts
from tokenspeed_kernel.numerics.torch.quantization import dequantize_nvfp4_blocks
from tokenspeed_kernel.ops.moe import moe_plan as kernel_moe_plan
NUM_EXPERTS = 16
TOP_K = 8
HIDDEN = 256
ISPP = 256 # the CuTe DSL tactic tiles 2 * ISPP and HIDDEN by 256
SITU_BETA = 4.0
SITU_LINEAR_BETA = 25.0
_FP8_E4M3_MAX = 448.0
_FP4_E2M1_MAX = 6.0
def _runtime_reason() -> str | None:
if not torch.cuda.is_available():
return "requires CUDA"
import tokenspeed_kernel.ops.moe.flashinfer_cutedsl # noqa: F401
from tokenspeed_kernel.platform import current_platform
from tokenspeed_kernel.registry import KernelRegistry
spec = KernelRegistry.get().get_by_name(
"flashinfer_cutedsl_nvfp4_situ_routed_moe_apply"
)
if spec is None or not spec.capability.satisfied_by(current_platform()):
return "outside the kernel's registered capability range"
return None
_reason = _runtime_reason()
requires_cutedsl_situ = pytest.mark.skipif(_reason is not None, reason=str(_reason))
def _quantize(w: torch.Tensor, unit_scale: bool):
"""NVFP4 of a 2-D weight: packed codes, linear E4M3 scales, weight_scale_2."""
from flashinfer import fp4_quantize
w = w.cuda().to(torch.bfloat16)
global_scale = (
torch.ones((), device="cuda")
if unit_scale
else _FP8_E4M3_MAX * _FP4_E2M1_MAX / w.float().abs().amax().clamp(min=1e-12)
).reshape(1)
packed, sf = fp4_quantize(w, global_scale, is_sf_swizzled_layout=False)
m, k = w.shape
sf = sf.reshape(-1)[: m * (k // 16)].view(m, k // 16).view(torch.float8_e4m3fn)
return packed.view(m, k // 2), sf, (1.0 / global_scale).reshape(())
def _checkpoint(w13_bf16, w2_bf16, *, unit_scale: bool) -> dict[str, torch.Tensor]:
"""Loader-layout NVFP4 experts: W13 rows [gate | up], one scale_2 per expert."""
q13 = [_quantize(w, unit_scale) for w in w13_bf16]
q2 = [_quantize(w, unit_scale) for w in w2_bf16]
return {
"w13_weight": torch.stack([p for p, _, _ in q13]),
"w13_weight_scale": torch.stack([s for _, s, _ in q13]),
"w13_weight_scale_2": torch.stack([g for _, _, g in q13]),
"w2_weight": torch.stack([p for p, _, _ in q2]),
"w2_weight_scale": torch.stack([s for _, s, _ in q2]),
"w2_weight_scale_2": torch.stack([g for _, _, g in q2]),
"w13_input_scale": torch.ones(1, device="cuda"),
"w2_input_scale": torch.ones(NUM_EXPERTS, device="cuda"),
}
class _Experts(torch.nn.Module):
"""Minimal module carrying what the preprocessors and applies consume."""
def __init__(self, raw: dict[str, torch.Tensor]) -> None:
from types import SimpleNamespace
from flashinfer.fused_moe import RoutingMethodType
super().__init__()
for name, value in raw.items():
self.register_parameter(
name, torch.nn.Parameter(value.clone(), requires_grad=False)
)
self.activation_situ_beta = SITU_BETA
self.activation_situ_linear_beta = SITU_LINEAR_BETA
self._routing_method_type = RoutingMethodType.RenormalizeNaive
self._routing_logits_dtype = torch.bfloat16
self._spec = SimpleNamespace(
num_experts=NUM_EXPERTS,
num_local_experts=NUM_EXPERTS,
top_k=TOP_K,
ep_rank=0,
)
def _weights(generator: torch.Generator):
w13 = torch.randn(NUM_EXPERTS, 2 * ISPP, HIDDEN, generator=generator) * 0.5
w2 = torch.randn(NUM_EXPERTS, HIDDEN, ISPP, generator=generator) * 0.5
return w13.bfloat16(), w2.bfloat16()
def _routing(generator: torch.Generator, num_tokens: int):
ids = torch.stack(
[
torch.randperm(NUM_EXPERTS, generator=generator)[:TOP_K]
for _ in range(num_tokens)
]
)
weights = torch.rand(num_tokens, TOP_K, generator=generator).softmax(dim=-1)
return ids.to(torch.int32).cuda(), weights.cuda()
def _quant_dequant(t: torch.Tensor) -> torch.Tensor:
"""The kernels' NVFP4 activation quantization (unit global scale) in fp32."""
packed, sf, scale_2 = _quantize(t, unit_scale=True)
return dequantize_nvfp4_blocks(packed, sf) * scale_2
def _reference(hidden_states, raw, topk_ids, topk_weights) -> torch.Tensor:
def experts(prefix: str) -> DequantizedExperts:
return DequantizedExperts(
lambda packed, sf, scale_2: dequantize_nvfp4_blocks(packed, sf) * scale_2,
raw[f"{prefix}_weight"],
raw[f"{prefix}_weight_scale"],
raw[f"{prefix}_weight_scale_2"],
)
return torch_moe_experts(
hidden_states,
experts("w13"),
experts("w2"),
topk_ids,
topk_weights,
activation=lambda gate_up: torch_situ_and_mul(
gate_up, SITU_BETA, SITU_LINEAR_BETA
),
w13_bias=None,
w2_bias=None,
quantize_input=lambda t: _quant_dequant(t.float()),
quantize_intermediate=_quant_dequant,
round_expert_output=None,
first_expert=0,
)
def _rel_l2(actual: torch.Tensor, expected: torch.Tensor) -> float:
return ((actual.float() - expected.float()).norm() / expected.float().norm()).item()
@requires_cutedsl_situ
def test_preprocessor_layout_matches_flashinfer_prepare() -> None:
"""Checkpoint experts land exactly where FlashInfer's own BF16 preparation
puts them: W13 [up | gate] interleaved by 64 rows, swizzled scales."""
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
from flashinfer.fused_moe.api import QuantConfig, QuantFormat
from flashinfer.fused_moe.prepare import prepare_cute_dsl_weights
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
flashinfer_cutedsl_nvfp4_situ_moe_weights,
)
w13, w2 = _weights(torch.Generator().manual_seed(20261116))
w = _Experts(_checkpoint(w13, w2, unit_scale=True)).cuda()
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, w)
up_gate = torch.cat([w13[:, ISPP:], w13[:, :ISPP]], dim=1).cuda()
ref = prepare_cute_dsl_weights(
up_gate,
w2.cuda(),
quant=QuantConfig(weight=QuantFormat.NVFP4, activation=QuantFormat.NVFP4),
num_local_experts=NUM_EXPERTS,
hidden_size=HIDDEN,
intermediate_size=ISPP,
)
def mma_view(sf: torch.Tensor, m: int, k: int) -> torch.Tensor:
return convert_sf_to_mma_layout(
sf.view(NUM_EXPERTS * m, -1), m=m, k=k, num_groups=NUM_EXPERTS
)
assert torch.equal(w.cutedsl_w1_weight, ref["w1_weight"].view(torch.uint8))
assert torch.equal(w.cutedsl_w2_weight, ref["w2_weight"].view(torch.uint8))
assert torch.equal(
mma_view(w.cutedsl_w1_weight_sf, 2 * ISPP, HIDDEN).view(torch.uint8),
ref["w1_weight_sf"].view(torch.uint8),
)
assert torch.equal(
mma_view(w.cutedsl_w2_weight_sf, HIDDEN, ISPP).view(torch.uint8),
ref["w2_weight_sf"].view(torch.uint8),
)
@requires_cutedsl_situ
def test_routed_moe_matches_dequant_reference_and_trtllm() -> None:
from flashinfer import fp4_quantize
from tokenspeed_kernel.ops.moe.flashinfer import (
flashinfer_trtllm_nvfp4_situ_moe_weights,
flashinfer_trtllm_nvfp4_situ_routed_moe_apply,
)
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
flashinfer_cutedsl_nvfp4_situ_moe_weights,
flashinfer_cutedsl_nvfp4_situ_routed_moe_apply,
)
generator = torch.Generator().manual_seed(20261117)
num_tokens = 64
w13, w2 = _weights(generator)
raw = _checkpoint(w13, w2, unit_scale=False)
hidden_states = (
(torch.randn(num_tokens, HIDDEN, generator=generator) * 0.2).bfloat16().cuda()
)
topk_ids, topk_weights = _routing(generator, num_tokens)
cute = _Experts(raw).cuda()
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, cute)
trtllm = _Experts(raw).cuda()
flashinfer_trtllm_nvfp4_situ_moe_weights({}, trtllm)
def run_cute(x):
return flashinfer_cutedsl_nvfp4_situ_routed_moe_apply(
{},
x,
cute,
router_logits=None,
topk_weights=topk_weights,
topk_ids=topk_ids,
)
actual = run_cute(hidden_states)
serving = flashinfer_trtllm_nvfp4_situ_routed_moe_apply(
{},
hidden_states,
trtllm,
router_logits=None,
topk_weights=topk_weights.bfloat16(),
topk_ids=topk_ids,
)
torch.cuda.synchronize()
expected = _reference(hidden_states, raw, topk_ids, topk_weights)
cute_err, serving_err = _rel_l2(actual, expected), _rel_l2(serving, expected)
assert cute_err < 0.06, f"{cute_err=:.4f}"
assert cute_err < 1.5 * serving_err + 0.005, f"{cute_err=:.4f} {serving_err=:.4f}"
# An NVFP4 pair from the latent projection gives the BF16-input result.
# The finalize adds experts with BF16 atomics in arrival order, so runs
# differ by rounding (measured 4.5e-3 here); a wrong layout is O(1).
x_fp4, x_sf = fp4_quantize(
hidden_states, cute.w13_input_scale_quant, is_sf_swizzled_layout=False
)
prequantized = run_cute((x_fp4, x_sf))
repeat = run_cute(hidden_states)
torch.cuda.synchronize()
assert _rel_l2(repeat, actual) < 1e-2
assert _rel_l2(prequantized, actual) < 1e-2
@requires_cutedsl_situ
def test_preprocessor_rejects_per_expert_gemm2_input_scales() -> None:
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
flashinfer_cutedsl_nvfp4_situ_moe_weights,
)
w13, w2 = _weights(torch.Generator().manual_seed(20261118))
raw = _checkpoint(w13, w2, unit_scale=False)
raw["w2_input_scale"] = torch.linspace(0.5, 1.0, NUM_EXPERTS, device="cuda")
with pytest.raises(RuntimeError, match="one global"):
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, _Experts(raw).cuda())
@requires_cutedsl_situ
@pytest.mark.parametrize(
"solution,kernel",
[
("flashinfer_cutedsl", "flashinfer_cutedsl_nvfp4_situ_routed_moe_apply"),
(None, "flashinfer_trtllm_nvfp4_situ_routed_moe_apply"),
],
)
def test_plan_selects_cutedsl_only_when_requested(solution, kernel) -> None:
plan = kernel_moe_plan(
"nvfp4",
input_dtype=torch.bfloat16,
activation="situ",
routing_mode="precomputed_topk",
ep_size=1,
ispp=3072,
internal_activation_dtype="input",
hidden=3584,
swiglu_form=None,
activation_clamped=False,
expert_id_repeats=False,
num_experts=896,
top_k=16,
fast_math=True,
combine_order="rank",
features=frozenset(),
solution=solution,
)
assert plan["apply_kernel_name"] == kernel