mirror of
https://github.com/lightseekorg/tokenspeed.git
synced 2026-11-21 01:53:23 +08:00
perf(moe): run Kimi-K3 NVFP4 SiTU experts on FlashInfer's CuTe DSL MoE
Register flashinfer_cutedsl_nvfp4_situ_routed_moe_apply (--moe-backend flashinfer_cutedsl) for NVFP4 SiTU experts on precomputed top-k. Its preprocessor converts NVFP4 checkpoint experts to the CuTe DSL layout: W13 rows [gate | up] become [up | gate] interleaved in 64-row groups, and the linear E4M3 block scales are swizzled per expert. The storage stays expert-major, so DWDP prefill streams these experts like the TRT-LLM-Gen layout. The kernel quantizes the GEMM2 input with one global scale, so the preprocessor rejects checkpoints whose per-expert GEMM2 input scales differ. The apply uses one fixed tactic and the fused finalize. auto still selects TRT-LLM-Gen. Kimi-K3 takes the backend through Kimi3MoEExecutionPlan.use_cutedsl (an explicit field), and the peer-weight check admits it. On GB300 at K3 prefill shapes (896 experts, top-16) the routed MoE layer is 5-6% faster than TRT-LLM-Gen at 16K rows and 11% faster at 24K rows beside a peer weight copy. GSM8K strict on a K3 PD deployment with this backend for prefill: 96.66% (TRT-LLM-Gen on the same deployment: 96.89-97.19%). End-to-end serving throughput was not measured for this configuration. Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
This commit is contained in:
@@ -969,7 +969,10 @@ long-context forwards.
|
||||
The initial K3 NVFP4 configuration requires `--attn-tp-size 1 --moe-tp-size 1
|
||||
--ep-size 1 --all2all-backend none`, `--disaggregation-mode prefill`,
|
||||
`--enforce-eager --prefill-graph-max-tokens 0`, and a fixed prefill token budget.
|
||||
The weight-sharing size must divide the instance size. Shared-expert and
|
||||
The routed experts run on FlashInfer's TRT-LLM-Gen kernels
|
||||
(`--moe-backend flashinfer_trtllm`) or its CuTe DSL kernels
|
||||
(`--moe-backend flashinfer_cutedsl`, 5-11% faster MoE layers at K3 prefill
|
||||
shapes on GB300). The weight-sharing size must divide the instance size. Shared-expert and
|
||||
attention-projection TP overrides must also remain local. The additional
|
||||
staging memory is allocated before KV cache sizing. Live checkpoint replacement
|
||||
requires restarting these workers.
|
||||
|
||||
@@ -570,10 +570,11 @@ class MoELayer(torch.nn.Module):
|
||||
if not (
|
||||
self._quant_kind == "nvfp4"
|
||||
and self.activation == "situ"
|
||||
and self.plan["solution"] == "flashinfer_trtllm"
|
||||
and self.plan["solution"] in ("flashinfer_trtllm", "flashinfer_cutedsl")
|
||||
):
|
||||
raise ValueError(
|
||||
"Peer weights currently require FlashInfer TRTLLM NVFP4 SiTU"
|
||||
"Peer weights currently require FlashInfer TRTLLM or CuTe DSL "
|
||||
"NVFP4 SiTU"
|
||||
)
|
||||
self._storage_spec = replace(
|
||||
self._spec,
|
||||
|
||||
@@ -139,6 +139,8 @@ class Kimi3MoEExecutionPlan:
|
||||
owns_ep_communication: bool
|
||||
overlap_shared_experts: bool
|
||||
joint_moe_reduce: bool
|
||||
# FlashInfer's CuTe DSL NVFP4 SiTU experts on precomputed top-k.
|
||||
use_cutedsl: bool
|
||||
use_marlin: bool = False
|
||||
|
||||
@classmethod
|
||||
@@ -175,12 +177,19 @@ class Kimi3MoEExecutionPlan:
|
||||
and not use_marlin
|
||||
and (moe_backend.is_auto() or moe_backend.is_flashinfer_trtllm())
|
||||
)
|
||||
use_cutedsl = (
|
||||
not owns_ep_communication
|
||||
and not use_native
|
||||
and not use_marlin
|
||||
and moe_backend.is_flashinfer_cutedsl()
|
||||
)
|
||||
return cls(
|
||||
use_native=use_native,
|
||||
use_trtllm=use_trtllm,
|
||||
use_mega_moe=use_mega_moe,
|
||||
owns_ep_communication=owns_ep_communication,
|
||||
use_marlin=use_marlin,
|
||||
use_cutedsl=use_cutedsl,
|
||||
overlap_shared_experts=(
|
||||
use_native and alt_stream is not None and mapping.moe.tp_ep_size == 1
|
||||
),
|
||||
|
||||
@@ -2057,10 +2057,11 @@ class KimiLinearMoE(nn.Module):
|
||||
and not plan.use_native
|
||||
and not plan.use_marlin
|
||||
):
|
||||
if not plan.use_trtllm:
|
||||
if not (plan.use_trtllm or plan.use_cutedsl):
|
||||
raise RuntimeError(
|
||||
"Kimi-K3 MXFP4 SiTU MoE requires the native, FlashInfer "
|
||||
"TRT-LLM, or Marlin (Hopper W4A16) backend; no portable SiTU "
|
||||
"Kimi-K3 SiTU MoE requires the native, FlashInfer TRT-LLM, "
|
||||
"FlashInfer CuTe DSL (NVFP4), or Marlin (Hopper W4A16) "
|
||||
"backend; no portable SiTU "
|
||||
f"Triton fallback exists (selected MoE backend: "
|
||||
f"{moe_backend.value!r})."
|
||||
)
|
||||
@@ -2112,7 +2113,7 @@ class KimiLinearMoE(nn.Module):
|
||||
else (
|
||||
_k3_trtllm_situ_internal_activation_dtype(quant_config, prefix)
|
||||
if self.execution_plan.use_trtllm
|
||||
else None
|
||||
else "input" if self.execution_plan.use_cutedsl else None
|
||||
)
|
||||
),
|
||||
)
|
||||
@@ -2424,10 +2425,9 @@ class KimiLinearMoE(nn.Module):
|
||||
|
||||
def process_weights_after_loading(self, module) -> None:
|
||||
"""Configure the latent projection from the processed expert input scale."""
|
||||
if (
|
||||
self.experts.plan["weight_dtype"] == "nvfp4"
|
||||
and self.experts.plan["solution"] == "flashinfer_trtllm"
|
||||
):
|
||||
if self.experts.plan["weight_dtype"] == "nvfp4" and self.experts.plan[
|
||||
"solution"
|
||||
] in ("flashinfer_trtllm", "flashinfer_cutedsl"):
|
||||
# The loader visits this parent before its expert child.
|
||||
self.experts.process_weights_after_loading(self.experts)
|
||||
self.routed_expert_down_proj.prepare_nvfp4_output(
|
||||
@@ -2448,11 +2448,12 @@ class KimiLinearMoE(nn.Module):
|
||||
plan.owns_ep_communication
|
||||
or plan.use_native
|
||||
or plan.use_trtllm
|
||||
or plan.use_cutedsl
|
||||
or plan.use_marlin
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Kimi-K3 has no portable SiTU Triton fallback; use the native, "
|
||||
"FlashInfer TRT-LLM, or Marlin SiTU MoE path."
|
||||
"FlashInfer TRT-LLM, FlashInfer CuTe DSL, or Marlin SiTU MoE path."
|
||||
)
|
||||
out = self.experts(
|
||||
hidden_states=routed_in,
|
||||
|
||||
@@ -291,6 +291,22 @@ def test_kimi3_moe_execution_policy_honours_a_forced_marlin_backend() -> None:
|
||||
assert not plan.use_native
|
||||
|
||||
|
||||
def test_kimi3_moe_execution_policy_honours_a_forced_cutedsl_backend() -> None:
|
||||
"""Asked for FlashInfer's CuTe DSL experts, the plan routes to them alone."""
|
||||
with mock.patch.object(
|
||||
latent_module, "native_latent_moe_available", return_value=False
|
||||
):
|
||||
plan = Kimi3MoEExecutionPlan.build(
|
||||
_plan_mapping(), MoeBackend.FLASHINFER_CUTEDSL, alt_stream=None
|
||||
)
|
||||
|
||||
assert plan.use_cutedsl
|
||||
assert not plan.use_trtllm
|
||||
assert not plan.use_marlin
|
||||
assert not plan.use_native
|
||||
assert not plan.owns_ep_communication
|
||||
|
||||
|
||||
def test_kimi3_moe_execution_policy_takes_marlin_when_its_probe_says_yes() -> None:
|
||||
"""The probe decides AUTO, and the contraction shard's gate rides on it."""
|
||||
with (
|
||||
|
||||
@@ -300,6 +300,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
)
|
||||
@@ -326,6 +327,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
)
|
||||
@@ -370,6 +372,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
),
|
||||
@@ -389,6 +392,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=False,
|
||||
use_trtllm=False,
|
||||
use_cutedsl=False,
|
||||
use_marlin=True,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
@@ -413,6 +417,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
),
|
||||
@@ -1002,6 +1007,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=True,
|
||||
use_trtllm=False,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=True,
|
||||
),
|
||||
@@ -1092,6 +1098,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=True,
|
||||
use_trtllm=False,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
),
|
||||
@@ -1134,6 +1141,7 @@ class TestKimiK3RegistrationTests:
|
||||
owns_ep_communication=False,
|
||||
use_native=True,
|
||||
use_trtllm=False,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=True,
|
||||
),
|
||||
|
||||
@@ -136,6 +136,7 @@ def test_k3_accepts_a_placement_only_on_the_attention_dp_megamoe_path(
|
||||
trtllm = Kimi3MoEExecutionPlan(
|
||||
use_native=False,
|
||||
use_trtllm=True,
|
||||
use_cutedsl=False,
|
||||
use_mega_moe=False,
|
||||
owns_ep_communication=False,
|
||||
overlap_shared_experts=False,
|
||||
|
||||
@@ -99,6 +99,7 @@ def test_attn_dp_replicates_dense_weights_and_selects_transport(
|
||||
owns_ep_communication=mega_moe,
|
||||
use_native=False,
|
||||
use_trtllm=not mega_moe,
|
||||
use_cutedsl=False,
|
||||
overlap_shared_experts=False,
|
||||
joint_moe_reduce=False,
|
||||
)
|
||||
|
||||
@@ -264,6 +264,25 @@ expert ids. With FP32 route weights the routed kernel stops before its
|
||||
finalize and `moe_finalize_fuse_shared` combines the per-route outputs with the
|
||||
FP32 weights, accumulating in FP32 and rounding once to BF16.
|
||||
|
||||
## FlashInfer CuTe DSL NVFP4 SiTU (`flashinfer_cutedsl.py`)
|
||||
|
||||
`flashinfer_cutedsl_nvfp4_situ_routed_moe_apply` runs Kimi-K3 NVFP4 SiTU
|
||||
experts on FlashInfer's CuTe DSL fused MoE (`--moe-backend
|
||||
flashinfer_cutedsl`); `auto` keeps the TRT-LLM-Gen kernel. The preprocessor
|
||||
converts the checkpoint layout: W13 rows `[gate | up]` become `[up | gate]`
|
||||
interleaved in 64-row groups, and the linear E4M3 block scales are swizzled
|
||||
per expert. The scale storage stays expert-major and the kernels read it
|
||||
through an MMA-layout view, so DWDP streams these experts like the TRT-LLM
|
||||
layout. FC1 alpha is `a13 * w13_scale_2` and FC2 alpha `a2 * w2_scale_2`. The
|
||||
kernel quantizes the GEMM2 input with one global scale (`1 / a2`), so the
|
||||
preprocessor rejects checkpoints whose per-expert `a2` differ (Kimi-K3-NVFP4
|
||||
ships 1.0 everywhere).
|
||||
|
||||
The apply uses one fixed tactic (256-row routing tiles, 2-CTA clusters) and
|
||||
the fused, atomic finalize; it always finalizes. On GB300 at K3 prefill shapes
|
||||
(896 experts, top-16, uniform routing) the routed MoE layer is 5-6% faster than
|
||||
TRT-LLM-Gen at 16K rows and 11% faster at 24K rows beside a peer weight copy.
|
||||
|
||||
## FP8 per-channel experts
|
||||
|
||||
Per-channel FP8 experts store weights in FP8 E4M3 with FP32 scales per output
|
||||
|
||||
@@ -2107,6 +2107,7 @@ import tokenspeed_kernel.ops.moe.cute_dsl # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.deep_ep # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.deep_gemm # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.flashinfer # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.flashinfer_cutedsl # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.gluon # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.marlin # noqa: E402,F401
|
||||
import tokenspeed_kernel.ops.moe.native # noqa: E402,F401
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""FlashInfer CuTe DSL NVFP4 routed MoE with the SiTU activation (precomputed top-k).
|
||||
|
||||
The preprocessor turns NVFP4 checkpoint experts (``[gate | up]`` W13 rows,
|
||||
linear E4M3 block scales) into the CuTe DSL layout: W13 rows become
|
||||
``[up | gate]`` interleaved in 64-row groups and every block-scale matrix is
|
||||
swizzled. The kernels read the scales through an MMA-layout view of that
|
||||
expert-major storage, so the peer-weight store can stream experts as it does
|
||||
the TRT-LLM layout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from tokenspeed_kernel.platform import (
|
||||
ArchVersion,
|
||||
CapabilityRequirement,
|
||||
current_platform,
|
||||
)
|
||||
from tokenspeed_kernel.registry import Priority, register_kernel
|
||||
from tokenspeed_kernel.signature import format_signatures
|
||||
|
||||
platform = current_platform()
|
||||
|
||||
_SF_VEC = 16
|
||||
_W13_ROW_GROUP = 64
|
||||
# One fixed tactic for both GEMMs (256-row routing tiles, 2-CTA clusters), so
|
||||
# each kernel compiles once, during the startup forward, for any row count.
|
||||
_TACTIC = (256, ((256, 256), (2, 1), False), ((256, 256), (2, 1), False))
|
||||
|
||||
_STREAMED = (
|
||||
"cutedsl_w1_weight",
|
||||
"cutedsl_w1_weight_sf",
|
||||
"cutedsl_w2_weight",
|
||||
"cutedsl_w2_weight_sf",
|
||||
)
|
||||
_REPLICATED = ("cutedsl_w1_alpha", "cutedsl_w2_alpha")
|
||||
|
||||
|
||||
if platform.is_nvidia:
|
||||
from tokenspeed_kernel.ops.moe.flashinfer import _positive_situ_value
|
||||
|
||||
def _w13_to_cute_rows(t: torch.Tensor) -> torch.Tensor:
|
||||
"""``[E, 2I, X]`` rows ``[gate | up]`` -> ``[up | gate]`` interleaved by 64 rows."""
|
||||
experts, rows, cols = t.shape
|
||||
halves = t.view(experts, 2, rows // (2 * _W13_ROW_GROUP), _W13_ROW_GROUP, cols)
|
||||
return halves.flip(1).transpose(1, 2).reshape(experts, rows, cols).contiguous()
|
||||
|
||||
def _swizzle_block_scales(linear: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-expert 128x4 swizzle of ``[E, rows, cols]`` E4M3 block scales (bytes)."""
|
||||
from flashinfer import nvfp4_block_scale_interleave
|
||||
|
||||
experts, rows, cols = linear.shape
|
||||
return torch.stack(
|
||||
[
|
||||
nvfp4_block_scale_interleave(linear[e].contiguous()).view(rows, cols)
|
||||
for e in range(experts)
|
||||
]
|
||||
)
|
||||
|
||||
def flashinfer_cutedsl_nvfp4_situ_moe_weights(plan: dict, w: torch.nn.Module):
|
||||
"""Convert NVFP4 checkpoint experts to the CuTe DSL layout.
|
||||
|
||||
Args:
|
||||
plan: The selected MoE plan.
|
||||
w: Expert module holding ``w13_*`` / ``w2_*`` NVFP4 parameters.
|
||||
|
||||
Returns:
|
||||
The expert-major processed weights for the peer-weight store.
|
||||
"""
|
||||
from tokenspeed_kernel.ops.communication.peer_weights import (
|
||||
ExpertWeightShard,
|
||||
PreparedExpertWeights,
|
||||
)
|
||||
|
||||
ws2 = w.w13_weight_scale_2
|
||||
gate_ws2, up_ws2 = (ws2[:, 0], ws2[:, 1]) if ws2.dim() == 2 else (ws2, ws2)
|
||||
if not torch.equal(gate_ws2, up_ws2):
|
||||
raise RuntimeError(
|
||||
"NVFP4 SiTU requires equal gate/up weight_scale_2 per expert."
|
||||
)
|
||||
w2_input_scale = w.w2_input_scale.to(torch.float32)
|
||||
if not torch.all(w2_input_scale == w2_input_scale[0]):
|
||||
raise RuntimeError(
|
||||
"CuTe DSL NVFP4 MoE quantizes the GEMM2 input with one global "
|
||||
"scale; the checkpoint has per-expert values."
|
||||
)
|
||||
w13_input_scale = w.w13_input_scale.to(torch.float32)
|
||||
|
||||
w1 = _w13_to_cute_rows(w.w13_weight.data)
|
||||
w1_sf = _swizzle_block_scales(
|
||||
_w13_to_cute_rows(w.w13_weight_scale.data.view(torch.uint8))
|
||||
)
|
||||
w2 = w.w2_weight.data.contiguous()
|
||||
w2_sf = _swizzle_block_scales(w.w2_weight_scale.data.view(torch.uint8))
|
||||
|
||||
def param(t: torch.Tensor) -> torch.nn.Parameter:
|
||||
return torch.nn.Parameter(t, requires_grad=False)
|
||||
|
||||
w.cutedsl_w1_weight = param(w1)
|
||||
w.cutedsl_w1_weight_sf = param(w1_sf)
|
||||
w.cutedsl_w2_weight = param(w2)
|
||||
w.cutedsl_w2_weight_sf = param(w2_sf)
|
||||
w.cutedsl_w1_alpha = param((w13_input_scale * gate_ws2).contiguous())
|
||||
w.cutedsl_w2_alpha = param(
|
||||
(w2_input_scale * w.w2_weight_scale_2.to(torch.float32)).contiguous()
|
||||
)
|
||||
w.cutedsl_fc2_input_scale = param((1.0 / w2_input_scale[:1]).contiguous())
|
||||
w.w13_input_scale_quant = param((1.0 / w13_input_scale).contiguous())
|
||||
w.cutedsl_situ_beta = _positive_situ_value(
|
||||
w, "activation_situ_beta", "situ_beta"
|
||||
)
|
||||
w.cutedsl_situ_linear_beta = _positive_situ_value(
|
||||
w, "activation_situ_linear_beta", "situ_linear_beta"
|
||||
)
|
||||
|
||||
for name in (
|
||||
"w13_weight",
|
||||
"w2_weight",
|
||||
"w13_weight_scale",
|
||||
"w2_weight_scale",
|
||||
"w13_weight_scale_2",
|
||||
"w2_weight_scale_2",
|
||||
"w13_input_scale",
|
||||
"w2_input_scale",
|
||||
):
|
||||
delattr(w, name)
|
||||
|
||||
parameters = dict(w.named_parameters(recurse=False))
|
||||
return PreparedExpertWeights(
|
||||
layout="flashinfer-cutedsl-nvfp4-situ-v1",
|
||||
shards=tuple(
|
||||
ExpertWeightShard(name, parameters[name], name in _REPLICATED)
|
||||
for name in (*_STREAMED, *_REPLICATED)
|
||||
),
|
||||
)
|
||||
|
||||
@register_kernel(
|
||||
"moe",
|
||||
"apply",
|
||||
name="flashinfer_cutedsl_nvfp4_situ_routed_moe_apply",
|
||||
solution="flashinfer_cutedsl",
|
||||
weight_preprocessor=flashinfer_cutedsl_nvfp4_situ_moe_weights,
|
||||
capability=CapabilityRequirement(
|
||||
vendors=frozenset({"nvidia"}),
|
||||
min_arch_version=ArchVersion(10, 0),
|
||||
max_arch_version=ArchVersion(10, 7),
|
||||
),
|
||||
signatures=format_signatures("x", "dense", {torch.bfloat16, torch.uint8}),
|
||||
traits={
|
||||
"weight_dtype": frozenset({"nvfp4"}),
|
||||
"activation": frozenset({"situ"}),
|
||||
"routing_mode": frozenset({"precomputed_topk"}),
|
||||
"supports_deferred_finalize": frozenset({False}),
|
||||
"supports_ep": frozenset({False}),
|
||||
"supports_all_to_all_ep": frozenset({False}),
|
||||
"ispp_alignment": frozenset({128}),
|
||||
"internal_activation_dtype": frozenset({"input"}),
|
||||
"supports_bias": frozenset({False}),
|
||||
},
|
||||
priority=Priority.SPECIALIZED - 1,
|
||||
)
|
||||
def flashinfer_cutedsl_nvfp4_situ_routed_moe_apply(
|
||||
plan: dict,
|
||||
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
w: torch.nn.Module,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
num_tokens_global: int | None = None,
|
||||
max_num_tokens_per_gpu: int | None = None,
|
||||
do_finalize: bool = True,
|
||||
enable_pdl: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Routed NVFP4 SiTU experts on precomputed top-k, finalized in the kernel.
|
||||
|
||||
Args:
|
||||
plan: The selected MoE plan.
|
||||
x: BF16 ``[tokens, hidden]`` activations, or an NVFP4 pair of packed
|
||||
``[tokens, hidden / 2]`` codes and linear ``[tokens, hidden / 16]``
|
||||
block scales quantized with ``w.w13_input_scale_quant``.
|
||||
w: Expert module prepared by the preprocessor.
|
||||
router_logits: Unused; routing is precomputed.
|
||||
topk_weights: ``[tokens, top_k]`` FP32 route weights.
|
||||
topk_ids: ``[tokens, top_k]`` expert ids.
|
||||
num_tokens_global: Unused.
|
||||
max_num_tokens_per_gpu: Unused.
|
||||
do_finalize: Must be True.
|
||||
enable_pdl: Whether the kernels use programmatic dependent launch.
|
||||
|
||||
Returns:
|
||||
The BF16 ``[tokens, hidden]`` weighted sum over each token's experts.
|
||||
"""
|
||||
from flashinfer import fp4_quantize
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
from flashinfer.fused_moe.cute_dsl.fused_moe import cute_dsl_fused_moe
|
||||
from flashinfer.tllm_enums import ActivationType
|
||||
|
||||
if topk_weights is None or topk_ids is None:
|
||||
raise ValueError("precomputed_topk plan requires topk_weights and topk_ids")
|
||||
if not do_finalize:
|
||||
raise ValueError("CuTe DSL NVFP4 SiTU MoE always finalizes")
|
||||
if isinstance(x, tuple):
|
||||
x_fp4, x_sf = x
|
||||
else:
|
||||
x_fp4, x_sf = fp4_quantize(
|
||||
x,
|
||||
w.w13_input_scale_quant,
|
||||
sf_vec_size=_SF_VEC,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
experts, w13_rows, half_hidden = w.cutedsl_w1_weight.shape
|
||||
hidden = half_hidden * 2
|
||||
w1_sf = convert_sf_to_mma_layout(
|
||||
w.cutedsl_w1_weight_sf.view(experts * w13_rows, -1),
|
||||
m=w13_rows,
|
||||
k=hidden,
|
||||
num_groups=experts,
|
||||
sf_vec_size=_SF_VEC,
|
||||
)
|
||||
w2_sf = convert_sf_to_mma_layout(
|
||||
w.cutedsl_w2_weight_sf.view(experts * hidden, -1),
|
||||
m=hidden,
|
||||
k=w13_rows // 2,
|
||||
num_groups=experts,
|
||||
sf_vec_size=_SF_VEC,
|
||||
)
|
||||
return cute_dsl_fused_moe(
|
||||
x_fp4,
|
||||
x_sf.view(torch.uint8).unsqueeze(-1),
|
||||
topk_ids.to(torch.int32),
|
||||
topk_weights,
|
||||
w.cutedsl_w1_weight,
|
||||
w1_sf,
|
||||
w.cutedsl_w1_alpha,
|
||||
w.cutedsl_fc2_input_scale,
|
||||
w.cutedsl_w2_weight,
|
||||
w2_sf,
|
||||
w.cutedsl_w2_alpha,
|
||||
num_experts=experts,
|
||||
top_k=topk_ids.shape[1],
|
||||
use_fused_finalize=True,
|
||||
enable_pdl=enable_pdl,
|
||||
activation_type=ActivationType.Swiglu.value,
|
||||
situ_beta=w.cutedsl_situ_beta,
|
||||
situ_linear_beta=w.cutedsl_situ_linear_beta,
|
||||
tactic=_TACTIC,
|
||||
)
|
||||
@@ -0,0 +1,324 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""flashinfer CuTe DSL NVFP4 SiTU MoE (w4a4) on NVFP4 checkpoint experts.
|
||||
|
||||
The preprocessor's layout is checked bytewise against FlashInfer's own
|
||||
``prepare_cute_dsl_weights``; the registered apply against an fp32 SiTU MoE
|
||||
over the exactly-dequantized weights (both activation quantizations modeled),
|
||||
anchored to the TRT-LLM-Gen SiTU kernel on the same checkpoint.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tokenspeed_kernel.numerics.torch.activation import torch_situ_and_mul
|
||||
from tokenspeed_kernel.numerics.torch.moe import DequantizedExperts, torch_moe_experts
|
||||
from tokenspeed_kernel.numerics.torch.quantization import dequantize_nvfp4_blocks
|
||||
from tokenspeed_kernel.ops.moe import moe_plan as kernel_moe_plan
|
||||
|
||||
NUM_EXPERTS = 16
|
||||
TOP_K = 8
|
||||
HIDDEN = 256
|
||||
ISPP = 256 # the CuTe DSL tactic tiles 2 * ISPP and HIDDEN by 256
|
||||
SITU_BETA = 4.0
|
||||
SITU_LINEAR_BETA = 25.0
|
||||
_FP8_E4M3_MAX = 448.0
|
||||
_FP4_E2M1_MAX = 6.0
|
||||
|
||||
|
||||
def _runtime_reason() -> str | None:
|
||||
if not torch.cuda.is_available():
|
||||
return "requires CUDA"
|
||||
import tokenspeed_kernel.ops.moe.flashinfer_cutedsl # noqa: F401
|
||||
from tokenspeed_kernel.platform import current_platform
|
||||
from tokenspeed_kernel.registry import KernelRegistry
|
||||
|
||||
spec = KernelRegistry.get().get_by_name(
|
||||
"flashinfer_cutedsl_nvfp4_situ_routed_moe_apply"
|
||||
)
|
||||
if spec is None or not spec.capability.satisfied_by(current_platform()):
|
||||
return "outside the kernel's registered capability range"
|
||||
return None
|
||||
|
||||
|
||||
_reason = _runtime_reason()
|
||||
requires_cutedsl_situ = pytest.mark.skipif(_reason is not None, reason=str(_reason))
|
||||
|
||||
|
||||
def _quantize(w: torch.Tensor, unit_scale: bool):
|
||||
"""NVFP4 of a 2-D weight: packed codes, linear E4M3 scales, weight_scale_2."""
|
||||
from flashinfer import fp4_quantize
|
||||
|
||||
w = w.cuda().to(torch.bfloat16)
|
||||
global_scale = (
|
||||
torch.ones((), device="cuda")
|
||||
if unit_scale
|
||||
else _FP8_E4M3_MAX * _FP4_E2M1_MAX / w.float().abs().amax().clamp(min=1e-12)
|
||||
).reshape(1)
|
||||
packed, sf = fp4_quantize(w, global_scale, is_sf_swizzled_layout=False)
|
||||
m, k = w.shape
|
||||
sf = sf.reshape(-1)[: m * (k // 16)].view(m, k // 16).view(torch.float8_e4m3fn)
|
||||
return packed.view(m, k // 2), sf, (1.0 / global_scale).reshape(())
|
||||
|
||||
|
||||
def _checkpoint(w13_bf16, w2_bf16, *, unit_scale: bool) -> dict[str, torch.Tensor]:
|
||||
"""Loader-layout NVFP4 experts: W13 rows [gate | up], one scale_2 per expert."""
|
||||
q13 = [_quantize(w, unit_scale) for w in w13_bf16]
|
||||
q2 = [_quantize(w, unit_scale) for w in w2_bf16]
|
||||
return {
|
||||
"w13_weight": torch.stack([p for p, _, _ in q13]),
|
||||
"w13_weight_scale": torch.stack([s for _, s, _ in q13]),
|
||||
"w13_weight_scale_2": torch.stack([g for _, _, g in q13]),
|
||||
"w2_weight": torch.stack([p for p, _, _ in q2]),
|
||||
"w2_weight_scale": torch.stack([s for _, s, _ in q2]),
|
||||
"w2_weight_scale_2": torch.stack([g for _, _, g in q2]),
|
||||
"w13_input_scale": torch.ones(1, device="cuda"),
|
||||
"w2_input_scale": torch.ones(NUM_EXPERTS, device="cuda"),
|
||||
}
|
||||
|
||||
|
||||
class _Experts(torch.nn.Module):
|
||||
"""Minimal module carrying what the preprocessors and applies consume."""
|
||||
|
||||
def __init__(self, raw: dict[str, torch.Tensor]) -> None:
|
||||
from types import SimpleNamespace
|
||||
|
||||
from flashinfer.fused_moe import RoutingMethodType
|
||||
|
||||
super().__init__()
|
||||
for name, value in raw.items():
|
||||
self.register_parameter(
|
||||
name, torch.nn.Parameter(value.clone(), requires_grad=False)
|
||||
)
|
||||
self.activation_situ_beta = SITU_BETA
|
||||
self.activation_situ_linear_beta = SITU_LINEAR_BETA
|
||||
self._routing_method_type = RoutingMethodType.RenormalizeNaive
|
||||
self._routing_logits_dtype = torch.bfloat16
|
||||
self._spec = SimpleNamespace(
|
||||
num_experts=NUM_EXPERTS,
|
||||
num_local_experts=NUM_EXPERTS,
|
||||
top_k=TOP_K,
|
||||
ep_rank=0,
|
||||
)
|
||||
|
||||
|
||||
def _weights(generator: torch.Generator):
|
||||
w13 = torch.randn(NUM_EXPERTS, 2 * ISPP, HIDDEN, generator=generator) * 0.5
|
||||
w2 = torch.randn(NUM_EXPERTS, HIDDEN, ISPP, generator=generator) * 0.5
|
||||
return w13.bfloat16(), w2.bfloat16()
|
||||
|
||||
|
||||
def _routing(generator: torch.Generator, num_tokens: int):
|
||||
ids = torch.stack(
|
||||
[
|
||||
torch.randperm(NUM_EXPERTS, generator=generator)[:TOP_K]
|
||||
for _ in range(num_tokens)
|
||||
]
|
||||
)
|
||||
weights = torch.rand(num_tokens, TOP_K, generator=generator).softmax(dim=-1)
|
||||
return ids.to(torch.int32).cuda(), weights.cuda()
|
||||
|
||||
|
||||
def _quant_dequant(t: torch.Tensor) -> torch.Tensor:
|
||||
"""The kernels' NVFP4 activation quantization (unit global scale) in fp32."""
|
||||
packed, sf, scale_2 = _quantize(t, unit_scale=True)
|
||||
return dequantize_nvfp4_blocks(packed, sf) * scale_2
|
||||
|
||||
|
||||
def _reference(hidden_states, raw, topk_ids, topk_weights) -> torch.Tensor:
|
||||
def experts(prefix: str) -> DequantizedExperts:
|
||||
return DequantizedExperts(
|
||||
lambda packed, sf, scale_2: dequantize_nvfp4_blocks(packed, sf) * scale_2,
|
||||
raw[f"{prefix}_weight"],
|
||||
raw[f"{prefix}_weight_scale"],
|
||||
raw[f"{prefix}_weight_scale_2"],
|
||||
)
|
||||
|
||||
return torch_moe_experts(
|
||||
hidden_states,
|
||||
experts("w13"),
|
||||
experts("w2"),
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
activation=lambda gate_up: torch_situ_and_mul(
|
||||
gate_up, SITU_BETA, SITU_LINEAR_BETA
|
||||
),
|
||||
w13_bias=None,
|
||||
w2_bias=None,
|
||||
quantize_input=lambda t: _quant_dequant(t.float()),
|
||||
quantize_intermediate=_quant_dequant,
|
||||
round_expert_output=None,
|
||||
first_expert=0,
|
||||
)
|
||||
|
||||
|
||||
def _rel_l2(actual: torch.Tensor, expected: torch.Tensor) -> float:
|
||||
return ((actual.float() - expected.float()).norm() / expected.float().norm()).item()
|
||||
|
||||
|
||||
@requires_cutedsl_situ
|
||||
def test_preprocessor_layout_matches_flashinfer_prepare() -> None:
|
||||
"""Checkpoint experts land exactly where FlashInfer's own BF16 preparation
|
||||
puts them: W13 [up | gate] interleaved by 64 rows, swizzled scales."""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
from flashinfer.fused_moe.api import QuantConfig, QuantFormat
|
||||
from flashinfer.fused_moe.prepare import prepare_cute_dsl_weights
|
||||
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights,
|
||||
)
|
||||
|
||||
w13, w2 = _weights(torch.Generator().manual_seed(20261116))
|
||||
w = _Experts(_checkpoint(w13, w2, unit_scale=True)).cuda()
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, w)
|
||||
up_gate = torch.cat([w13[:, ISPP:], w13[:, :ISPP]], dim=1).cuda()
|
||||
ref = prepare_cute_dsl_weights(
|
||||
up_gate,
|
||||
w2.cuda(),
|
||||
quant=QuantConfig(weight=QuantFormat.NVFP4, activation=QuantFormat.NVFP4),
|
||||
num_local_experts=NUM_EXPERTS,
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size=ISPP,
|
||||
)
|
||||
|
||||
def mma_view(sf: torch.Tensor, m: int, k: int) -> torch.Tensor:
|
||||
return convert_sf_to_mma_layout(
|
||||
sf.view(NUM_EXPERTS * m, -1), m=m, k=k, num_groups=NUM_EXPERTS
|
||||
)
|
||||
|
||||
assert torch.equal(w.cutedsl_w1_weight, ref["w1_weight"].view(torch.uint8))
|
||||
assert torch.equal(w.cutedsl_w2_weight, ref["w2_weight"].view(torch.uint8))
|
||||
assert torch.equal(
|
||||
mma_view(w.cutedsl_w1_weight_sf, 2 * ISPP, HIDDEN).view(torch.uint8),
|
||||
ref["w1_weight_sf"].view(torch.uint8),
|
||||
)
|
||||
assert torch.equal(
|
||||
mma_view(w.cutedsl_w2_weight_sf, HIDDEN, ISPP).view(torch.uint8),
|
||||
ref["w2_weight_sf"].view(torch.uint8),
|
||||
)
|
||||
|
||||
|
||||
@requires_cutedsl_situ
|
||||
def test_routed_moe_matches_dequant_reference_and_trtllm() -> None:
|
||||
from flashinfer import fp4_quantize
|
||||
from tokenspeed_kernel.ops.moe.flashinfer import (
|
||||
flashinfer_trtllm_nvfp4_situ_moe_weights,
|
||||
flashinfer_trtllm_nvfp4_situ_routed_moe_apply,
|
||||
)
|
||||
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights,
|
||||
flashinfer_cutedsl_nvfp4_situ_routed_moe_apply,
|
||||
)
|
||||
|
||||
generator = torch.Generator().manual_seed(20261117)
|
||||
num_tokens = 64
|
||||
w13, w2 = _weights(generator)
|
||||
raw = _checkpoint(w13, w2, unit_scale=False)
|
||||
hidden_states = (
|
||||
(torch.randn(num_tokens, HIDDEN, generator=generator) * 0.2).bfloat16().cuda()
|
||||
)
|
||||
topk_ids, topk_weights = _routing(generator, num_tokens)
|
||||
|
||||
cute = _Experts(raw).cuda()
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, cute)
|
||||
trtllm = _Experts(raw).cuda()
|
||||
flashinfer_trtllm_nvfp4_situ_moe_weights({}, trtllm)
|
||||
|
||||
def run_cute(x):
|
||||
return flashinfer_cutedsl_nvfp4_situ_routed_moe_apply(
|
||||
{},
|
||||
x,
|
||||
cute,
|
||||
router_logits=None,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
|
||||
actual = run_cute(hidden_states)
|
||||
serving = flashinfer_trtllm_nvfp4_situ_routed_moe_apply(
|
||||
{},
|
||||
hidden_states,
|
||||
trtllm,
|
||||
router_logits=None,
|
||||
topk_weights=topk_weights.bfloat16(),
|
||||
topk_ids=topk_ids,
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
expected = _reference(hidden_states, raw, topk_ids, topk_weights)
|
||||
cute_err, serving_err = _rel_l2(actual, expected), _rel_l2(serving, expected)
|
||||
assert cute_err < 0.06, f"{cute_err=:.4f}"
|
||||
assert cute_err < 1.5 * serving_err + 0.005, f"{cute_err=:.4f} {serving_err=:.4f}"
|
||||
|
||||
# An NVFP4 pair from the latent projection gives the BF16-input result.
|
||||
# The finalize adds experts with BF16 atomics in arrival order, so runs
|
||||
# differ by rounding (measured 4.5e-3 here); a wrong layout is O(1).
|
||||
x_fp4, x_sf = fp4_quantize(
|
||||
hidden_states, cute.w13_input_scale_quant, is_sf_swizzled_layout=False
|
||||
)
|
||||
prequantized = run_cute((x_fp4, x_sf))
|
||||
repeat = run_cute(hidden_states)
|
||||
torch.cuda.synchronize()
|
||||
assert _rel_l2(repeat, actual) < 1e-2
|
||||
assert _rel_l2(prequantized, actual) < 1e-2
|
||||
|
||||
|
||||
@requires_cutedsl_situ
|
||||
def test_preprocessor_rejects_per_expert_gemm2_input_scales() -> None:
|
||||
from tokenspeed_kernel.ops.moe.flashinfer_cutedsl import (
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights,
|
||||
)
|
||||
|
||||
w13, w2 = _weights(torch.Generator().manual_seed(20261118))
|
||||
raw = _checkpoint(w13, w2, unit_scale=False)
|
||||
raw["w2_input_scale"] = torch.linspace(0.5, 1.0, NUM_EXPERTS, device="cuda")
|
||||
with pytest.raises(RuntimeError, match="one global"):
|
||||
flashinfer_cutedsl_nvfp4_situ_moe_weights({}, _Experts(raw).cuda())
|
||||
|
||||
|
||||
@requires_cutedsl_situ
|
||||
@pytest.mark.parametrize(
|
||||
"solution,kernel",
|
||||
[
|
||||
("flashinfer_cutedsl", "flashinfer_cutedsl_nvfp4_situ_routed_moe_apply"),
|
||||
(None, "flashinfer_trtllm_nvfp4_situ_routed_moe_apply"),
|
||||
],
|
||||
)
|
||||
def test_plan_selects_cutedsl_only_when_requested(solution, kernel) -> None:
|
||||
plan = kernel_moe_plan(
|
||||
"nvfp4",
|
||||
input_dtype=torch.bfloat16,
|
||||
activation="situ",
|
||||
routing_mode="precomputed_topk",
|
||||
ep_size=1,
|
||||
ispp=3072,
|
||||
internal_activation_dtype="input",
|
||||
hidden=3584,
|
||||
swiglu_form=None,
|
||||
activation_clamped=False,
|
||||
expert_id_repeats=False,
|
||||
num_experts=896,
|
||||
top_k=16,
|
||||
fast_math=True,
|
||||
combine_order="rank",
|
||||
features=frozenset(),
|
||||
solution=solution,
|
||||
)
|
||||
assert plan["apply_kernel_name"] == kernel
|
||||
Reference in New Issue
Block a user