"""Kernel dispatch for the NVFP4 MoE GEMMs: which kernel runs, with which tiles and switches, and its launch. The kernels are templates (:mod:`freetoken.kernel.triton.nvfp4_fused_moe`, :mod:`freetoken.kernel.gluon.nvfp4.moe.sm120`, :mod:`freetoken.kernel.gluon.nvfp4.moe.sm120_w4a4`); :mod:`freetoken.moe.fused_nvfp4` strings the GEMMs into the MoE layer. """ from __future__ import annotations import functools import re from dataclasses import dataclass, replace from typing import Any, Dict, Tuple import torch import triton import triton.language as tl from triton.tools.tensor_descriptor import TensorDescriptor from freetoken.kernel.backend import is_rocm from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native from freetoken.kernel.triton.nvfp4_fused_moe import ( _decode_nvfp4_marlin_kernel, _decode_nvfp4_moe_kernel, _e2m1_lut, _prefill_nvfp4_moe_kernel, ) from freetoken.utils import init_logger logger = init_logger(__name__) # Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune, # which benchmarks at run time). Tuned offline against the NVFP4 decode kernels. # These drive the original LUT-gather decode (decode_gemm), kept only for A/B. _DECODE_BLOCK_N = 64 _DECODE_BLOCK_KB = 128 _DECODE_WARPS = 4 # Marlin-style decode: every K-loop load must keep the word tile's register layout, so BLOCK_N * BLOCK_KW <= 128 * num_warps and BLOCK_KW <= 32 at 4 warps. # Long K (gate_up) uses the narrow N tile for more programs at M=1; short K (down) uses the wide one. _DECODE_MARLIN_BLOCK_N = 16 _DECODE_MARLIN_BLOCK_KW = 16 _DECODE_MARLIN_UNROLL = 2 _DECODE_MARLIN_WARPS = 4 _DECODE_MARLIN_DEEPK_BLOCK_N = 8 _DECODE_MARLIN_DEEPK_BLOCK_KW = 32 _DECODE_MARLIN_DEEPK_UNROLL = 2 _DECODE_MARLIN_DEEPK_THRESHOLD = 2048 def _tl_dtype(dt: torch.dtype): if dt == torch.bfloat16: return tl.bfloat16 if dt == torch.float16: return tl.float16 return tl.float32 def decode_gemm( a: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, glob: torch.Tensor, c: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, mul_routed_weight: bool, a_row_is_route: bool, ) -> None: M, top_k = topk_ids.shape N = packed.shape[1] K = packed.shape[2] * 2 scale = e4m3_kernel_view(scale) total_routes = M * top_k grid = (total_routes, triton.cdiv(N, _DECODE_BLOCK_N)) _decode_nvfp4_moe_kernel[grid]( a, packed, scale, glob, c, topk_weights, topk_ids, _e2m1_lut(a.device.index), total_routes, N, K, a.stride(0), a.stride(1), packed.stride(0), packed.stride(1), packed.stride(2), scale.stride(0), scale.stride(1), scale.stride(2), glob.stride(0), glob.stride(1), c.stride(0), c.stride(1), c.stride(2), topk_weights.stride(0), topk_weights.stride(1), topk_ids.stride(0), topk_ids.stride(1), BLOCK_SIZE_N=_DECODE_BLOCK_N, BLOCK_SIZE_KB=_DECODE_BLOCK_KB, TOP_K=top_k, A_ROW_IS_ROUTE=a_row_is_route, MUL_ROUTED_WEIGHT=mul_routed_weight, compute_type=_tl_dtype(c.dtype), num_warps=_DECODE_WARPS, ) def decode_gemm_marlin( a: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, glob: torch.Tensor, c: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, mul_routed_weight: bool, a_row_is_route: bool, ) -> None: """Marlin-style decode GEMV: int32 wide loads + deferred reduction (:func:`_decode_nvfp4_marlin_kernel`). ``packed`` is the uint8 ``[S, N, K//2]`` bank; it is reinterpreted as int32 ``[S, N, K//8]`` (contiguous, K%8==0 for NVFP4).""" M, top_k = topk_ids.shape N = packed.shape[1] K = packed.shape[2] * 2 packed_i32 = packed.view(torch.int32) # [S, N, K // 8] scale = e4m3_kernel_view(scale) total_routes = M * top_k deep_k = K >= _DECODE_MARLIN_DEEPK_THRESHOLD block_n = _DECODE_MARLIN_DEEPK_BLOCK_N if deep_k else _DECODE_MARLIN_BLOCK_N block_kw = _DECODE_MARLIN_DEEPK_BLOCK_KW if deep_k else _DECODE_MARLIN_BLOCK_KW unroll = _DECODE_MARLIN_DEEPK_UNROLL if deep_k else _DECODE_MARLIN_UNROLL # int64 activation loads need contiguous 16-bit rows that start 16-byte aligned a_wide = ( a.dtype in (torch.bfloat16, torch.float16) and a.stride(1) == 1 and a.stride(0) % 8 == 0 and a.data_ptr() % 16 == 0 ) grid = (total_routes, triton.cdiv(N, block_n)) _decode_nvfp4_marlin_kernel[grid]( a, packed_i32, scale, glob, c, topk_weights, topk_ids, total_routes, N, K, a.stride(0), a.stride(1), packed_i32.stride(0), packed_i32.stride(1), packed_i32.stride(2), scale.stride(0), scale.stride(1), scale.stride(2), glob.stride(0), glob.stride(1), c.stride(0), c.stride(1), c.stride(2), topk_weights.stride(0), topk_weights.stride(1), topk_ids.stride(0), topk_ids.stride(1), BLOCK_SIZE_N=block_n, BLOCK_SIZE_KW=block_kw, UNROLL=unroll, TOP_K=top_k, A_ROW_IS_ROUTE=a_row_is_route, A_WIDE=a_wide, MUL_ROUTED_WEIGHT=mul_routed_weight, compute_type=_tl_dtype(c.dtype), num_warps=_DECODE_MARLIN_WARPS, ) # rho = bf16 tensor-core FLOP / fp32 ALU FLOP per SM per clock, with fp32 accumulate (the only accumulator for bf16 MMA). # Each M block decodes its weight tile once on the ALUs, and the MMA also runs on the padding up to BLOCK_SIZE_M. # High rho makes decode the cost, so big blocks win; low rho makes padded MMA the cost, so small blocks win. # Spec, FLOP/clk/SM: GeForce RTX 30/40/50 512 / 256 = 2, as GeForce runs fp32 accumulate at half rate. # RTX A, RTX Ada Generation, RTX PRO Blackwell, A10, A40, L4, L40S 1024 / 256 = 4; A100 2048 / 128 = 16; H100 4096 / 256 = 16; H20 148 / 44 TFLOP/s = 3.4. # Measured (cuBLAS bf16 / Triton fp32 FMA): RTX PRO 6000 4.5, RTX 5090 2.3, RTX 4090 1.9, RTX 3080 Ti 1.6. _PRO_GPU_RE = re.compile( r"RTX\s*A\d{3,4}" # RTX A400 .. A6000 r"|RTX\s*\d{3,4}E?\s*(SFF\s*)?-?\s*Ada" # RTX 500 .. 6000 Ada Generation r"|RTX\s*PRO\s*\d{3,4}" # RTX PRO 500 .. 6000 Blackwell r"|RTX\s*\d{4}D\b" # RTX 6000D r"|Quadro|TITAN" r"|^(NVIDIA\s+)?(GRID\s+)?(A2|A10[GM]?|A16|A40|L2|L20|L4|L40S?)\b", re.I, ) # also matches names without "GeForce", e.g. "NVIDIA RTX 4090" _GEFORCE_GPU_RE = re.compile(r"GeForce|RTX\s*[2-6]0[5-9]0(?!\d)", re.I) def _tensor_rho(name: str, major: int, minor: int) -> float: if major not in (8, 12) or (major, minor) == (8, 0): return 3.4 if re.search(r"\bH20\b", name) else 16.0 if _PRO_GPU_RE.search(name): return 4.0 if _GEFORCE_GPU_RE.search(name): return 2.0 logger.warning("unknown GPU %r: NVFP4 MoE prefill assumes full-rate fp32-accumulate tensor cores", name) return 4.0 # BLOCK_SIZE_M -> (BLOCK_SIZE_N, BLOCK_SIZE_KB, num_stages) _PREFILL_TILES_H100 = {16: (64, 64, 3), 32: (64, 32, 3), 64: (64, 32, 3), 128: (64, 32, 3)} _PREFILL_TILES_RTX_PRO = {16: (64, 64, 4), 32: (128, 64, 3), 64: (128, 64, 2), 128: (64, 32, 2)} _PREFILL_TILES_GEFORCE = {16: (64, 64, 4), 32: (128, 64, 2), 64: (128, 32, 2)} @dataclass(frozen=True) class _PrefillTuning: """Triton prefill GEMM tuning of one GPU class.""" steps: Tuple[float, float, float] # tokens per expert where BLOCK_SIZE_M becomes 32, 64, 128 tiles: Dict[int, Tuple[int, int, int]] pair_decode_below: float = 0.0 # tokens per expert under which FAST_DECODE stays 0 small_tile_decode: int = 2 # FAST_DECODE of BLOCK_SIZE_M <= 32 large_tile_decode: int = 1 # FAST_DECODE of BLOCK_SIZE_M >= 64 scale_words: bool = False # SCALE_WORDS on BLOCK_SIZE_M 64 word-decode tiles long_k_warps: int = 4 # num_warps of SCALE_WORDS GEMMs with K >= _LONG_K _INF = float("inf") _LONG_K = 2048 _TUNING_H100 = _PrefillTuning((12, 24, 64), _PREFILL_TILES_H100) _TUNING_RTX_PRO = _PrefillTuning((16, 36, 150), _PREFILL_TILES_RTX_PRO) _TUNING_GEFORCE = _PrefillTuning((16, 150, _INF), _PREFILL_TILES_GEFORCE) # RTX 4090 A/B: up to 32 tokens per expert the pair decode streams the weights 2-18% faster, from 51 words win 11-14%, also on 32-row tiles _SM89_DECODE = dict(pair_decode_below=48, small_tile_decode=1) # H100 A/B: bytes beat words at BLOCK_SIZE_M 64 / 128 by 1-17% per layer (24-31% on Gemma-4-26B-A4B's K = 704 down GEMM) _SM90_DECODE = dict(small_tile_decode=2, large_tile_decode=2) # RTX 4090 / 3080 Ti layer A/B: scale words + 8 warps on the long-K GEMM give 1.03-1.09x / 1.05-1.19x; with them sm_86 gains from 64-row tiles at 100 tokens per expert, sm_89 does not _GEFORCE_SCALE_WORDS = dict(scale_words=True, long_k_warps=8) # (rho class, compute capability) -> tuning, from sweeps on H100, RTX PRO 6000 and RTX 5090 / 4090 / 3080 Ti; (class, None) covers the rest of the class. # The RTX tiles come from cc 8.6 / 8.9 / 12.0 sweeps (mma.sync, 99 KB shared memory); other GPUs keep the H100 tiles. _PREFILL_TUNING = { ("wide", (9, 0)): replace(_TUNING_H100, **_SM90_DECODE), ("wide", None): _TUNING_H100, ("pro", (8, 6)): _TUNING_RTX_PRO, ("pro", (8, 9)): replace(_TUNING_RTX_PRO, **_SM89_DECODE), ("pro", (9, 0)): replace(_TUNING_RTX_PRO, tiles=_PREFILL_TILES_H100, **_SM90_DECODE), ("pro", (12, 0)): _TUNING_RTX_PRO, ("pro", None): replace(_TUNING_RTX_PRO, tiles=_PREFILL_TILES_H100), ("geforce", (8, 6)): replace(_TUNING_GEFORCE, steps=(16, 100, _INF), **_GEFORCE_SCALE_WORDS), ("geforce", (8, 9)): replace(_TUNING_GEFORCE, **_SM89_DECODE, **_GEFORCE_SCALE_WORDS), ("geforce", (12, 0)): _TUNING_GEFORCE, ("geforce", None): replace(_TUNING_GEFORCE, tiles=_PREFILL_TILES_H100), } # steps while the Gluon kernel runs BLOCK_SIZE_M >= 64, from an RTX PRO 6000 sweep; classes without an entry keep the Triton steps _PREFILL_BLOCK_M_STEPS_GLUON = ((3.0, (16, 36, 96)),) # Gluon prefill GEMM tiles: BLOCK_SIZE_N -> (BLOCK_SIZE_K, setmaxnreg of the loader / decode warps) _GLUON_PREFILL_TILES = {128: (64, 96, 96), 256: (32, 88, 72)} def _gluon_prefill_config(N: int, block_m: int, mul_routed_weight: bool) -> Dict[str, Any]: # BLOCK_SIZE_M 64 tiles are short (few tokens per expert, often short K): persistent CTAs overlap one tile's epilogue and ring fill with the next tile, and a third x / decoded-weight stage absorbs DRAM latency persistent = block_m == 64 # a 256-row tile halves the token loads per weight byte, but a partial last tile still decodes all 256 rows # the persistent epilogue keeps one 64-bit address per element in registers, which spills with the routed-weight loads block_n = 256 if N % 256 == 0 and not (persistent and mul_routed_weight) else 128 block_k, load_regs, decode_regs = _GLUON_PREFILL_TILES[block_n] return dict(BLOCK_SIZE_M=block_m, BLOCK_SIZE_N=block_n, BLOCK_SIZE_K=block_k, GROUP_SIZE_M=8, STAGES=3 if persistent else 2, W_STAGES=4, PERSISTENT=persistent, DECODE_WARPS=4, MMA_WARPS_N=4, LOAD_REGS=load_regs, DECODE_REGS=decode_regs, num_warps=8) @functools.lru_cache(maxsize=None) def _gluon_prefill(device_index: int): """The Gluon module whose prefill GEMM replaces the Triton kernel on this GPU, or None.""" from freetoken.kernel import gluon if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available(): return None from freetoken.kernel.gluon.nvfp4.moe import sm120 return sm120 @functools.lru_cache(maxsize=None) def _device_rho(device_index: int) -> float: props = torch.cuda.get_device_properties(device_index) return _tensor_rho(props.name, props.major, props.minor) @functools.lru_cache(maxsize=None) def _prefill_tuning(device_index: int) -> _PrefillTuning: if is_rocm(): # TODO: tune for AMD GPUs; they keep the H100 tuning return _TUNING_H100 rho = _device_rho(device_index) rho_class = "wide" if rho >= 8.0 else "pro" if rho >= 3.0 else "geforce" cc = torch.cuda.get_device_capability(device_index) return _PREFILL_TUNING.get((rho_class, cc), _PREFILL_TUNING[(rho_class, None)]) @functools.lru_cache(maxsize=None) def _prefill_block_m_steps(device_index: int) -> Tuple[float, float, float]: steps = _prefill_tuning(device_index).steps if _gluon_prefill(device_index) is not None: rho = _device_rho(device_index) steps = next((s for min_rho, s in _PREFILL_BLOCK_M_STEPS_GLUON if rho >= min_rho), steps) return steps @functools.lru_cache(maxsize=None) def _tma_device(device_index: int) -> bool: return not is_rocm() and torch.cuda.get_device_capability(device_index) == (12, 0) def _prefill_use_tma(packed: torch.Tensor, scale: torch.Tensor, K: int, device_index: int) -> bool: # a TMA box row is >= 16 bytes and every global stride a multiple of 16 bytes, so K // 16 scale bytes need K % 256 == 0; # the TMA scale path reads the scales as fp8, so the forced e4m3 emulation stays on pointer loads return (_tma_device(device_index) and e4m3_native() and K % 256 == 0 and packed.is_contiguous() and scale.is_contiguous() and packed.data_ptr() % 16 == 0 and scale.data_ptr() % 16 == 0) def _prefill_use_scale_words(scale: torch.Tensor, K: int, cfg: Dict[str, Any], device_index: int) -> bool: # K // 16 scale bytes per row split into whole int32 words, and a K step reads at least one word return (_prefill_tuning(device_index).scale_words and cfg["BLOCK_SIZE_M"] == 64 and cfg["BLOCK_SIZE_KB"] >= 32 and K % 64 == 0 and scale.is_contiguous() and scale.data_ptr() % 4 == 0) def _prefill_fast_decode(block_m: int, per_expert: float, device_index: int) -> int: """FAST_DECODE of the prefill kernel: 0 keeps the fp16 pair decode (ROCm: the fast decode is PTX inline asm), 1 words, 2 bytes.""" if is_rocm(): return 0 tuning = _prefill_tuning(device_index) if per_expert < tuning.pair_decode_below: return 0 # the word decode halves the loop but stages the decoded tile through shared memory; small tiles keep the byte form return tuning.small_tile_decode if block_m <= 32 else tuning.large_tile_decode def prefill_config(M: int, top_k: int, num_experts: int, device_index: int) -> Dict[str, int]: # moe_align_block_size pads each expert's token count to a multiple of BLOCK_SIZE_M, so triton.autotune cannot pick it. to32, to64, to128 = _prefill_block_m_steps(device_index) per_expert = M * top_k / num_experts block_m = 128 if per_expert >= to128 else 64 if per_expert >= to64 else 32 if per_expert >= to32 else 16 block_n, block_kb, stages = _prefill_tuning(device_index).tiles[block_m] return dict(BLOCK_SIZE_M=block_m, BLOCK_SIZE_N=block_n, BLOCK_SIZE_KB=block_kb, GROUP_SIZE_M=8, num_warps=4, num_stages=stages, FAST_DECODE=_prefill_fast_decode(block_m, per_expert, device_index)) def prefill_gemm( a: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, glob: torch.Tensor, c: torch.Tensor, topk_weights_flat: torch.Tensor, sorted_ids: torch.Tensor, expert_ids: torch.Tensor, num_tokens_post_padded: torch.Tensor, num_valid_tokens: int, kernel_top_k: int, mul_routed_weight: bool, cfg: Dict[str, Any], ) -> None: gluon = _gluon_prefill(a.device.index) if gluon is not None and cfg["BLOCK_SIZE_M"] >= 64 and gluon.supports(a, packed, scale, c): gluon.prefill_gemm(a, packed, scale, glob, c, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded, num_valid_tokens, kernel_top_k, mul_routed_weight, _gluon_prefill_config(packed.shape[1], cfg["BLOCK_SIZE_M"], mul_routed_weight)) return N = packed.shape[1] K = packed.shape[2] * 2 EM = sorted_ids.shape[0] scale = e4m3_kernel_view(scale) grid = lambda META: ( # noqa: E731 triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), ) cfg = dict(cfg) fast = cfg.pop("FAST_DECODE", 0) if K % 8: fast = 0 if fast == 1: packed = packed.view(torch.int32) use_tma = _prefill_use_tma(packed, scale, K, a.device.index) # the 16-byte scale box costs registers that the BLOCK_SIZE_M >= 64 tiles do not have tma_scales = cfg["BLOCK_SIZE_M"] <= 32 scale_words = not use_tma and fast == 1 and _prefill_use_scale_words(scale, K, cfg, a.device.index) if scale_words: scale = scale.view(torch.int32) if K >= _LONG_K: cfg["num_warps"] = _prefill_tuning(a.device.index).long_k_warps w_desc = s_desc = None if use_tma: rows = packed.shape[0] * N box_k = cfg["BLOCK_SIZE_KB"] // 4 if fast == 1 else cfg["BLOCK_SIZE_KB"] w_desc = TensorDescriptor.from_tensor(packed.view(rows, packed.shape[2]), [cfg["BLOCK_SIZE_N"], box_k]) if tma_scales: s_desc = TensorDescriptor.from_tensor(scale.view(rows, K // 16), [cfg["BLOCK_SIZE_N"], 16]) _prefill_nvfp4_moe_kernel[grid]( a, packed, scale, glob, c, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded, w_desc, s_desc, N, K, EM, num_valid_tokens, a.stride(0), a.stride(1), packed.stride(0), packed.stride(1), packed.stride(2), scale.stride(0), scale.stride(1), scale.stride(2), glob.stride(0), glob.stride(1), c.stride(1), c.stride(2), topk_weights_flat.stride(0), MUL_ROUTED_WEIGHT=mul_routed_weight, top_k=kernel_top_k, compute_type=_tl_dtype(c.dtype), USE_TMA=use_tma, TMA_SCALES=tma_scales, FAST_DECODE=fast, SCALE_WORDS=scale_words, **cfg, ) @functools.lru_cache(maxsize=None) def _w4a4_prefill(device_index: int): """The Gluon module of the W4A4 prefill GEMMs on this GPU, or None.""" from freetoken.kernel import gluon if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available(): return None from freetoken.kernel.gluon.nvfp4.moe import sm120_w4a4 return sm120_w4a4 def w4a4_available(device_index: int) -> bool: return _w4a4_prefill(device_index) is not None # Below this many tokens per expert both prefills stream the weights at the same speed (RTX PRO 6000 and RTX 5090 sweeps over the # NVFP4 MoE shapes); W4A16 keeps the exact activations there _W4A4_MIN_TOKENS_PER_EXPERT = 4 # (min rho, tokens per expert where BLOCK_SIZE_M becomes 128), from the same sweeps _W4A4_BLOCK_M_STEPS = ((3.0, 128), (0.0, 64)) @functools.lru_cache(maxsize=None) def _w4a4_device(device_index: int) -> Tuple[float, int]: props = torch.cuda.get_device_properties(device_index) rho = _tensor_rho(props.name, props.major, props.minor) return next(t for min_rho, t in _W4A4_BLOCK_M_STEPS if rho >= min_rho), props.L2_cache_size def w4a4_prefill_config(M: int, top_k: int, num_experts: int, H: int, I: int, activation: str, dtype: torch.dtype, device_index: int) -> Dict[str, Any] | None: """Tiles of the W4A4 prefill GEMMs, or None where the W4A16 prefill runs instead.""" w4a4 = _w4a4_prefill(device_index) if w4a4 is None or dtype != torch.bfloat16 or activation not in w4a4.ACTIVATIONS or not w4a4.supports(H, I): return None to128, l2_bytes = _w4a4_device(device_index) per_expert = M * top_k / num_experts if per_expert < _W4A4_MIN_TOKENS_PER_EXPERT: return None block_m = 128 if per_expert >= to128 else 64 common = dict(BLOCK_SIZE_M=block_m, GROUP_SIZE_M=8, MMA_WARPS_N=4, LOAD_WARPS=4, num_warps=8) # the gated tile holds BN / 2 gate rows and their up rows, so I must split into whole half tiles gate_up = dict(common, BLOCK_SIZE_N=256 if I % 128 == 0 else 128, BLOCK_SIZE_K=128 if H % 128 == 0 else 64, N_GROUP=0, STAGES=3, LOAD_REGS=88) if I < 1024: # short K: the persistent kernel overlaps a tile's atomics with the next mainloop; N strips keep the [M, H] output # in L2 once it outgrows half of it down = dict(common, BLOCK_SIZE_N=256, BLOCK_SIZE_K=64, STAGES=4, N_GROUP=2 if M * H * 2 > l2_bytes // 2 else 0, LOAD_REGS=56, EPI_WARPS=4, EPI_REGS=72, PERSISTENT=True) else: down = dict(common, BLOCK_SIZE_N=256 if H % 256 == 0 else 128, BLOCK_SIZE_K=128 if I % 128 == 0 else 64, N_GROUP=0, STAGES=3, LOAD_REGS=88, PERSISTENT=False) return dict(BLOCK_SIZE_M=block_m, gate_up=gate_up, down=down) def w4a4_gate_up_gemm(xq, xs, g_in, packed, scale, glob, q, qs, g_out, sorted_ids, expert_ids, num_tokens_post_padded, num_valid_tokens, top_k, activation, act_alpha, act_limit, cfg): _w4a4_prefill(xq.device.index).gate_up_gemm( xq, xs, g_in, packed, scale, glob, q, qs, g_out, sorted_ids, expert_ids, num_tokens_post_padded, num_valid_tokens, top_k, activation, act_alpha, act_limit, cfg) def w4a4_down_gemm(xq, xs, g_in, packed, scale, glob, out, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded, num_valid_tokens, top_k, cfg): _w4a4_prefill(xq.device.index).down_gemm( xq, xs, g_in, packed, scale, glob, out, topk_weights_flat, sorted_ids, expert_ids, num_tokens_post_padded, num_valid_tokens, top_k, cfg)