"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp32 global per output row.""" from __future__ import annotations import functools from types import SimpleNamespace from typing import Any import torch from freetoken.kernel import backend from freetoken.utils import init_logger from ..registry import LayerKind, register_method from ..scheme import NVFP4_GROUP as GROUP, QuantKind from .base import LinearConfig, LinearKernel, LinearMethod logger = init_logger(__name__) FP8 = torch.float8_e4m3fn # fused parts whose weight globals are further apart than this ratio get a load-time warning _PART_GLOBAL_WARN_RATIO = 1.05 class TritonNvfp4LinearKernel(LinearKernel): """W4A16 on the K-major resident layout.""" name = "triton" def finalize(self, layer: Any) -> None: from freetoken.kernel.triton.nvfp4_linear import nvfp4_transpose_resident layer.weight, layer.weight_scale = nvfp4_transpose_resident(layer.weight, layer.weight_scale) def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: from freetoken.kernel.triton.nvfp4_linear import nvfp4_dense_linear_t return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias) def _tensor_global(layer: Any) -> torch.Tensor: """The one weight global the W4A4 GEMMs and the small-batch kernels take: the largest per-part global, the block scales unchanged.""" g = layer.weight_global.float() g_max = g.max() if g_max > _PART_GLOBAL_WARN_RATIO * g.min(): logger.warning_once( "an NVFP4 linear fuses parts whose weight global scales differ by more than %d%%; the W4A4 kernels use the largest " "for every part, which scales the other parts' outputs by their ratio; --quant-backend linear.nvfp4=triton keeps " "each part's global", round((_PART_GLOBAL_WARN_RATIO - 1) * 100), ) return g_max def _swizzle_128x4(scale: torch.Tensor) -> torch.Tensor: """[N, K/16] block scales -> the flat cuBLAS / CUTLASS 128x4 layout, rows padded to 128 and columns to 4 with zeros.""" rows, cols = scale.shape rb, cb = -(-rows // 128), -(-cols // 4) padded = torch.zeros(rb * 128, cb * 4, dtype=scale.dtype, device=scale.device) padded[:rows, :cols] = scale blocks = padded.view(rb, 128, cb, 4).permute(0, 2, 1, 3) return blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1).contiguous() @functools.cache def _sm120_gemv(device_index: int): """The row-major W4A16 GEMV / skinny GEMM module for this GPU, or None.""" from freetoken.kernel import gluon if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available(): return None from freetoken.kernel.gluon.nvfp4.linear import sm120_gemv return sm120_gemv def _small_batch(layer: Any, x: torch.Tensor, max_tokens: int) -> torch.Tensor | None: """W4A16 on the W4A4 kernels' own weight for decode-sized batches, where the W4A4 GEMMs leave bandwidth unused; None to run the GEMM.""" gemv = _sm120_gemv(x.device.index) if gemv is None or x.dtype != torch.bfloat16 or x.shape[0] == 0: return None route = gemv.gemv_config(layer.weight.shape[0], x.shape[1], x.shape[0], max_tokens) if route is None: return None kernel, tile = route return kernel(x if x.stride(1) == 1 else x.contiguous(), layer.weight, layer.weight_scale, layer.weight_global, **tile) @functools.cache def _flashinfer_cuda_major() -> int: """The CUDA version flashinfer's b12x GEMM checks at call time: nvcc's, else torch's.""" from flashinfer.jit.cpp_ext import get_cuda_version return get_cuda_version().major def _small_batch_usable() -> bool: return torch.cuda.is_available() and _sm120_gemv(torch.cuda.current_device()) is not None class CublasNvfp4LinearKernel(LinearKernel): """W4A4: activations quantized under the checkpoint's input_scale, then torch's NVFP4 scaled_mm (cuBLASLt) on the row-major weight.""" name = "cublas" def unusable_reason(self, cfg: LinearConfig) -> str | None: if backend.is_rocm() or not torch.cuda.is_available(): return "needs CUDA" if cfg.scheme is None or not cfg.scheme.has("input_scale"): return "the weight declares no static activation scale (W4A16 checkpoint)" if backend.device_capability() < (10, 0): return "NVFP4 tensor cores need sm_100+" if not hasattr(torch.nn.functional, "scaled_mm") or not hasattr(torch, "float4_e2m1fn_x2"): return f"torch {torch.__version__} has no NVFP4 scaled_mm" if cfg.out_features % 16 or cfg.in_features % 32: return f"scaled_mm needs N % 16 == 0 and K % 32 == 0, got N={cfg.out_features} K={cfg.in_features}" return None def worth_it(self, cfg: LinearConfig) -> bool: # cuBLAS alone runs decode-sized batches up to 1.4x slower than the triton GEMV; the small-batch kernels cover those on sm_120 return _small_batch_usable() def finalize(self, layer: Any) -> None: g_w = _tensor_global(layer) layer.weight_scale = _swizzle_128x4(layer.weight_scale) layer.weight_global = g_w.reshape(()) layer.input_scale = layer.input_scale.float().reshape(()) def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: from torch.nn import functional as F from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4 *lead, K = x.shape out = _small_batch(layer, x.reshape(-1, K), 16) if out is not None: out = out.reshape(*lead, layer.weight.shape[0]) return out + layer.bias.to(out.dtype) if layer.bias is not None else out xq, xs = quantize_nvfp4(x.reshape(-1, K), layer.input_scale.reshape(1), swizzled=True) block, tensor = F.ScalingType.BlockWise1x16, F.ScalingType.TensorWise swizzle = [F.SwizzleType.SWIZZLE_32_4_4, F.SwizzleType.NO_SWIZZLE] out = F.scaled_mm( xq.view(torch.float4_e2m1fn_x2), layer.weight.view(torch.float4_e2m1fn_x2).t(), [xs, layer.input_scale], [block, tensor], [layer.weight_scale, layer.weight_global], [block, tensor], swizzle_a=swizzle, swizzle_b=swizzle, output_dtype=x.dtype, ).reshape(*lead, layer.weight.shape[0]) return out + layer.bias.to(out.dtype) if layer.bias is not None else out class B12xNvfp4LinearKernel(LinearKernel): """W4A4 through flashinfer alone: its activation quant and its SM12x CuTe-DSL GEMM (mm_fp4 backend b12x) on the row-major weight.""" name = "b12x" def unusable_reason(self, cfg: LinearConfig) -> str | None: if backend.is_rocm() or not torch.cuda.is_available(): return "needs CUDA" if cfg.scheme is None or not cfg.scheme.has("input_scale"): return "the weight declares no static activation scale (W4A16 checkpoint)" cc = backend.device_capability() if cc[0] != 12: return f"b12x requires sm_120 / sm_121, got sm_{cc[0]}{cc[1]}" if not backend.is_flashinfer_installed(): return "flashinfer is not installed" drv = backend.driver_cuda_version() if drv is not None and drv < 13000: return f"b12x JIT-compiles through the driver and needs CUDA 13+, the driver supports {drv // 1000}.{drv % 1000 // 10}" if cfg.in_features % 32 or cfg.out_features % 8: # flashinfer checks only K; N % 8 != 0 returns wrong output return f"b12x needs K % 32 == 0 and N % 8 == 0, got K={cfg.in_features} N={cfg.out_features}" try: from flashinfer import fp4_quantize, mm_fp4, nvfp4_block_scale_interleave # noqa: F401 cuda_major = _flashinfer_cuda_major() except Exception as exc: return f"flashinfer FP4 GEMM symbols are unusable ({exc!r})" if cuda_major < 13: return f"flashinfer's b12x GEMM needs a CUDA 13 toolkit (nvcc, else torch), got CUDA {cuda_major}" return None def worth_it(self, cfg: LinearConfig) -> bool: return _small_batch_usable() def finalize(self, layer: Any) -> None: from flashinfer import nvfp4_block_scale_interleave g_w = _tensor_global(layer) layer.weight_scale = nvfp4_block_scale_interleave(layer.weight_scale.view(torch.uint8)).view(FP8) # flashinfer takes the quant-side activation global and alpha = 1 / (that * the weight's quant-side global) layer.input_scale_inv = (1.0 / layer.input_scale.float()).reshape(1) layer.alpha = (layer.input_scale.float() * g_w).reshape(1) layer.weight_global = g_w.reshape(()) def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: from flashinfer import fp4_quantize, mm_fp4 *lead, K = x.shape x2 = x.reshape(-1, K) if x2.shape[0] == 0: return x.new_empty(*lead, layer.weight.shape[0]) # b12x's W4A4 GEMM beats the W4A16 kernels from 9 tokens on GeForce, where bf16 mma accumulates in fp32 at half rate out = _small_batch(layer, x2, 8) if out is not None: out = out.reshape(*lead, layer.weight.shape[0]) return out + layer.bias.to(out.dtype) if layer.bias is not None else out xq, xs = fp4_quantize(x2.contiguous(), layer.input_scale_inv, is_sf_swizzled_layout=True) out = mm_fp4(xq, layer.weight.T, xs, layer.weight_scale, layer.alpha, x.dtype, backend="b12x").reshape(*lead, layer.weight.shape[0]) return out + layer.bias.to(out.dtype) if layer.bias is not None else out class MarlinNvfp4LinearKernel(LinearKernel): """vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight.""" name = "marlin" def unusable_reason(self, cfg: LinearConfig) -> str | None: if backend.is_rocm(): return "Marlin is CUDA-only and unavailable on ROCm; use triton" if not backend.is_vllm_installed(): return "vLLM is not installed" if len(cfg.output_sizes) > 1: return "fused projection carries one global scale per part" try: from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401 apply_fp4_marlin_linear, is_fp4_marlin_supported, prepare_fp4_layer_for_marlin, ) except Exception as exc: return f"vLLM Marlin FP4 symbols are unusable ({exc!r})" if not is_fp4_marlin_supported(): return "vLLM reports FP4 Marlin unsupported on this device" return None def finalize(self, layer: Any) -> None: from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import prepare_fp4_layer_for_marlin # Marlin wants the per-tensor scalar; weight_global is that scalar broadcast to [N] packed = SimpleNamespace( weight=layer.weight, weight_scale=layer.weight_scale, weight_scale_2=layer.weight_global.reshape(-1)[0].to(torch.bfloat16), output_size_per_partition=layer.out_features, input_size_per_partition=layer.in_features, params_dtype=torch.bfloat16, ) prepare_fp4_layer_for_marlin(packed) layer._marlin_weight = packed.weight layer._marlin_scale = packed.weight_scale layer._marlin_global = packed.weight_scale_2 layer._marlin_workspace = packed.workspace dev = layer.weight.device layer.weight = torch.empty(0, dtype=torch.uint8, device=dev) layer.weight_scale = torch.empty(0, dtype=FP8, device=dev) layer.weight_global = torch.empty(0, dtype=torch.float32, device=dev) def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear out = apply_fp4_marlin_linear( input=x, weight=layer._marlin_weight, weight_scale=layer._marlin_scale, weight_scale_2=layer._marlin_global, workspace=layer._marlin_workspace, size_n=layer.out_features, size_k=layer.in_features, ) return out + layer.bias.to(out.dtype) if layer.bias is not None else out class EmulationNvfp4LinearKernel(LinearKernel): """Dequantize to bf16 with dequant_nvfp4, then a plain matmul; keeps the row-major layout.""" name = "emulation" def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor: from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4 *lead, K = x.shape slots = torch.zeros(1, dtype=torch.int32, device=x.device) w = dequant_nvfp4( layer.weight.unsqueeze(0), layer.weight_scale.unsqueeze(0), layer.weight_global.unsqueeze(0), slots, dtype=torch.bfloat16, )[0] out = (x.reshape(-1, K) @ w.t()).to(x.dtype).reshape(*lead, w.shape[0]) return out + layer.bias.to(out.dtype) if layer.bias is not None else out @register_method(QuantKind.NVFP4, LayerKind.LINEAR) class Nvfp4LinearMethod(LinearMethod): candidates = (B12xNvfp4LinearKernel, CublasNvfp4LinearKernel, TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel) def create_weights(self, layer: Any) -> None: g = self.cfg if g.in_features % GROUP: raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}") layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8) layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8) layer.weight_global = torch.empty(g.out_features, dtype=torch.float32) # no W4A16 kernel reads it; declared so a W4A4 checkpoint loads complete and a W4A4 kernel finds it in place layer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else None