Files
Xiaoze Fan 33cbb3c54e feat(utils): add once-per-message logger methods (#858)
debug_once / info_once / warning_once log on rank 0, once per distinct message and args. select_kernel, which logged its choice once per layer, and the hand-rolled log-once guards now use them.
2026-11-03 15:31:08 -08:00

302 lines
14 KiB
Python

"""e2m1 weight packed two per byte, e4m3 scales per 16 input elements, one fp32 global per output row."""
from __future__ import annotations
import functools
from types import SimpleNamespace
from typing import Any
import torch
from freetoken.kernel import backend
from freetoken.utils import init_logger
from ..registry import LayerKind, register_method
from ..scheme import NVFP4_GROUP as GROUP, QuantKind
from .base import LinearConfig, LinearKernel, LinearMethod
logger = init_logger(__name__)
FP8 = torch.float8_e4m3fn
# fused parts whose weight globals are further apart than this ratio get a load-time warning
_PART_GLOBAL_WARN_RATIO = 1.05
class TritonNvfp4LinearKernel(LinearKernel):
"""W4A16 on the K-major resident layout."""
name = "triton"
def finalize(self, layer: Any) -> None:
from freetoken.kernel.triton.nvfp4_linear import nvfp4_transpose_resident
layer.weight, layer.weight_scale = nvfp4_transpose_resident(layer.weight, layer.weight_scale)
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.nvfp4_linear import nvfp4_dense_linear_t
return nvfp4_dense_linear_t(x, layer.weight, layer.weight_scale, layer.weight_global, layer.bias)
def _tensor_global(layer: Any) -> torch.Tensor:
"""The one weight global the W4A4 GEMMs and the small-batch kernels take: the largest per-part global, the block scales unchanged."""
g = layer.weight_global.float()
g_max = g.max()
if g_max > _PART_GLOBAL_WARN_RATIO * g.min():
logger.warning_once(
"an NVFP4 linear fuses parts whose weight global scales differ by more than %d%%; the W4A4 kernels use the largest "
"for every part, which scales the other parts' outputs by their ratio; --quant-backend linear.nvfp4=triton keeps "
"each part's global", round((_PART_GLOBAL_WARN_RATIO - 1) * 100),
)
return g_max
def _swizzle_128x4(scale: torch.Tensor) -> torch.Tensor:
"""[N, K/16] block scales -> the flat cuBLAS / CUTLASS 128x4 layout, rows padded to 128 and columns to 4 with zeros."""
rows, cols = scale.shape
rb, cb = -(-rows // 128), -(-cols // 4)
padded = torch.zeros(rb * 128, cb * 4, dtype=scale.dtype, device=scale.device)
padded[:rows, :cols] = scale
blocks = padded.view(rb, 128, cb, 4).permute(0, 2, 1, 3)
return blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1).contiguous()
@functools.cache
def _sm120_gemv(device_index: int):
"""The row-major W4A16 GEMV / skinny GEMM module for this GPU, or None."""
from freetoken.kernel import gluon
if torch.cuda.get_device_capability(device_index) != (12, 0) or not gluon.available():
return None
from freetoken.kernel.gluon.nvfp4.linear import sm120_gemv
return sm120_gemv
def _small_batch(layer: Any, x: torch.Tensor, max_tokens: int) -> torch.Tensor | None:
"""W4A16 on the W4A4 kernels' own weight for decode-sized batches, where the W4A4 GEMMs leave bandwidth unused; None to run the GEMM."""
gemv = _sm120_gemv(x.device.index)
if gemv is None or x.dtype != torch.bfloat16 or x.shape[0] == 0:
return None
route = gemv.gemv_config(layer.weight.shape[0], x.shape[1], x.shape[0], max_tokens)
if route is None:
return None
kernel, tile = route
return kernel(x if x.stride(1) == 1 else x.contiguous(), layer.weight, layer.weight_scale, layer.weight_global, **tile)
@functools.cache
def _flashinfer_cuda_major() -> int:
"""The CUDA version flashinfer's b12x GEMM checks at call time: nvcc's, else torch's."""
from flashinfer.jit.cpp_ext import get_cuda_version
return get_cuda_version().major
def _small_batch_usable() -> bool:
return torch.cuda.is_available() and _sm120_gemv(torch.cuda.current_device()) is not None
class CublasNvfp4LinearKernel(LinearKernel):
"""W4A4: activations quantized under the checkpoint's input_scale, then torch's NVFP4 scaled_mm (cuBLASLt) on the row-major weight."""
name = "cublas"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
if backend.is_rocm() or not torch.cuda.is_available():
return "needs CUDA"
if cfg.scheme is None or not cfg.scheme.has("input_scale"):
return "the weight declares no static activation scale (W4A16 checkpoint)"
if backend.device_capability() < (10, 0):
return "NVFP4 tensor cores need sm_100+"
if not hasattr(torch.nn.functional, "scaled_mm") or not hasattr(torch, "float4_e2m1fn_x2"):
return f"torch {torch.__version__} has no NVFP4 scaled_mm"
if cfg.out_features % 16 or cfg.in_features % 32:
return f"scaled_mm needs N % 16 == 0 and K % 32 == 0, got N={cfg.out_features} K={cfg.in_features}"
return None
def worth_it(self, cfg: LinearConfig) -> bool:
# cuBLAS alone runs decode-sized batches up to 1.4x slower than the triton GEMV; the small-batch kernels cover those on sm_120
return _small_batch_usable()
def finalize(self, layer: Any) -> None:
g_w = _tensor_global(layer)
layer.weight_scale = _swizzle_128x4(layer.weight_scale)
layer.weight_global = g_w.reshape(())
layer.input_scale = layer.input_scale.float().reshape(())
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from torch.nn import functional as F
from freetoken.kernel.triton.nvfp4_act_quant import quantize_nvfp4
*lead, K = x.shape
out = _small_batch(layer, x.reshape(-1, K), 16)
if out is not None:
out = out.reshape(*lead, layer.weight.shape[0])
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
xq, xs = quantize_nvfp4(x.reshape(-1, K), layer.input_scale.reshape(1), swizzled=True)
block, tensor = F.ScalingType.BlockWise1x16, F.ScalingType.TensorWise
swizzle = [F.SwizzleType.SWIZZLE_32_4_4, F.SwizzleType.NO_SWIZZLE]
out = F.scaled_mm(
xq.view(torch.float4_e2m1fn_x2), layer.weight.view(torch.float4_e2m1fn_x2).t(),
[xs, layer.input_scale], [block, tensor], [layer.weight_scale, layer.weight_global], [block, tensor],
swizzle_a=swizzle, swizzle_b=swizzle, output_dtype=x.dtype,
).reshape(*lead, layer.weight.shape[0])
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
class B12xNvfp4LinearKernel(LinearKernel):
"""W4A4 through flashinfer alone: its activation quant and its SM12x CuTe-DSL GEMM (mm_fp4 backend b12x) on the row-major weight."""
name = "b12x"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
if backend.is_rocm() or not torch.cuda.is_available():
return "needs CUDA"
if cfg.scheme is None or not cfg.scheme.has("input_scale"):
return "the weight declares no static activation scale (W4A16 checkpoint)"
cc = backend.device_capability()
if cc[0] != 12:
return f"b12x requires sm_120 / sm_121, got sm_{cc[0]}{cc[1]}"
if not backend.is_flashinfer_installed():
return "flashinfer is not installed"
drv = backend.driver_cuda_version()
if drv is not None and drv < 13000:
return f"b12x JIT-compiles through the driver and needs CUDA 13+, the driver supports {drv // 1000}.{drv % 1000 // 10}"
if cfg.in_features % 32 or cfg.out_features % 8:
# flashinfer checks only K; N % 8 != 0 returns wrong output
return f"b12x needs K % 32 == 0 and N % 8 == 0, got K={cfg.in_features} N={cfg.out_features}"
try:
from flashinfer import fp4_quantize, mm_fp4, nvfp4_block_scale_interleave # noqa: F401
cuda_major = _flashinfer_cuda_major()
except Exception as exc:
return f"flashinfer FP4 GEMM symbols are unusable ({exc!r})"
if cuda_major < 13:
return f"flashinfer's b12x GEMM needs a CUDA 13 toolkit (nvcc, else torch), got CUDA {cuda_major}"
return None
def worth_it(self, cfg: LinearConfig) -> bool:
return _small_batch_usable()
def finalize(self, layer: Any) -> None:
from flashinfer import nvfp4_block_scale_interleave
g_w = _tensor_global(layer)
layer.weight_scale = nvfp4_block_scale_interleave(layer.weight_scale.view(torch.uint8)).view(FP8)
# flashinfer takes the quant-side activation global and alpha = 1 / (that * the weight's quant-side global)
layer.input_scale_inv = (1.0 / layer.input_scale.float()).reshape(1)
layer.alpha = (layer.input_scale.float() * g_w).reshape(1)
layer.weight_global = g_w.reshape(())
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from flashinfer import fp4_quantize, mm_fp4
*lead, K = x.shape
x2 = x.reshape(-1, K)
if x2.shape[0] == 0:
return x.new_empty(*lead, layer.weight.shape[0])
# b12x's W4A4 GEMM beats the W4A16 kernels from 9 tokens on GeForce, where bf16 mma accumulates in fp32 at half rate
out = _small_batch(layer, x2, 8)
if out is not None:
out = out.reshape(*lead, layer.weight.shape[0])
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
xq, xs = fp4_quantize(x2.contiguous(), layer.input_scale_inv, is_sf_swizzled_layout=True)
out = mm_fp4(xq, layer.weight.T, xs, layer.weight_scale, layer.alpha, x.dtype, backend="b12x").reshape(*lead, layer.weight.shape[0])
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
class MarlinNvfp4LinearKernel(LinearKernel):
"""vLLM's fused FP4 Marlin GEMM (W4A16); needs one global scale for the whole weight."""
name = "marlin"
def unusable_reason(self, cfg: LinearConfig) -> str | None:
if backend.is_rocm():
return "Marlin is CUDA-only and unavailable on ROCm; use triton"
if not backend.is_vllm_installed():
return "vLLM is not installed"
if len(cfg.output_sizes) > 1:
return "fused projection carries one global scale per part"
try:
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( # noqa: F401
apply_fp4_marlin_linear,
is_fp4_marlin_supported,
prepare_fp4_layer_for_marlin,
)
except Exception as exc:
return f"vLLM Marlin FP4 symbols are unusable ({exc!r})"
if not is_fp4_marlin_supported():
return "vLLM reports FP4 Marlin unsupported on this device"
return None
def finalize(self, layer: Any) -> None:
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import prepare_fp4_layer_for_marlin
# Marlin wants the per-tensor scalar; weight_global is that scalar broadcast to [N]
packed = SimpleNamespace(
weight=layer.weight,
weight_scale=layer.weight_scale,
weight_scale_2=layer.weight_global.reshape(-1)[0].to(torch.bfloat16),
output_size_per_partition=layer.out_features,
input_size_per_partition=layer.in_features,
params_dtype=torch.bfloat16,
)
prepare_fp4_layer_for_marlin(packed)
layer._marlin_weight = packed.weight
layer._marlin_scale = packed.weight_scale
layer._marlin_global = packed.weight_scale_2
layer._marlin_workspace = packed.workspace
dev = layer.weight.device
layer.weight = torch.empty(0, dtype=torch.uint8, device=dev)
layer.weight_scale = torch.empty(0, dtype=FP8, device=dev)
layer.weight_global = torch.empty(0, dtype=torch.float32, device=dev)
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import apply_fp4_marlin_linear
out = apply_fp4_marlin_linear(
input=x,
weight=layer._marlin_weight,
weight_scale=layer._marlin_scale,
weight_scale_2=layer._marlin_global,
workspace=layer._marlin_workspace,
size_n=layer.out_features,
size_k=layer.in_features,
)
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
class EmulationNvfp4LinearKernel(LinearKernel):
"""Dequantize to bf16 with dequant_nvfp4, then a plain matmul; keeps the row-major layout."""
name = "emulation"
def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:
from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
*lead, K = x.shape
slots = torch.zeros(1, dtype=torch.int32, device=x.device)
w = dequant_nvfp4(
layer.weight.unsqueeze(0), layer.weight_scale.unsqueeze(0), layer.weight_global.unsqueeze(0),
slots, dtype=torch.bfloat16,
)[0]
out = (x.reshape(-1, K) @ w.t()).to(x.dtype).reshape(*lead, w.shape[0])
return out + layer.bias.to(out.dtype) if layer.bias is not None else out
@register_method(QuantKind.NVFP4, LayerKind.LINEAR)
class Nvfp4LinearMethod(LinearMethod):
candidates = (B12xNvfp4LinearKernel, CublasNvfp4LinearKernel, TritonNvfp4LinearKernel, MarlinNvfp4LinearKernel, EmulationNvfp4LinearKernel)
def create_weights(self, layer: Any) -> None:
g = self.cfg
if g.in_features % GROUP:
raise ValueError(f"nvfp4 needs in_features divisible by {GROUP}, got {g.in_features}")
layer.weight = torch.empty(g.out_features, g.in_features // 2, dtype=torch.uint8)
layer.weight_scale = torch.empty(g.out_features, g.in_features // GROUP, dtype=FP8)
layer.weight_global = torch.empty(g.out_features, dtype=torch.float32)
# no W4A16 kernel reads it; declared so a W4A4 checkpoint loads complete and a W4A4 kernel finds it in place
layer.input_scale = torch.empty((), dtype=torch.float32) if self.scheme.has("input_scale") else None