mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* fix(kernel): convert fp32 to e4m3 in one rounding on sm_89 * fix(kernel): avoid nvidia fp8 cast asm on hip --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
358 lines
17 KiB
Python
358 lines
17 KiB
Python
"""Per-tensor (per-output-row) FP8 W8A16 dense linear, shared across models.
|
|
|
|
The nvidia/modelopt ``MIXED_PRECISION`` checkpoints (Qwen3.5/3.6, ...) quantize the attention
|
|
and GatedDeltaNet projections to **per-tensor FP8** (``weight`` fp8-e4m3 + a single scalar
|
|
``weight_scale``; ``weight_bf16 = weight_fp8 * weight_scale``) while the routed experts are
|
|
NVFP4. FreeToken used to *dequantize these dense weights to bf16 at load* and run them through
|
|
cuBLAS bf16 GEMV -- which is the dominant decode cost (~54% of the step) because it reads
|
|
2x the bytes of the resident FP8 weight. This module keeps the weight FP8 and reads it
|
|
directly in a W8A16 kernel (bf16 activation, fp8 weight), halving that traffic.
|
|
|
|
Fused projections (q/k/v -> ``qkv_proj``; GDN qkv|z -> ``in_proj_qkvz``) concatenate parts
|
|
that each carry their *own* per-tensor scalar, so the scalar is broadcast to a per-output-row
|
|
vector ``weight_scale`` ``[N]`` (piecewise-constant after fusion) and applied per output row.
|
|
This is exactly equal to applying each part's scalar to its own rows.
|
|
|
|
Numerics: the kernel converts fp8->f32 and accumulates in fp32 (standard W8A16; strictly more
|
|
accurate than the previous ``weight.to(bf16) * scale`` materialization, which it replaces).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import re
|
|
|
|
import torch
|
|
import triton
|
|
import triton.language as tl
|
|
|
|
from freetoken.kernel.triton.e4m3_compat import (
|
|
e4m3_kernel_view,
|
|
e4m3_native,
|
|
e4m3_native_cx,
|
|
e4m3_u8_to_f32,
|
|
f32_to_e4m3,
|
|
)
|
|
|
|
FP8 = torch.float8_e4m3fn
|
|
_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
|
|
|
|
|
|
# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the
|
|
# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and
|
|
# some builds (Windows) have no row-wise kernel (#227). Fallback: one tensor-wise GEMM per part.
|
|
def _torch_version() -> tuple[int, int]:
|
|
m = re.match(r"(\d+)\.(\d+)", torch.__version__)
|
|
return (int(m.group(1)), int(m.group(2))) if m else (0, 0)
|
|
|
|
|
|
@functools.cache
|
|
def rowwise_scaled_mm_ok() -> bool:
|
|
"""Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU.
|
|
Decided once per process, at load (never under graph capture)."""
|
|
if not torch.cuda.is_available():
|
|
return True
|
|
from freetoken.gpu_select import assigned_visible_gpu
|
|
|
|
idx = assigned_visible_gpu()
|
|
dev = torch.device("cuda", torch.cuda.current_device() if idx is None else idx)
|
|
if torch.cuda.get_device_capability(dev) == (8, 9) and _torch_version() < (2, 12):
|
|
return False
|
|
# Probe on the default stream (safe even where the launch ignores the current stream); a
|
|
# build without the row-wise kernel raises here instead of at the first forward.
|
|
try:
|
|
with torch.cuda.device(dev), torch.cuda.stream(torch.cuda.default_stream(dev)):
|
|
a = torch.zeros(16, 32, dtype=FP8, device=dev)
|
|
b = torch.zeros(32, 32, dtype=FP8, device=dev)
|
|
torch._scaled_mm(
|
|
a, b.t(), scale_a=torch.ones(16, 1, device=dev),
|
|
scale_b=torch.ones(1, 32, device=dev), out_dtype=torch.bfloat16,
|
|
)
|
|
torch.cuda.synchronize(dev)
|
|
except RuntimeError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def weight_scale_segments(weight_scale: torch.Tensor) -> list[tuple[int, int]]:
|
|
"""``[start, end)`` row ranges over which ``weight_scale`` is constant (the fused parts).
|
|
Syncs; call at load."""
|
|
s = weight_scale.detach().reshape(-1).float().cpu()
|
|
change = (torch.nonzero(s[1:] != s[:-1]).flatten() + 1).tolist()
|
|
bounds = [0, *change, s.numel()]
|
|
return list(zip(bounds[:-1], bounds[1:]))
|
|
|
|
|
|
_MAX_SEGMENTS = 8 # q/k/v = 3, GDN qkv|z = 2; a genuine per-row scale stays W8A16 instead
|
|
|
|
|
|
def _segments_w8a8_ok(segments: list[tuple[int, int]]) -> bool:
|
|
"""cuBLASLt needs 16-row aligned fp8 operands; more parts than a fused projection has
|
|
means a genuine per-row scale."""
|
|
return 0 < len(segments) <= _MAX_SEGMENTS and all((e - s) % 16 == 0 for s, e in segments)
|
|
|
|
|
|
# ======================================================================================
|
|
# Decode (M==1) split-K GEMV: raw fp8 x bf16 reduction in fp32, per-row scale at reduce.
|
|
# ======================================================================================
|
|
@triton.jit
|
|
def _gemv_splitk_kernel(
|
|
a_ptr, w_ptr, part_ptr, N, K, n_kb, kb_per,
|
|
stride_ak, stride_wn, stride_wk, stride_pk, stride_pn,
|
|
BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
|
|
):
|
|
"""Each (pid_n, pid_k) computes the partial sum over ``kb_per`` BLOCK_K chunks for a
|
|
BLOCK_N slice of outputs. ``kb_per`` ceil-tiles K so K only needs to be a multiple of
|
|
BLOCK_K is *not* required (loads are k-masked). Per-row scale is applied in the reduce."""
|
|
pid_n = tl.program_id(0)
|
|
pid_k = tl.program_id(1)
|
|
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
|
n_mask = offs_n < N
|
|
kb_start = pid_k * kb_per
|
|
acc = tl.zeros((BLOCK_N,), dtype=tl.float32)
|
|
for i in range(kb_per):
|
|
kb = kb_start + i
|
|
if kb < n_kb:
|
|
offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)
|
|
k_mask = offs_k < K
|
|
a = tl.load(a_ptr + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)
|
|
if e4m3_native_cx():
|
|
w = tl.load(
|
|
w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
|
|
mask=n_mask[:, None] & k_mask[None, :], other=0.0,
|
|
).to(tl.float32)
|
|
else:
|
|
w = e4m3_u8_to_f32(tl.load(
|
|
w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
|
|
mask=n_mask[:, None] & k_mask[None, :], other=0,
|
|
))
|
|
acc += tl.sum(w * a[None, :], axis=1)
|
|
tl.store(part_ptr + pid_k * stride_pk + offs_n * stride_pn, acc, mask=n_mask)
|
|
|
|
|
|
@triton.jit
|
|
def _splitk_reduce_kernel(
|
|
part_ptr, scale_ptr, out_ptr, N, SPLIT_K: tl.constexpr,
|
|
stride_pk, stride_pn, BLOCK: tl.constexpr, OUT: tl.constexpr,
|
|
):
|
|
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
|
mask = offs < N
|
|
acc = tl.zeros((BLOCK,), dtype=tl.float32)
|
|
for k in tl.static_range(SPLIT_K):
|
|
acc += tl.load(part_ptr + k * stride_pk + offs * stride_pn, mask=mask, other=0.0)
|
|
scale = tl.load(scale_ptr + offs, mask=mask, other=0.0).to(tl.float32)
|
|
tl.store(out_ptr + offs, (acc * scale).to(OUT), mask=mask)
|
|
|
|
|
|
def _gemv(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
|
|
out_dtype: torch.dtype) -> torch.Tensor:
|
|
"""M==1 split-K GEMV. ``a`` [K] bf16; ``weight`` [N, K] fp8; ``weight_scale`` [N] fp32."""
|
|
N, K = weight.shape
|
|
BLOCK_K = 128
|
|
n_kb = triton.cdiv(K, BLOCK_K)
|
|
BLOCK_N = 16
|
|
n_tiles = triton.cdiv(N, BLOCK_N)
|
|
split_k = max(1, min(1536 // n_tiles, n_kb))
|
|
split_k = 1 << (split_k.bit_length() - 1) # pow2 -> stable reduction order
|
|
kb_per = triton.cdiv(n_kb, split_k)
|
|
part = torch.empty((split_k, N), dtype=torch.float32, device=a.device)
|
|
_gemv_splitk_kernel[(n_tiles, split_k)](
|
|
a, weight, part, N, K, n_kb, kb_per,
|
|
a.stride(0), weight.stride(0), weight.stride(1), part.stride(0), part.stride(1),
|
|
BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, num_warps=1,
|
|
)
|
|
out = torch.empty(N, dtype=out_dtype, device=a.device)
|
|
_splitk_reduce_kernel[(triton.cdiv(N, 256),)](
|
|
part, weight_scale, out, N, split_k, part.stride(0), part.stride(1),
|
|
BLOCK=256, OUT=_TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16],
|
|
num_warps=2,
|
|
)
|
|
return out
|
|
|
|
|
|
# ======================================================================================
|
|
# Prefill (M>1) W8A16 GEMM: fp8 weight read from HBM, upcast to bf16 in-register for the
|
|
# tensor-core dot (fp8 e4m3 -> bf16 is lossless), per-row scale applied after accumulation.
|
|
# ======================================================================================
|
|
@triton.jit
|
|
def _gemm_kernel(
|
|
a_ptr, w_ptr, scale_ptr, c_ptr, M, N, K,
|
|
stride_am, stride_ak, stride_wn, stride_wk, stride_cm, stride_cn,
|
|
BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
|
|
compute_type: tl.constexpr,
|
|
):
|
|
pid_m = tl.program_id(0)
|
|
pid_n = tl.program_id(1)
|
|
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
|
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
|
offs_k = tl.arange(0, BLOCK_K)
|
|
m_mask = offs_m < M
|
|
n_mask = offs_n < N
|
|
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak
|
|
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
|
|
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
|
for k in range(tl.cdiv(K, BLOCK_K)):
|
|
k_rem = K - k * BLOCK_K
|
|
a = tl.load(a_ptrs, mask=m_mask[:, None] & (offs_k[None, :] < k_rem), other=0.0)
|
|
w_mask = n_mask[:, None] & (offs_k[None, :] < k_rem)
|
|
if e4m3_native_cx():
|
|
w = tl.load(w_ptrs, mask=w_mask, other=0.0).to(a.dtype)
|
|
else:
|
|
w = e4m3_u8_to_f32(tl.load(w_ptrs, mask=w_mask, other=0)).to(a.dtype)
|
|
acc += tl.dot(a, tl.trans(w), out_dtype=tl.float32)
|
|
a_ptrs += BLOCK_K * stride_ak
|
|
w_ptrs += BLOCK_K * stride_wk
|
|
scale = tl.load(scale_ptr + offs_n, mask=n_mask, other=0.0).to(tl.float32)
|
|
acc = acc * scale[None, :]
|
|
c_ptrs = c_ptr + offs_m[:, None] * stride_cm + offs_n[None, :] * stride_cn
|
|
tl.store(c_ptrs, acc.to(compute_type), mask=m_mask[:, None] & n_mask[None, :])
|
|
|
|
|
|
def _gemm(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
|
|
out_dtype: torch.dtype) -> torch.Tensor:
|
|
"""M>1 W8A16 GEMM. ``a`` [M, K] bf16; ``weight`` [N, K] fp8; ``weight_scale`` [N] fp32."""
|
|
M, K = a.shape
|
|
N = weight.shape[0]
|
|
compute = out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16
|
|
out = torch.empty((M, N), dtype=compute, device=a.device)
|
|
BLOCK_M = 64 if M >= 64 else 32
|
|
BLOCK_N, BLOCK_K = 128, 64
|
|
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
|
|
_gemm_kernel[grid](
|
|
a, weight, weight_scale, out, M, N, K,
|
|
a.stride(0), a.stride(1), weight.stride(0), weight.stride(1), out.stride(0), out.stride(1),
|
|
BLOCK_M=BLOCK_M, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, compute_type=_TL_DTYPE[compute],
|
|
num_warps=8 if M >= 64 else 4, num_stages=3,
|
|
)
|
|
return out
|
|
|
|
|
|
# ======================================================================================
|
|
# Batched decode (M > 1) W8A8: quantize the activation with the checkpoint's static
|
|
# per-tensor ``input_scale`` and hand both fp8 operands to cuBLASLt via torch._scaled_mm.
|
|
#
|
|
# ``_gemm`` above is a *prefill* kernel: grid is (cdiv(M, BLOCK_M), cdiv(N, 128)) with no
|
|
# split-K, so at decode batch sizes the M axis contributes a single block row and the whole
|
|
# GEMM runs on cdiv(N, 128) CTAs -- 8 of them for N=1024, on a 188-SM part. Its runtime then
|
|
# tracks K and ignores both N and M: latency-bound, not bandwidth-bound. Measured on
|
|
# Qwen3.5-27B's projections (RTX PRO 6000, 1461 GB/s copy roof): 594 GB/s flat across
|
|
# M=2..16, against 1334 GB/s for the M=1 GEMV over the same weights. Routing M>=2 to
|
|
# cuBLASLt restores ~1300 GB/s -- 10.15 ms -> 5.56 ms of per-decode-step GEMM.
|
|
#
|
|
# Requires sm_89+ (torch._scaled_mm's floor; Ampere has no FP8 tensor cores) *and* a
|
|
# checkpoint carrying ``input_scale``. Without either, ``_gemm`` still runs: glm_moe_dsa
|
|
# quantizes to fp8 at load with genuine per-row scales and has no activation scale at all.
|
|
# ======================================================================================
|
|
_E4M3_MAX = 448.0 # largest finite e4m3 magnitude
|
|
|
|
|
|
@triton.jit
|
|
def _static_quant_kernel(x_ptr, out_ptr, scale_ptr, n_elements, BLOCK: tl.constexpr):
|
|
"""bf16 activation -> fp8-e4m3 under one broadcast per-tensor scale."""
|
|
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
|
mask = offs < n_elements
|
|
inv = 1.0 / tl.load(scale_ptr).to(tl.float32)
|
|
v = tl.load(x_ptr + offs, mask=mask, other=0.0).to(tl.float32) * inv
|
|
v = tl.minimum(tl.maximum(v, -448.0), 448.0)
|
|
tl.store(out_ptr + offs, f32_to_e4m3(v), mask=mask)
|
|
|
|
|
|
def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor:
|
|
"""One launch, no host sync: the scale is read from device memory inside the kernel, so
|
|
this stays CUDA-graph safe (fixed shapes, no dependence on tensor *values* on the host)."""
|
|
n = a.numel()
|
|
out = torch.empty_like(a, dtype=FP8)
|
|
BLOCK = 1024
|
|
_static_quant_kernel[(triton.cdiv(n, BLOCK),)](
|
|
a, out, input_scale, n, BLOCK=BLOCK, num_warps=4,
|
|
)
|
|
return out
|
|
|
|
|
|
def _scaled_mm(
|
|
a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
|
|
input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype,
|
|
scale_segments: list[tuple[int, int]] | None = None,
|
|
) -> torch.Tensor:
|
|
"""``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM.
|
|
|
|
``weight`` stays in the checkpoint-native row-major ``[N, K]``: ``.t()`` is exactly the
|
|
column-major ``[K, N]`` operand cuBLASLt wants, so the transpose is a stride change and
|
|
never a copy (unlike Marlin, or our own NVFP4 path's K-major repack).
|
|
|
|
A genuine per-tensor weight (one scalar repeated down ``weight_scale``) takes the
|
|
tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constant
|
|
because each part carries its own scalar, takes the row-wise path -- that keeps every
|
|
part's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum
|
|
and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step).
|
|
|
|
Where row-wise is unsafe (:func:`rowwise_scaled_mm_ok`) ``scale_segments`` is passed and
|
|
each part runs its own tensor-wise GEMM over ``weight[s:e]`` (still stride-only), outputs
|
|
concatenated: the same W8A8 scheme, not bit-identical (accumulation order differs)."""
|
|
qa = _static_quant(a, input_scale)
|
|
wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only
|
|
sa = input_scale.reshape(())
|
|
if uniform_scale:
|
|
return torch._scaled_mm(
|
|
qa, wt, scale_a=sa, scale_b=weight_scale[0].reshape(()), out_dtype=out_dtype,
|
|
)
|
|
if scale_segments is not None:
|
|
return torch.cat([
|
|
torch._scaled_mm(
|
|
qa, weight[s:e].t(), scale_a=sa, scale_b=weight_scale[s].reshape(()),
|
|
out_dtype=out_dtype,
|
|
)
|
|
for s, e in scale_segments
|
|
], dim=1)
|
|
return torch._scaled_mm(
|
|
qa, wt,
|
|
scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(),
|
|
scale_b=weight_scale.reshape(1, -1),
|
|
out_dtype=out_dtype,
|
|
)
|
|
|
|
|
|
def fp8_pertensor_linear(
|
|
x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,
|
|
bias: torch.Tensor | None = None,
|
|
input_scale: torch.Tensor | None = None,
|
|
uniform_scale: bool = False,
|
|
scale_segments: list[tuple[int, int]] | None = None,
|
|
) -> torch.Tensor:
|
|
"""``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale``
|
|
[N] fp32 (per output row). ``scale_segments``: the fused parts' row ranges, precomputed at
|
|
load by the layer; derived here (with a sync) when omitted and needed.
|
|
|
|
Whether the activation is quantized is a property of the *deployment*, never of the batch:
|
|
with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K
|
|
GEMV at M=1, GEMM above it). Picking per-M would make a request's numerics depend on how
|
|
many unrelated requests happened to be in flight, so a reply would not reproduce at bs=1 --
|
|
and W8A16 at M=1 is worth only ~0.5% (5.53 ms vs 5.56 ms of per-step GEMM) anyway. vLLM and
|
|
SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores."""
|
|
*lead, K = x.shape
|
|
N = weight.shape[0]
|
|
w8a8 = input_scale is not None and e4m3_native()
|
|
segments = None
|
|
if w8a8 and not uniform_scale and not rowwise_scaled_mm_ok():
|
|
segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale)
|
|
if not _segments_w8a8_ok(segments):
|
|
w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mm
|
|
if w8a8:
|
|
out = _scaled_mm(
|
|
x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,
|
|
scale_segments=segments,
|
|
).reshape(*lead, N)
|
|
elif x.numel() // K == 1:
|
|
out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N)
|
|
else:
|
|
out = _gemm(
|
|
x.reshape(-1, K), e4m3_kernel_view(weight), weight_scale, x.dtype,
|
|
).reshape(*lead, N)
|
|
if bias is not None:
|
|
out = out + bias.to(out.dtype)
|
|
return out
|
|
|
|
|
|
# ======================================================================================
|
|
# ======================================================================================
|