mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-23 02:00:02 +08:00
* fix(kernel): convert fp32 to e4m3 in one rounding on sm_89 * fix(kernel): avoid nvidia fp8 cast asm on hip --------- Co-authored-by: zihaomu <zihaomu@outlook.com>
163 lines
7.1 KiB
Python
163 lines
7.1 KiB
Python
"""e4m3 (fp8e4nv) emulation for pre-sm_89 CUDA GPUs.
|
|
|
|
Triton rejects the fp8e4nv type anywhere in a kernel compiled for sm < 89 -- the
|
|
check sits in ``dtype.to_ir``, so even an fp8 *pointer argument* is illegal.
|
|
Affected kernels branch on :func:`e4m3_native_cx` (a compile-time constexpr): the
|
|
native branch casts fp32 through :func:`f32_to_e4m3`, the emulated branch is dead-code
|
|
eliminated there. When the emulated branch is active, wrappers must pass e4m3
|
|
tensors as ``.view(torch.uint8)`` and allocate act-quant outputs as bf16 -- use
|
|
the host-side twin :func:`e4m3_native` for those decisions.
|
|
|
|
``FREETOKEN_FORCE_E4M3_EMU=1`` (or true/yes/on) forces the emulated path on any
|
|
GPU (for A/B validation against the native fp8 unit). The flag is read ONCE at
|
|
import and is deliberately NOT part of triton's compilation cache key, so:
|
|
flipping it later in the same process raises (see :func:`e4m3_native`), and when
|
|
it is set without an explicit ``TRITON_CACHE_DIR`` the default cache dir is
|
|
salted so a warm native cache can never serve the other branch's binary for the
|
|
signature-invariant kernels (``_act_quant_inplace_kernel``).
|
|
|
|
The primitives are validated bit-exact against the native fp8 unit on H100 (all
|
|
254 non-NaN codes; 2.1M adversarial rounding samples covering every grid
|
|
midpoint, the subnormal range and the 2^-6 boundary). Known deviations: the NaN
|
|
codes 0x7F/0xFF decode to +-480 (checkpoints never store NaN weights), and
|
|
``round_e4m3(-0.0)`` returns +0.0.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import torch
|
|
import triton.language as tl
|
|
from triton import jit
|
|
from triton.language import target_info
|
|
from triton.runtime.jit import constexpr_function
|
|
|
|
def _env_force() -> bool:
|
|
return os.environ.get("FREETOKEN_FORCE_E4M3_EMU", "").lower() in ("1", "true", "yes", "on")
|
|
|
|
|
|
FORCE_EMU = _env_force()
|
|
|
|
if FORCE_EMU and "TRITON_CACHE_DIR" not in os.environ:
|
|
os.environ["TRITON_CACHE_DIR"] = os.path.join(
|
|
os.path.expanduser("~/.triton"), "cache-e4m3emu")
|
|
|
|
_native: bool | None = None
|
|
|
|
|
|
def e4m3_native() -> bool:
|
|
"""Host-side twin of :func:`e4m3_native_cx`: True when kernels take fp8e4nv
|
|
tensors directly. False: pass ``.view(torch.uint8)`` and bf16 act buffers."""
|
|
global _native
|
|
if _env_force() != FORCE_EMU:
|
|
raise RuntimeError(
|
|
"FREETOKEN_FORCE_E4M3_EMU changed after import: the flag is read once at "
|
|
"import and is not part of triton's compile cache key -- set it before "
|
|
"the process starts (with its own TRITON_CACHE_DIR)"
|
|
)
|
|
if _native is None:
|
|
if FORCE_EMU:
|
|
_native = False
|
|
elif torch.version.hip is not None:
|
|
# ROCm reports gfx1101 as capability (11, 0), which is not a CUDA
|
|
# compute capability and must not select the native fp8e4nv path.
|
|
_native = False
|
|
else:
|
|
from freetoken.gpu_select import assigned_visible_gpu
|
|
|
|
# one process runs on one GPU, so its convention is that GPU's; None (-> the current device) only before the process binds
|
|
_native = torch.cuda.get_device_capability(assigned_visible_gpu()) >= (8, 9)
|
|
return _native
|
|
|
|
|
|
def e4m3_kernel_view(t: torch.Tensor) -> torch.Tensor:
|
|
"""An e4m3 tensor as the branched kernels expect it: unchanged when native,
|
|
the uint8 view otherwise (the fp8 pointer type is illegal pre-sm_89)."""
|
|
return t if e4m3_native() else t.view(torch.uint8)
|
|
|
|
|
|
def e4m3_act_dtype() -> torch.dtype:
|
|
"""Buffer dtype for quantized activations: fp8 when native, else bf16 (every
|
|
e4m3 grid value is exactly representable)."""
|
|
return torch.float8_e4m3fn if e4m3_native() else torch.bfloat16
|
|
|
|
|
|
@constexpr_function
|
|
def _f32_cast_rounds_once_cx():
|
|
return not target_info.is_cuda() or target_info.cuda_capability_geq(9, 0)
|
|
|
|
|
|
@constexpr_function
|
|
def e4m3_native_cx():
|
|
"""Compile-time: does the compilation target have native fp8e4nv (sm_89+)?
|
|
Delegates to ``target_info`` (reads the active driver's target, so
|
|
cross-compilation tests that patch ``driver.active.get_current_target``
|
|
resolve consistently)."""
|
|
return not FORCE_EMU and target_info.cuda_capability_geq(8, 9)
|
|
|
|
|
|
@jit
|
|
def e4m3_u8_to_f32(v):
|
|
"""Decode e4m3 bits (uint8) to fp32: place exp+mantissa in the fp16 field
|
|
(exact, incl. e4m3 subnormals) and rescale by 2^(15-7). NaN codes -> +-480."""
|
|
h = ((v & 0x80).to(tl.uint16) << 8) | ((v & 0x7F).to(tl.uint16) << 7)
|
|
return h.to(tl.float16, bitcast=True).to(tl.float32) * 256.0
|
|
|
|
|
|
@jit
|
|
def e4m3_u8_to_f16_x128(v):
|
|
"""Decode e4m3 bits (uint8) to fp16 pre-scaled by 128 (the nvfp4 GEMM form:
|
|
native is ``.to(tl.float16) * 128``). (val/256) * 2^15 stays in fp16 range
|
|
(max 448*128 = 57344) and is exact (power-of-two scaling)."""
|
|
h = ((v & 0x80).to(tl.uint16) << 8) | ((v & 0x7F).to(tl.uint16) << 7)
|
|
return h.to(tl.float16, bitcast=True) * 32768.0
|
|
|
|
|
|
@jit
|
|
def e4m3_f32_to_u8(x):
|
|
"""Encode an fp32 value that already sits on the e4m3 grid (see :func:`round_e4m3`, |x| <= 448)
|
|
into its e4m3 bit pattern (uint8). Normals: fp32 exponent rebased 127 -> 7 with the top three
|
|
mantissa bits; subnormals (|x| < 2^-6, grid 2^-9): mantissa = |x| / 2^-9, exponent field 0."""
|
|
b = x.to(tl.uint32, bitcast=True)
|
|
sign = ((b >> 31) & 1).to(tl.uint8) << 7
|
|
exp = ((b >> 23) & 0xFF).to(tl.int32) - 127 + 7
|
|
man = ((b >> 20) & 0x7).to(tl.uint8)
|
|
normal = (exp.to(tl.uint8) << 3) | man
|
|
sub = (tl.abs(x) * 512.0).to(tl.uint8) # 0..7 on the 2^-9 grid
|
|
return sign | tl.where(tl.abs(x) >= 0.015625, normal, sub)
|
|
|
|
|
|
@jit
|
|
def round_e4m3(x):
|
|
"""Round fp32 onto the e4m3 value grid (RNE), fp32 -> fp32, in a SINGLE
|
|
rounding step -- an fp32 -> fp16 -> 3-bit chain double-rounds when the fp16
|
|
result lands exactly on an e4m3 tie. Caller clamps to +-448 first.
|
|
|
|
Normal range: RNE-truncate the fp32 mantissa 23 -> 3 bits with the integer
|
|
round-half-to-even trick (carry into the exponent rounds up correctly; it
|
|
cannot reach the sign bit for |x| <= 448). Subnormal range (|x| < 2^-6, grid
|
|
fixed at 2^-9): quantize via the add-magic trick -- at magnitude 2^14 the
|
|
fp32 ulp is exactly 2^-9, so the add rounds RNE onto the grid and the
|
|
subtract is exact."""
|
|
b = x.to(tl.uint32, bitcast=True)
|
|
lsb = (b >> 20) & 1
|
|
y_norm = ((b + 524287 + lsb) & 0xFFF00000).to(tl.float32, bitcast=True)
|
|
y_sub = (x + 24576.0) - 24576.0
|
|
return tl.where(tl.abs(x) >= 0.015625, y_norm, y_sub)
|
|
|
|
|
|
@jit
|
|
def f32_to_e4m3(x):
|
|
"""Native-branch fp32 -> fp8e4nv cast with one RNE rounding (satfinite).
|
|
|
|
Triton 3.8 lowers this cast on sm_89 as fp32 -RTZ-> fp16 -RNE-> e4m3, which double-rounds a
|
|
value just above an e4m3 midpoint (triton-lang/triton#11753); emit the direct cvt there."""
|
|
if _f32_cast_rounds_once_cx():
|
|
return x.to(tl.float8e4nv)
|
|
# FIXME: remove the sm_89 inline asm once the minimum Triton is 3.9 (triton-lang/triton#11924).
|
|
# cvt packs its first source into the high byte, so the low byte is fp8(x)
|
|
h = tl.inline_asm_elementwise(
|
|
"cvt.rn.satfinite.e4m3x2.f32 $0, 0f00000000, $1;", "=h,r", [x], dtype=tl.uint16, is_pure=True, pack=1)
|
|
return h.to(tl.uint8).to(tl.float8e4nv, bitcast=True)
|