mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
83 lines
3.5 KiB
Python
83 lines
3.5 KiB
Python
"""NVFP4 helpers shared by the Gluon kernels."""
|
|
|
|
from triton.experimental import gluon
|
|
from triton.experimental.gluon import language as gl
|
|
from triton.experimental.gluon.language import _core as ttgl
|
|
from triton.experimental.gluon.language._layouts import (
|
|
DotOperandLayout,
|
|
NVMMADistributedLayout,
|
|
)
|
|
|
|
|
|
@gluon.jit
|
|
def bf16x2_mul(a, b):
|
|
# sm_90+; sm_80-89 need fma.rn.bf16x2 with a -0.0 addend
|
|
return gl.inline_asm_elementwise("mul.rn.bf16x2 $0, $1, $2;", "=r,r,r", [a, b], gl.uint32, True, 1)
|
|
|
|
|
|
@gluon.jit
|
|
def prmt(a, b, sel: gl.constexpr):
|
|
return gl.inline_asm_elementwise(f"prmt.b32 $0, $1, $2, {sel};", "=r,r,r", [a, b], gl.uint32, True, 1)
|
|
|
|
|
|
@gluon.jit
|
|
def scale_bf16x2(s):
|
|
"""e4m3 scale bytes -> bf16x2 words holding scale * 2^118 twice."""
|
|
# 2^118 * the 2^-126 of e2m1x8_bf16x2 leaves 2^-8 for the caller to undo; every step is exact in bf16
|
|
s = (s.to(gl.float8e4nv, bitcast=True).to(gl.float32) * 3.3230699894622794e35).to(gl.bfloat16)
|
|
s = s.to(gl.uint16, bitcast=True).to(gl.uint32)
|
|
return s | (s << 16)
|
|
|
|
|
|
@gluon.jit
|
|
def e2m1x8_bf16x2(q, s2):
|
|
"""8 e2m1 codes in a uint32 -> 4 bf16x2 words in k order, each value times the bf16x2 scale s2 and 2^-126."""
|
|
# each 16-bit half's top nibble lands in a bf16 as sign + exponent/mantissa bits, a value of e2m1 * 2^-126
|
|
o0 = (q & 0x80008000) | ((q >> 6) & 0x01C001C0)
|
|
q1 = q << 4
|
|
o1 = (q1 & 0x80008000) | ((q1 >> 6) & 0x01C001C0)
|
|
q2 = q << 8
|
|
o2 = (q2 & 0x80008000) | ((q2 >> 6) & 0x01C001C0)
|
|
q3 = q << 12
|
|
o3 = (q3 & 0x80008000) | ((q3 >> 6) & 0x01C001C0)
|
|
# o0..o3 hold the k pairs (3, 7), (2, 6), (1, 5), (0, 4)
|
|
w0 = bf16x2_mul(prmt(o3, o2, 0x5410), s2)
|
|
w1 = bf16x2_mul(prmt(o1, o0, 0x5410), s2)
|
|
w2 = bf16x2_mul(prmt(o3, o2, 0x7632), s2)
|
|
w3 = bf16x2_mul(prmt(o1, o0, 0x7632), s2)
|
|
return w0, w1, w2, w3
|
|
|
|
|
|
def scale_layout(op_idx: int, rows: int, kgroups: int, mma_warps: list[int]) -> gl.DistributedLinearLayout:
|
|
"""Layout of a [rows, K / 16] e4m3 scale tile for operand op_idx of an NVFP4 mma_v2; mirrors Triton's getSM120DotScaledScaleLayout."""
|
|
def bits(n):
|
|
return n.bit_length() - 1
|
|
|
|
w0, w1 = mma_warps
|
|
reg = [[0, 1 << i] for i in range(bits(kgroups))]
|
|
if op_idx == 0:
|
|
lane = [[8, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
|
|
warp = [[0, 0]] * bits(w1) + [[16 << i, 0] for i in range(bits(w0))]
|
|
cover = 16 * w0
|
|
else:
|
|
lane = [[0, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
|
|
warp = [[8 << i, 0] for i in range(bits(w1))] + [[0, 0]] * bits(w0)
|
|
cover = 8 * w1
|
|
reg += [[cover << i, 0] for i in range(max(0, bits(rows) - bits(cover)))]
|
|
# bases past the tile's rows broadcast
|
|
lane = [[b[0] if b[0] < rows else 0, b[1]] for b in lane]
|
|
warp = [[b[0] if b[0] < rows else 0, b[1]] for b in warp]
|
|
return gl.DistributedLinearLayout(reg, lane, warp, [], [rows, kgroups])
|
|
|
|
|
|
@ttgl.builtin
|
|
def mma_nvf4(a, a_scale, b, b_scale, acc, _semantic=None):
|
|
"""acc += (a * a_scale) @ (b * b_scale) on sm_120 (mma kind::mxf4nvf4, scale_vec::4X).
|
|
|
|
a [M, K/2] and b [K/2, N] are packed e2m1 in DotOperandLayout(k_width=4) of acc's mma_v2 layout; the e4m3 scales are [M, K/16] / [N, K/16] in scale_layout.
|
|
"""
|
|
assert isinstance(acc.type.layout, NVMMADistributedLayout) and acc.type.layout.version == [2, 0]
|
|
assert isinstance(a.type.layout, DotOperandLayout) and isinstance(b.type.layout, DotOperandLayout)
|
|
out = _semantic.dot_scaled(a, a_scale, "e2m1", b, b_scale, "e2m1", acc, False, True, True, ttgl.float32)
|
|
return ttgl.tensor(out.handle, acc.type)
|