Files

83 lines
3.5 KiB
Python

"""NVFP4 helpers shared by the Gluon kernels."""
from triton.experimental import gluon
from triton.experimental.gluon import language as gl
from triton.experimental.gluon.language import _core as ttgl
from triton.experimental.gluon.language._layouts import (
DotOperandLayout,
NVMMADistributedLayout,
)
@gluon.jit
def bf16x2_mul(a, b):
# sm_90+; sm_80-89 need fma.rn.bf16x2 with a -0.0 addend
return gl.inline_asm_elementwise("mul.rn.bf16x2 $0, $1, $2;", "=r,r,r", [a, b], gl.uint32, True, 1)
@gluon.jit
def prmt(a, b, sel: gl.constexpr):
return gl.inline_asm_elementwise(f"prmt.b32 $0, $1, $2, {sel};", "=r,r,r", [a, b], gl.uint32, True, 1)
@gluon.jit
def scale_bf16x2(s):
"""e4m3 scale bytes -> bf16x2 words holding scale * 2^118 twice."""
# 2^118 * the 2^-126 of e2m1x8_bf16x2 leaves 2^-8 for the caller to undo; every step is exact in bf16
s = (s.to(gl.float8e4nv, bitcast=True).to(gl.float32) * 3.3230699894622794e35).to(gl.bfloat16)
s = s.to(gl.uint16, bitcast=True).to(gl.uint32)
return s | (s << 16)
@gluon.jit
def e2m1x8_bf16x2(q, s2):
"""8 e2m1 codes in a uint32 -> 4 bf16x2 words in k order, each value times the bf16x2 scale s2 and 2^-126."""
# each 16-bit half's top nibble lands in a bf16 as sign + exponent/mantissa bits, a value of e2m1 * 2^-126
o0 = (q & 0x80008000) | ((q >> 6) & 0x01C001C0)
q1 = q << 4
o1 = (q1 & 0x80008000) | ((q1 >> 6) & 0x01C001C0)
q2 = q << 8
o2 = (q2 & 0x80008000) | ((q2 >> 6) & 0x01C001C0)
q3 = q << 12
o3 = (q3 & 0x80008000) | ((q3 >> 6) & 0x01C001C0)
# o0..o3 hold the k pairs (3, 7), (2, 6), (1, 5), (0, 4)
w0 = bf16x2_mul(prmt(o3, o2, 0x5410), s2)
w1 = bf16x2_mul(prmt(o1, o0, 0x5410), s2)
w2 = bf16x2_mul(prmt(o3, o2, 0x7632), s2)
w3 = bf16x2_mul(prmt(o1, o0, 0x7632), s2)
return w0, w1, w2, w3
def scale_layout(op_idx: int, rows: int, kgroups: int, mma_warps: list[int]) -> gl.DistributedLinearLayout:
"""Layout of a [rows, K / 16] e4m3 scale tile for operand op_idx of an NVFP4 mma_v2; mirrors Triton's getSM120DotScaledScaleLayout."""
def bits(n):
return n.bit_length() - 1
w0, w1 = mma_warps
reg = [[0, 1 << i] for i in range(bits(kgroups))]
if op_idx == 0:
lane = [[8, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
warp = [[0, 0]] * bits(w1) + [[16 << i, 0] for i in range(bits(w0))]
cover = 16 * w0
else:
lane = [[0, 0], [0, 0], [1, 0], [2, 0], [4, 0]]
warp = [[8 << i, 0] for i in range(bits(w1))] + [[0, 0]] * bits(w0)
cover = 8 * w1
reg += [[cover << i, 0] for i in range(max(0, bits(rows) - bits(cover)))]
# bases past the tile's rows broadcast
lane = [[b[0] if b[0] < rows else 0, b[1]] for b in lane]
warp = [[b[0] if b[0] < rows else 0, b[1]] for b in warp]
return gl.DistributedLinearLayout(reg, lane, warp, [], [rows, kgroups])
@ttgl.builtin
def mma_nvf4(a, a_scale, b, b_scale, acc, _semantic=None):
"""acc += (a * a_scale) @ (b * b_scale) on sm_120 (mma kind::mxf4nvf4, scale_vec::4X).
a [M, K/2] and b [K/2, N] are packed e2m1 in DotOperandLayout(k_width=4) of acc's mma_v2 layout; the e4m3 scales are [M, K/16] / [N, K/16] in scale_layout.
"""
assert isinstance(acc.type.layout, NVMMADistributedLayout) and acc.type.layout.version == [2, 0]
assert isinstance(a.type.layout, DotOperandLayout) and isinstance(b.type.layout, DotOperandLayout)
out = _semantic.dot_scaled(a, a_scale, "e2m1", b, b_scale, "e2m1", acc, False, True, True, ttgl.float32)
return ttgl.tensor(out.handle, acc.type)