Files
Xiaoze Fan 3d23551c5b perf(cpu-moe): speed up the nvfp4 W4A8 kernels (#851)
The VNNI dots were compute bound per core; with fewer instructions per weight they reach the DRAM read bandwidth.
The W4A8 kernel now follows FREETOKEN_CPU_MOE_ISA.
2026-11-02 15:05:17 -08:00

557 lines
27 KiB
C++

#include <vector>
#include "../format.h"
namespace freetoken::cpu_moe {
namespace {
// --------------------------- NVFP4 (W4A16) dequant ---------------------------
// Weights: e2m1 4-bit codes (2/byte, low nibble first), per-16 block scale in
// fp8-e4m3, per-output-row global scale in fp32. Dequant matches the GPU kernels
// (freetoken/kernel/triton/nvfp4_dequant.py): w = E2M1[code] * e4m3(scale) * global.
// Activations stay bf16 (W4A16); the GEMV dequantizes weights inside the K-loop.
// e2m1 * 2 as exact int8 (all e2m1 values are multiples of 0.5); the W4A8 dots undo the *2 with a 0.5 in the final scale. Mirrors ggml's kvalues_mxfp4.
alignas(16) const int8_t kE2M1x2[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};
// kE2M1x2 + kE2M1Bias, the PSHUFB LUT of the VNNI dots: VPDPBUSD takes the weight as the unsigned operand, and W4a8Act::corr carries the -kE2M1Bias * sum(a) correction.
alignas(16) const uint8_t kE2M1x2u[16] = {12, 13, 14, 15, 16, 18, 20, 24, 12, 11, 10, 9, 8, 6, 4, 0};
constexpr int kE2M1Bias = 12;
// Activations pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]); see the
// ds_fp4 dot in ds_fp4.cpp for why (drops the hot loop to ~1.5 shuffle ops / 16 weights).
using nvdot_fn = float (*)(const uint8_t*, const uint8_t*, float, const float*, const float*,
int, const float*, const float*);
float dot_nvfp4_scalar(const uint8_t* packed, const uint8_t* scale, float global,
const float* xe, const float* xo, int K, const float* e2m1,
const float* e4m3) {
float acc = 0.0f;
const int nb = K / 16;
for (int b = 0; b < nb; ++b) {
const float bs = e4m3[scale[b]];
const uint8_t* pk = packed + (size_t)b * 8;
const float* xeb = xe + (size_t)b * 8; // 16 K -> 8 even + 8 odd
const float* xob = xo + (size_t)b * 8;
float bsum = 0.0f;
for (int j = 0; j < 8; ++j) {
const uint8_t byte = pk[j];
bsum += e2m1[byte & 0xF] * xeb[j];
bsum += e2m1[byte >> 4] * xob[j];
}
acc += bs * bsum;
}
return acc * global;
}
// ---- NVFP4 W4A8 (int8 activations) dot: nibble->int8 via LUT, per-16 act scale ----
// Activations are grouped in chunks of 8 16-K blocks: the even-K int8 of the chunk's blocks, then its odd-K int8.
// A 64-byte load of packed weights then splits into low (even-K) and high (odd-K) nibbles that line up with the activations, and int32 lane l of a dot holds block l / 2.
struct W4a8Act {
const int8_t* q; // [K]
const float* scale; // [K/16], absmax / 127 of each block
const int32_t* corr; // [K/8], one per int32 lane: -kE2M1Bias * the sum of the 8 activations it multiplies
};
using nvi8dot_fn = float (*)(const uint8_t*, const uint8_t*, float, const W4a8Act&, int);
using nvi8quant_fn = void (*)(const bf16_t*, int, int8_t*, float*, int32_t*);
#if CPU_MOE_X86
// Two 16-K blocks (16 packed bytes) per iter: lo nibbles -> even-K, hi -> odd-K,
// gathered via two vpermps. The per-16 e4m3 scale differs across the two blocks, so
// it is applied per lane (low 8 lanes = block b, high 8 = block b+1).
CPU_MOE_TARGET("avx512f")
inline __m512 nvfp4_blk2(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,
__m512i loma, float s0, float s1) {
__m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));
__m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);
__m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);
__m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));
// lanes 0-7 (block b) -> s0, lanes 8-15 (block b+1) -> s1 (pure AVX512F mask move)
__m512 scv = _mm512_mask_mov_ps(_mm512_set1_ps(s0), 0xFF00, _mm512_set1_ps(s1));
return _mm512_mul_ps(prod, scv);
}
CPU_MOE_TARGET("avx512f")
float dot_nvfp4_avx512(const uint8_t* packed, const uint8_t* scale, float global,
const float* xe, const float* xo, int K, const float* e2m1,
const float* e4m3) {
const __m512 lut = _mm512_loadu_ps(e2m1); // 16 e2m1 values
const __m512i loma = _mm512_set1_epi32(0xF);
__m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();
const int nb = K / 16; // 8 packed bytes + one e4m3 scale per block
int b = 0;
for (; b + 4 <= nb; b += 4) { // two blk2 calls -> 4 blocks, 2 accumulators
acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,
xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));
acc1 = _mm512_add_ps(acc1, nvfp4_blk2(packed + (size_t)(b + 2) * 8, xe + (size_t)(b + 2) * 8,
xo + (size_t)(b + 2) * 8, lut, loma, e4m3[scale[b + 2]], e4m3[scale[b + 3]]));
}
for (; b + 2 <= nb; b += 2)
acc0 = _mm512_add_ps(acc0, nvfp4_blk2(packed + (size_t)b * 8, xe + (size_t)b * 8,
xo + (size_t)b * 8, lut, loma, e4m3[scale[b]], e4m3[scale[b + 1]]));
float s = _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));
for (; b < nb; ++b) { // odd final 16-K block
const uint8_t* pk = packed + (size_t)b * 8;
const float* xeb = xe + (size_t)b * 8;
const float* xob = xo + (size_t)b * 8;
float bsum = 0.0f;
for (int j = 0; j < 8; ++j) {
bsum += e2m1[pk[j] & 0xF] * xeb[j] + e2m1[pk[j] >> 4] * xob[j];
}
s += e4m3[scale[b]] * bsum;
}
return s * global;
}
// AVX2: one 16-K block (8 packed bytes) per call, 8 even + 8 odd lanes.
CPU_MOE_TARGET("avx2,fma")
inline __m256 nvfp4_blk_avx2(const uint8_t* pk, const float* xeb, const float* xob,
__m256 mag8, float sc) {
__m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));
__m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);
__m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);
__m256 prod = _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));
return _mm256_mul_ps(prod, _mm256_set1_ps(sc));
}
CPU_MOE_TARGET("avx2,fma")
float dot_nvfp4_avx2(const uint8_t* packed, const uint8_t* scale, float global,
const float* xe, const float* xo, int K, const float* e2m1,
const float* e4m3) {
const __m256 mag8 = _mm256_loadu_ps(e2m1); // e2m1[0..7] magnitudes
__m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();
const int nb = K / 16;
int b = 0;
for (; b + 2 <= nb; b += 2) {
acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,
xo + (size_t)b * 8, mag8, e4m3[scale[b]]));
acc1 = _mm256_add_ps(acc1, nvfp4_blk_avx2(packed + (size_t)(b + 1) * 8, xe + (size_t)(b + 1) * 8,
xo + (size_t)(b + 1) * 8, mag8, e4m3[scale[b + 1]]));
}
for (; b < nb; ++b)
acc0 = _mm256_add_ps(acc0, nvfp4_blk_avx2(packed + (size_t)b * 8, xe + (size_t)b * 8,
xo + (size_t)b * 8, mag8, e4m3[scale[b]]));
return hsum256(_mm256_add_ps(acc0, acc1)) * global;
}
// Software-prefetch distance for the W4A8 weight stream, in 16-K blocks (8 packed
// bytes each). Returns -1 when FREETOKEN_CPU_MOE_PF_BLOCKS is unset: the kernel then
// uses the built-in default min(512 blocks = 4 KB, 2 rows) -- 4 KB is the empirical
// optimum on large-row machines (Emerald Rapids sweep), while the 2-row cap keeps a small-row
// model's overshoot bounded (the executor works in 32-row tiles, so a fixed byte
// distance otherwise prefetches another worker's tile: duplicated DRAM traffic that
// regresses at the bandwidth ceiling). An EXPLICIT env value is honored verbatim
// (no clamp; 0 disables): the per-machine optimum can sit past the safe default (+20%
// at 4 KB on a 24-thread Ice Lake with 256B rows), so the escape hatch must reach it.
// Prefetch never faults, so overshooting a row/bank tail is safe.
static int nvfp4_pf_blocks() {
static const int v = [] {
const char* s = getenv("FREETOKEN_CPU_MOE_PF_BLOCKS");
return (s && s[0]) ? atoi(s) : -1;
}();
return v;
}
// e4m3 -> fp16 is a shift: exponent and mantissa move up 7 bits, and the bias gap (15 - 7) leaves a 2^-8 that the kernels undo in the final scale.
// The byte is sign-extended first, so its sign also lands in bit 15 and & 0xBF80 keeps it.
constexpr float kE4m3ViaF16 = 256.0f;
constexpr int16_t kE4m3F16Mask = (int16_t)0xBF80;
int w4a8_pf(int nb) {
const int pfb = nvfp4_pf_blocks();
return (pfb < 0) ? std::min(512, 2 * nb) : pfb;
}
// Weight scale * act scale of 8 blocks, in 2^-8 units.
CPU_MOE_TARGET("avx2,fma,f16c")
inline __m256 w4a8_scales256(__m128i s8, const float* asc) {
__m128i h = _mm_cvtepi8_epi16(s8);
h = _mm_and_si128(_mm_slli_epi16(h, 7), _mm_set1_epi16(kE4m3F16Mask));
return _mm256_mul_ps(_mm256_cvtph_ps(h), _mm256_loadu_ps(asc));
}
CPU_MOE_TARGET("avx2,fma,f16c")
inline void w4a8_split256(__m256i v, __m256i lut, __m256i& lo, __m256i& hi) {
const __m256i m4 = _mm256_set1_epi8(0x0F);
lo = _mm256_shuffle_epi8(lut, _mm256_and_si256(v, m4));
hi = _mm256_shuffle_epi8(lut, _mm256_and_si256(_mm256_srli_epi16(v, 4), m4));
}
CPU_MOE_TARGET("avx2,fma,f16c,avxvnni")
inline __m256i w4a8_dot256_vnni(__m256i corr, __m256i lo, __m256i qe, __m256i hi, __m256i qo) {
return _mm256_dpbusd_avx_epi32(_mm256_dpbusd_avx_epi32(corr, lo, qe), hi, qo);
}
// sum of e4m3(scale) * act scale * int dot over blocks [b0, nb) in 2^-8 units, before the final 0.5 * global
CPU_MOE_TARGET("avx2,fma,f16c")
float w4a8_blocks_scalar(const uint8_t* packed, const uint8_t* scale, const W4a8Act& a, int b0, int nb) {
float acc = 0.0f;
for (int b = b0; b < nb; ++b) {
const int c0 = b & ~7, nbc = std::min(8, nb - c0);
const int8_t* qe = a.q + (size_t)c0 * 16 + (size_t)(b - c0) * 8;
const int8_t* qo = qe + nbc * 8;
const uint8_t* pk = packed + (size_t)b * 8;
int isum = 0;
for (int j = 0; j < 8; ++j) isum += kE2M1x2[pk[j] & 0xF] * qe[j] + kE2M1x2[pk[j] >> 4] * qo[j];
acc += (_cvtsh_ss((uint16_t)(((int16_t)(int8_t)scale[b] << 7) & kE4m3F16Mask)) * a.scale[b]) * (float)isum;
}
return acc;
}
CPU_MOE_TARGET("avx2,fma,f16c,avxvnni")
float dot_nvfp4_i8_vnni(const uint8_t* packed, const uint8_t* scale, float global, const W4a8Act& a, int K) {
const __m256i lut = _mm256_broadcastsi128_si256(_mm_load_si128(reinterpret_cast<const __m128i*>(kE2M1x2u)));
const __m256i lanes_lo = _mm256_set_epi32(3, 3, 2, 2, 1, 1, 0, 0);
const __m256i lanes_hi = _mm256_set_epi32(7, 7, 6, 6, 5, 5, 4, 4);
const int nb = K / 16, pf = w4a8_pf(nb);
__m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();
__m256i lo, hi;
int b = 0;
for (; b + 8 <= nb; b += 8) {
const uint8_t* pk = packed + (size_t)b * 8;
if (pf > 0) _mm_prefetch(reinterpret_cast<const char*>(pk + (size_t)pf * 8), _MM_HINT_T0);
const int8_t* q = a.q + (size_t)b * 16;
const int32_t* c = a.corr + (size_t)b * 2;
const __m256 sc = w4a8_scales256(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(scale + b)), a.scale + b);
w4a8_split256(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(pk)), lut, lo, hi);
__m256i d = w4a8_dot256_vnni(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(c)), lo,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q)), hi,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + 64)));
acc0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d), _mm256_permutevar8x32_ps(sc, lanes_lo), acc0);
w4a8_split256(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(pk + 32)), lut, lo, hi);
d = w4a8_dot256_vnni(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(c + 8)), lo,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + 32)), hi,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + 96)));
acc1 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d), _mm256_permutevar8x32_ps(sc, lanes_hi), acc1);
}
// A last chunk of 4..7 blocks runs its first half here (its odd-K bytes start 8 * n after the even-K ones); fewer than 4 left go scalar.
if (b + 4 <= nb) {
const int n = nb - b;
const int8_t* q = a.q + (size_t)b * 16;
int s4;
std::memcpy(&s4, scale + b, 4);
float as8[8] = {a.scale[b], a.scale[b + 1], a.scale[b + 2], a.scale[b + 3]};
const __m256 sc = w4a8_scales256(_mm_cvtsi32_si128(s4), as8);
w4a8_split256(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(packed + (size_t)b * 8)), lut, lo, hi);
const __m256i d = w4a8_dot256_vnni(_mm256_loadu_si256(reinterpret_cast<const __m256i*>(a.corr + (size_t)b * 2)), lo,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q)), hi,
_mm256_loadu_si256(reinterpret_cast<const __m256i*>(q + 8 * n)));
acc0 = _mm256_fmadd_ps(_mm256_cvtepi32_ps(d), _mm256_permutevar8x32_ps(sc, lanes_lo), acc0);
b += 4;
}
const float tail = w4a8_blocks_scalar(packed, scale, a, b, nb);
return (hsum256(_mm256_add_ps(acc0, acc1)) + tail) * (0.5f * kE4m3ViaF16 * global);
}
CPU_MOE_TARGET("avx2")
inline int hsum_epi32_256(__m256i v) {
__m128i s = _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1));
s = _mm_add_epi32(s, _mm_shuffle_epi32(s, 0x4E));
s = _mm_add_epi32(s, _mm_shuffle_epi32(s, 0xB1));
return _mm_cvtsi128_si32(s);
}
CPU_MOE_TARGET("avx2,fma")
inline __m256i w4a8_round256(__m256 y) {
const __m256 absm = _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF));
const __m256 t = _mm256_round_ps(y, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
const __m256 away = _mm256_cmp_ps(_mm256_and_ps(_mm256_sub_ps(y, t), absm), _mm256_set1_ps(0.5f), _CMP_GE_OQ);
const __m256 step = _mm256_or_ps(_mm256_and_ps(y, _mm256_set1_ps(-0.0f)), _mm256_set1_ps(1.0f));
const __m256 r = _mm256_add_ps(t, _mm256_and_ps(away, step));
return _mm256_cvtps_epi32(_mm256_min_ps(_mm256_max_ps(r, _mm256_set1_ps(-127.0f)), _mm256_set1_ps(127.0f)));
}
// The quants match a scalar absmax / 127 per-block quant that rounds with lround (half away from zero), bit for bit.
// 1 / scale overflows for absmax below ~3.7e-37, so such a block quantizes to zeros.
// Dot lanes 2b and 2b + 1 multiply the first and the last 8 activations of block b, so corr is two plain sums.
CPU_MOE_TARGET("avx2,fma")
void quant_i8_avx2(const bf16_t* x, int K, int8_t* q, float* scale, int32_t* corr) {
const __m128i even_odd = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
const __m256 absm = _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF));
const int nb = K / 16;
for (int b = 0; b < nb; ++b) {
const int c0 = b & ~7, nbc = std::min(8, nb - c0);
int8_t* qe = q + (size_t)c0 * 16 + (size_t)(b - c0) * 8;
const __m256i raw = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + (size_t)b * 16));
const __m256 v0 = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(raw)), 16));
const __m256 v1 = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm256_extracti128_si256(raw, 1)), 16));
const __m256 m8 = _mm256_max_ps(_mm256_and_ps(v0, absm), _mm256_and_ps(v1, absm));
__m128 m4 = _mm_max_ps(_mm256_castps256_ps128(m8), _mm256_extractf128_ps(m8, 1));
m4 = _mm_max_ps(m4, _mm_movehl_ps(m4, m4));
m4 = _mm_max_ss(m4, _mm_shuffle_ps(m4, m4, 0x55));
const float amax = _mm_cvtss_f32(m4);
const float s = amax > 0.0f ? amax / 127.0f : 1.0f;
scale[b] = s;
const float inv_s = 1.0f / s;
const __m256 inv = _mm256_set1_ps(inv_s < INFINITY ? inv_s : 0.0f);
const __m256i i0 = w4a8_round256(_mm256_mul_ps(v0, inv)), i1 = w4a8_round256(_mm256_mul_ps(v1, inv));
const __m256i p16 = _mm256_permute4x64_epi64(_mm256_packs_epi32(i0, i1), 0xD8);
const __m128i bytes = _mm_shuffle_epi8(_mm_packs_epi16(_mm256_castsi256_si128(p16), _mm256_extracti128_si256(p16, 1)), even_odd);
_mm_storel_epi64(reinterpret_cast<__m128i*>(qe), bytes);
_mm_storel_epi64(reinterpret_cast<__m128i*>(qe + nbc * 8), _mm_unpackhi_epi64(bytes, bytes));
corr[2 * b] = -kE2M1Bias * hsum_epi32_256(i0);
corr[2 * b + 1] = -kE2M1Bias * hsum_epi32_256(i1);
}
}
#ifdef CPU_MOE_HAS_AVX512VNNI
// Weight scale * act scale of n <= 16 blocks, in 2^-8 units.
CPU_MOE_TARGET("avx512f,avx512bw,avx512vl")
inline __m512 w4a8_scales512(const uint8_t* scale, const float* asc, int n) {
const __mmask16 m = (__mmask16)((1u << n) - 1);
__m256i h = _mm256_cvtepi8_epi16(_mm_maskz_loadu_epi8(m, scale));
h = _mm256_and_si256(_mm256_slli_epi16(h, 7), _mm256_set1_epi16(kE4m3F16Mask));
return _mm512_mul_ps(_mm512_cvtph_ps(h), _mm512_maskz_loadu_ps(m, asc));
}
CPU_MOE_TARGET("avx512f,avx512bw,avx512vl")
inline void w4a8_split512(__m512i v, __m512i lut, __m512i& lo, __m512i& hi) {
const __m512i m4 = _mm512_set1_epi8(0x0F);
lo = _mm512_shuffle_epi8(lut, _mm512_and_si512(v, m4));
hi = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(v, 4), m4));
}
CPU_MOE_TARGET("avx512f,avx512bw,avx512vl,avx512vnni")
inline __m512i w4a8_dot512_vnni(__m512i corr, __m512i lo, __m512i qe, __m512i hi, __m512i qo) {
return _mm512_dpbusd_epi32(_mm512_dpbusd_epi32(corr, lo, qe), hi, qo);
}
CPU_MOE_TARGET("avx512f,avx512bw,avx512vl,avx512vnni")
float dot_nvfp4_i8_avx512vnni(const uint8_t* packed, const uint8_t* scale, float global, const W4a8Act& a, int K) {
const __m512i lut = _mm512_broadcast_i32x4(_mm_load_si128(reinterpret_cast<const __m128i*>(kE2M1x2u)));
const __m512i lanes_lo = _mm512_set_epi32(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0);
const __m512i lanes_hi = _mm512_add_epi32(lanes_lo, _mm512_set1_epi32(8));
const int nb = K / 16, pf = w4a8_pf(nb);
__m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();
__m512i lo, hi;
int b = 0;
for (; b + 16 <= nb; b += 16) {
const uint8_t* pk = packed + (size_t)b * 8;
// The HW prefetcher alone falls short here: with pf = 0 a GLM-5.3-Flash executor step is 1.27x slower on Emerald Rapids.
if (pf > 0) {
_mm_prefetch(reinterpret_cast<const char*>(pk + (size_t)pf * 8), _MM_HINT_T0);
_mm_prefetch(reinterpret_cast<const char*>(pk + (size_t)pf * 8 + 64), _MM_HINT_T0);
}
const int8_t* q = a.q + (size_t)b * 16;
const int32_t* c = a.corr + (size_t)b * 2;
const __m512 sc = w4a8_scales512(scale + b, a.scale + b, 16);
w4a8_split512(_mm512_loadu_si512(pk), lut, lo, hi);
__m512i d = w4a8_dot512_vnni(_mm512_loadu_si512(c), lo, _mm512_loadu_si512(q), hi, _mm512_loadu_si512(q + 64));
acc0 = _mm512_fmadd_ps(_mm512_cvtepi32_ps(d), _mm512_permutexvar_ps(lanes_lo, sc), acc0);
w4a8_split512(_mm512_loadu_si512(pk + 64), lut, lo, hi);
d = w4a8_dot512_vnni(_mm512_loadu_si512(c + 16), lo, _mm512_loadu_si512(q + 128), hi, _mm512_loadu_si512(q + 192));
acc1 = _mm512_fmadd_ps(_mm512_cvtepi32_ps(d), _mm512_permutexvar_ps(lanes_hi, sc), acc1);
}
// The tail runs chunks of n <= 8 blocks with masked loads: masked-off activations are zero, so those lanes add nothing.
for (; b < nb; b += 8) {
const int n = std::min(8, nb - b);
const __mmask64 mb = n == 8 ? ~0ULL : (1ULL << (8 * n)) - 1;
const int8_t* q = a.q + (size_t)b * 16;
const __m512 sc = w4a8_scales512(scale + b, a.scale + b, n);
w4a8_split512(_mm512_maskz_loadu_epi8(mb, packed + (size_t)b * 8), lut, lo, hi);
const __m512i c = _mm512_maskz_loadu_epi32((__mmask16)((1u << (2 * n)) - 1), a.corr + (size_t)b * 2);
const __m512i d = w4a8_dot512_vnni(c, lo, _mm512_maskz_loadu_epi8(mb, q), hi, _mm512_maskz_loadu_epi8(mb, q + 8 * n));
acc0 = _mm512_fmadd_ps(_mm512_cvtepi32_ps(d), _mm512_permutexvar_ps(lanes_lo, sc), acc0);
}
return _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1)) * (0.5f * kE4m3ViaF16 * global);
}
CPU_MOE_TARGET("avx512f,avx512bw,avx512vl")
void quant_i8_avx512(const bf16_t* x, int K, int8_t* q, float* scale, int32_t* corr) {
const __m128i even_odd = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
const __m512 half = _mm512_set1_ps(0.5f);
const __m512i one = _mm512_castps_si512(_mm512_set1_ps(1.0f)), sign = _mm512_set1_epi32(INT32_MIN);
const __m512 lim = _mm512_set1_ps(127.0f), nlim = _mm512_set1_ps(-127.0f);
const int nb = K / 16;
for (int b = 0; b < nb; ++b) {
const int c0 = b & ~7, nbc = std::min(8, nb - c0);
int8_t* qe = q + (size_t)c0 * 16 + (size_t)(b - c0) * 8;
const __m256i raw = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + (size_t)b * 16));
const __m512 v = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16));
const float amax = _mm512_reduce_max_ps(_mm512_abs_ps(v));
const float s = amax > 0.0f ? amax / 127.0f : 1.0f;
scale[b] = s;
const float inv = 1.0f / s;
const __m512 y = _mm512_mul_ps(v, _mm512_set1_ps(inv < INFINITY ? inv : 0.0f));
const __m512 t = _mm512_roundscale_ps(y, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
const __mmask16 away = _mm512_cmp_ps_mask(_mm512_abs_ps(_mm512_sub_ps(y, t)), half, _CMP_GE_OQ);
const __m512 step = _mm512_castsi512_ps(_mm512_or_si512(_mm512_and_si512(_mm512_castps_si512(y), sign), one));
const __m512 r = _mm512_mask_add_ps(t, away, t, step);
const __m512i ri = _mm512_cvtps_epi32(_mm512_min_ps(_mm512_max_ps(r, nlim), lim));
const __m128i bytes = _mm_shuffle_epi8(_mm512_cvtepi32_epi8(ri), even_odd);
_mm_storel_epi64(reinterpret_cast<__m128i*>(qe), bytes);
_mm_storel_epi64(reinterpret_cast<__m128i*>(qe + nbc * 8), _mm_unpackhi_epi64(bytes, bytes));
corr[2 * b] = -kE2M1Bias * hsum_epi32_256(_mm512_castsi512_si256(ri));
corr[2 * b + 1] = -kE2M1Bias * hsum_epi32_256(_mm512_extracti64x4_epi64(ri, 1));
}
}
#endif // avx512vnni available
#endif
nvdot_fn select_nvdot() {
const IsaTier t = pick_isa();
#if CPU_MOE_X86
if (t >= ISA_AVX512) return dot_nvfp4_avx512;
if (t >= ISA_AVX2) return dot_nvfp4_avx2;
#endif
(void)t;
return dot_nvfp4_scalar;
}
struct W4a8Kernel {
nvi8dot_fn dot;
nvi8quant_fn quant;
const char* isa;
};
// Best W4A8 (int8-activation) nvfp4 dot for the picked ISA tier, or a null dot if that tier has no VNNI (caller keeps the faithful fp32 nvdot path).
W4a8Kernel select_nvi8dot() {
#if CPU_MOE_X86
const IsaTier t = pick_isa();
#if defined(CPU_MOE_HAS_AVX512VNNI)
if (t >= ISA_AVX512 && cpu_has_avx512vnni()) return {dot_nvfp4_i8_avx512vnni, quant_i8_avx512, "+avx512vnni(nvfp4-w4a8)"};
#endif
if (t >= ISA_AVX2 && cpu_has_avxvnni()) return {dot_nvfp4_i8_vnni, quant_i8_avx2, "+vnni(nvfp4-w4a8)"};
#endif
return {};
}
struct W4a8Rows {
AlignedVector<int8_t> q;
AlignedVector<float> scale;
AlignedVector<int32_t> corr;
int K = 0;
void resize(size_t rows, int k) {
K = k;
q.resize(rows * k);
scale.resize(rows * (k / 16));
corr.resize(rows * (k / 8));
}
void quant(nvi8quant_fn fn, size_t row, const bf16_t* x) {
fn(x, K, q.data() + row * K, scale.data() + row * (K / 16), corr.data() + row * (K / 8));
}
W4a8Act at(size_t row) const {
return {q.data() + row * K, scale.data() + row * (K / 16), corr.data() + row * (K / 8)};
}
};
class Nvfp4Format final : public MoeFormat {
public:
Nvfp4Format(const MoeParams& p, const FormatArgs& a)
: MoeFormat(p), nvdot_(select_nvdot()), w4a8_(select_nvi8dot()) {
gate_up_ = a.bank("gate_up");
gu_scale_ = a.bank("gate_up_scale");
gu_global_ = a.bank("gate_up_global");
down_ = a.bank("down");
dn_scale_ = a.bank("down_scale");
dn_global_ = a.bank("down_global");
const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;
if (w4a8_.dot != nullptr) {
xq_.resize(tokens, p.H);
gq_.resize(routes, p.I);
} else {
xe_.resize(tokens * (p.H / 2));
xo_.resize(tokens * (p.H / 2));
ge_.resize(routes * (p.I / 2));
go_.resize(routes * (p.I / 2));
}
isa_ = tier_name(pick_isa());
if (w4a8_.dot != nullptr) isa_ += w4a8_.isa;
for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];
for (int i = 0; i < 256; ++i) e4m3_lut_[i] = e4m3_decode((uint8_t)i);
}
std::string isa_name() const override { return isa_; }
void prep_input(int tok, const bf16_t* x) override {
if (w4a8_.dot != nullptr) {
xq_.quant(w4a8_.quant, tok, x);
return;
}
float* xe = xe_.data() + (size_t)tok * (p.H / 2);
float* xo = xo_.data() + (size_t)tok * (p.H / 2);
deinterleave_bf16_f32(x, xe, xo, p.H);
}
bool preps_inter() const override { return true; }
void prep_inter(int64_t r, bf16_t* g) override {
if (w4a8_.dot != nullptr) {
gq_.quant(w4a8_.quant, r, g);
return;
}
float* ge = ge_.data() + (size_t)r * (p.I / 2);
float* go = go_.data() + (size_t)r * (p.I / 2);
deinterleave_bf16_f32(g, ge, go, p.I);
}
void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float w_in,
bf16_t* g) const override {
const int H = p.H;
const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));
const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));
const float* global = static_cast<const float*>(tbl_at(gu_global_, layer));
if (w4a8_.dot != nullptr) {
const W4a8Act act = xq_.at(tok);
gated_rows(p, i0, i1, w_in, g, [&](int row) {
const size_t r = (size_t)e * (2 * p.I) + row;
return w4a8_.dot(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16), global[r], act, H);
});
return;
}
const float* xe = xe_.data() + (size_t)tok * (H / 2);
const float* xo = xo_.data() + (size_t)tok * (H / 2);
gated_rows(p, i0, i1, w_in, g, [&](int row) {
const size_t r = (size_t)e * (2 * p.I) + row;
return nvdot_(packed + r * (size_t)(H / 2), scale + r * (size_t)(H / 16),
global[r], xe, xo, H, e2m1_lut_, e4m3_lut_);
});
}
void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {
const int I = p.I;
const uint8_t* packed = static_cast<const uint8_t*>(tbl_at(down_, layer));
const uint8_t* scale = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));
const float* global = static_cast<const float*>(tbl_at(dn_global_, layer));
for (int h = h0; h < h1; ++h) {
for (int j = 0; j < n; ++j) {
const size_t r = (size_t)routes[j].r;
const size_t row = (size_t)routes[j].e * p.H + h;
out[j * kRowBlock + h - h0] = w4a8_.dot != nullptr
? w4a8_.dot(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row], gq_.at(r), I)
: nvdot_(packed + row * (size_t)(I / 2), scale + row * (size_t)(I / 16), global[row],
ge_.data() + r * (I / 2), go_.data() + r * (I / 2), I, e2m1_lut_, e4m3_lut_);
}
}
}
private:
nvdot_fn nvdot_;
const W4a8Kernel w4a8_; // null dot -> use fp32 nvdot
// ModelOpt rows: packed e2m1 [E,2I,H/2], e4m3 scales per 16 along K [E,2I,H/16], fp32 row globals [E,2I]; down likewise with K = I.
const uint64_t* gate_up_ = nullptr;
const uint64_t* gu_scale_ = nullptr;
const uint64_t* gu_global_ = nullptr;
const uint64_t* down_ = nullptr;
const uint64_t* dn_scale_ = nullptr;
const uint64_t* dn_global_ = nullptr;
std::string isa_;
float e2m1_lut_[16];
float e4m3_lut_[256];
// fp32 even/odd activations for the fp32 dot, or the W4A8 int8 rows.
AlignedVector<float> xe_, xo_, ge_, go_;
W4a8Rows xq_, gq_;
};
} // namespace
std::unique_ptr<MoeFormat> make_nvfp4_format(const MoeParams& p, const FormatArgs& a) {
return std::make_unique<Nvfp4Format>(p, a);
}
} // namespace freetoken::cpu_moe