Files
Xiaoze Fanandzihaomu 2010271266 refactor(cpu-moe): split the executor into weight formats and gpu sync backends (#670)
* refactor(cpu-moe): split the executor into weight formats and gpu sync backends

- move each expert weight format into kernels/<fmt>.cpp behind the MoeFormat
  interface in format.h; the executor keeps the threads, the work split and
  the pass 2 route sum
- pass each format its banks and integer options by name; MoeParams keeps only
  the shared shape and activation fields
- key the formats by name in one table that weight_formats() exports, replacing
  the integer format ids and the fp8_block_isa_name hook
- replace the per-(layer, batch size) flag slots, pinned IO buffers and task
  table with one doorbell word MoeTask{layer, num_tokens} and one shared pinned
  IO block of max_tokens rows
- move the GPU/CPU handshake behind GpuSync with a CUDA and a HIP backend picked
  by setup.py; HIP waits on signal memory and adds explicit batch-memop graph
  nodes where the runtime does not record the native memops under capture
- raise on graph capture when the handshake is off and host functions are not
  replay-safe
- allocate format scratch on 64-byte boundaries
- size max_tokens by the explicit cuda_graph_bs as well

Co-authored-by: zihaomu <zihaomu@outlook.com>

* perf(cpu-moe): skip the worker pool for tasks without routed experts

* perf(cpu-moe): publish and combine the cpu handshake with kernels

* fix(cpu-moe): clean up failed hip capability probes

---------

Co-authored-by: zihaomu <zihaomu@outlook.com>
2026-10-13 20:12:58 -07:00

270 lines
12 KiB
C++

#include <vector>
#include "../format.h"
namespace freetoken::cpu_moe {
namespace {
// ----------------------- DeepSeek-V4 ds_fp4 (W4A8) ---------------------------
// Row-major e2m1 (2/byte, low nibble first) + e8m0 per-32 block scale, no global
// (w = E2M1[code] * 2^(e8m0-127)); activations are FP8-e4m3 round-tripped (per-128
// block, ue8m0 scale) before each GEMM. Matches kernel/triton/dsv4 (fused_moe +
// fp8_linear): silu(clamp(gate,max=lim)) * clamp(up,-lim,lim), router weight on the
// down output.
// Activations are pre-deinterleaved to fp32 (xe[m]=x[2m], xo[m]=x[2m+1]) once per
// token/route and reused across all output rows. This drops the hot dot to one
// vpmovzxbd + two vpermps per 32 weights (1.5 shuffle ops / 16 vs 4 for a bf16,
// dup-permute, per-element gather), so the row-major fp4 GEMV stops being port-5
// bound and approaches the bf16 memory-bandwidth ceiling.
using dsdot_fn = float (*)(const uint8_t*, const uint8_t*, const float*, const float*, int,
const float*, const float*);
float dot_dsfp4_scalar(const uint8_t* packed, const uint8_t* scale, const float* xe,
const float* xo, int K, const float* e2m1, const float* e8m0) {
float acc = 0.0f;
const int nb = K / 32;
for (int b = 0; b < nb; ++b) {
const float sc = e8m0[scale[b]];
const uint8_t* pk = packed + (size_t)b * 16; // 16 bytes = 32 codes
const float* xeb = xe + (size_t)b * 16;
const float* xob = xo + (size_t)b * 16;
float bsum = 0.0f;
for (int j = 0; j < 16; ++j) {
const uint8_t byte = pk[j];
bsum += e2m1[byte & 0xF] * xeb[j]; // low nibble -> even-K activation
bsum += e2m1[byte >> 4] * xob[j]; // high nibble -> odd-K activation
}
acc += sc * bsum;
}
return acc;
}
#if CPU_MOE_X86
// One 32-block: 16 bytes -> 16 low + 16 high nibble values via two vpermps, times
// the pre-split even/odd fp32 activations, folded by the per-32 e8m0 scale.
CPU_MOE_TARGET("avx512f")
inline __m512 dsfp4_blk(const uint8_t* pk, const float* xeb, const float* xob, __m512 lut,
__m512i loma, float sc) {
__m512i wi = _mm512_cvtepu8_epi32(_mm_loadu_si128(reinterpret_cast<const __m128i*>(pk)));
__m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);
__m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);
__m512 prod = _mm512_fmadd_ps(vlo, _mm512_loadu_ps(xeb), _mm512_mul_ps(vhi, _mm512_loadu_ps(xob)));
return _mm512_mul_ps(prod, _mm512_set1_ps(sc));
}
CPU_MOE_TARGET("avx512f")
float dot_dsfp4_avx512(const uint8_t* packed, const uint8_t* scale, const float* xe,
const float* xo, int K, const float* e2m1, const float* e8m0) {
const __m512 lut = _mm512_loadu_ps(e2m1);
const __m512i loma = _mm512_set1_epi32(0xF);
__m512 acc0 = _mm512_setzero_ps(), acc1 = _mm512_setzero_ps();
const int nb = K / 32; // 16 packed bytes + one e8m0 scale per block
int b = 0;
for (; b + 2 <= nb; b += 2) { // two independent accumulators hide FMA latency
acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,
xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));
acc1 = _mm512_add_ps(acc1, dsfp4_blk(packed + (size_t)(b + 1) * 16, xe + (size_t)(b + 1) * 16,
xo + (size_t)(b + 1) * 16, lut, loma, e8m0[scale[b + 1]]));
}
for (; b < nb; ++b)
acc0 = _mm512_add_ps(acc0, dsfp4_blk(packed + (size_t)b * 16, xe + (size_t)b * 16,
xo + (size_t)b * 16, lut, loma, e8m0[scale[b]]));
return _mm512_reduce_add_ps(_mm512_add_ps(acc0, acc1));
}
// AVX2: a 32-K block is 16 bytes -> two 8-lane halves (8 even + 8 odd each).
CPU_MOE_TARGET("avx2,fma")
inline __m256 dsfp4_half_avx2(const uint8_t* pk, const float* xeb, const float* xob, __m256 mag8) {
__m256i wi = _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast<const __m128i*>(pk)));
__m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);
__m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);
return _mm256_fmadd_ps(vlo, _mm256_loadu_ps(xeb), _mm256_mul_ps(vhi, _mm256_loadu_ps(xob)));
}
CPU_MOE_TARGET("avx2,fma")
float dot_dsfp4_avx2(const uint8_t* packed, const uint8_t* scale, const float* xe,
const float* xo, int K, const float* e2m1, const float* e8m0) {
const __m256 mag8 = _mm256_loadu_ps(e2m1);
__m256 acc0 = _mm256_setzero_ps(), acc1 = _mm256_setzero_ps();
const int nb = K / 32;
for (int b = 0; b < nb; ++b) {
const uint8_t* pk = packed + (size_t)b * 16;
const float* xeb = xe + (size_t)b * 16;
const float* xob = xo + (size_t)b * 16;
const __m256 sc = _mm256_set1_ps(e8m0[scale[b]]);
acc0 = _mm256_fmadd_ps(dsfp4_half_avx2(pk, xeb, xob, mag8), sc, acc0);
acc1 = _mm256_fmadd_ps(dsfp4_half_avx2(pk + 8, xeb + 8, xob + 8, mag8), sc, acc1);
}
return hsum256(_mm256_add_ps(acc0, acc1));
}
#endif
dsdot_fn select_dsdot() {
const IsaTier t = pick_isa();
#if CPU_MOE_X86
if (t >= ISA_AVX512) return dot_dsfp4_avx512;
if (t >= ISA_AVX2) return dot_dsfp4_avx2;
#endif
(void)t;
return dot_dsfp4_scalar;
}
// Round a clamped |x|<=448 to nearest float8-e4m3 (RNE), back to fp32. Matches
// torch.float8_e4m3fn / triton .to(float8e4nv).
inline float e4m3_round(float x) {
const float sign = x < 0.0f ? -1.0f : 1.0f;
const float a = std::fabs(x);
if (a == 0.0f) return 0.0f;
if (a >= 448.0f) return sign * 448.0f;
int e;
std::frexp(a, &e); // a in [2^(e-1), 2^e)
float step = std::ldexp(1.0f, e - 4);
const float min_step = std::ldexp(1.0f, -9); // e4m3 subnormal step (2^-9)
if (step < min_step) step = min_step;
float r = std::nearbyint(a / step) * step;
if (r > 448.0f) r = 448.0f;
return sign * r;
}
// IEEE ceil(log2(v)) for v>0 (matches dsv4 _log2_ceil / fast_round_scale).
inline int ceil_log2_pos(float v) {
uint32_t bits;
std::memcpy(&bits, &v, sizeof(bits));
const int exp = (int)((bits >> 23) & 0xFF);
const int man = (int)(bits & 0x7FFFFF);
return exp - 127 + (man != 0 ? 1 : 0);
}
// DeepSeek-V4 / V4.1 activation FP8 round-trip (bf16 in/out): per `block` (the checkpoint's
// fp8 block: 128 on V4, 32 on V4.1), s = 2^ceil(log2(max(|x|,1e-4)/448));
// y = round_e4m3(clamp(x/s,+-448)) * s.
void fp8_roundtrip_bf16(const bf16_t* src, bf16_t* dst, int K, int block) {
for (int b0 = 0; b0 < K; b0 += block) {
const int b1 = std::min(K, b0 + block);
float amax = 1e-4f;
for (int i = b0; i < b1; ++i) amax = std::max(amax, std::fabs(bf16_to_f32(src[i])));
const float s = std::ldexp(1.0f, ceil_log2_pos(amax * (1.0f / 448.0f)));
const float inv_s = 1.0f / s;
for (int i = b0; i < b1; ++i) {
float q = bf16_to_f32(src[i]) * inv_s;
q = std::min(448.0f, std::max(-448.0f, q));
dst[i] = f32_to_bf16(e4m3_round(q) * s);
}
}
}
class DsFp4Format final : public MoeFormat {
public:
DsFp4Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_dsdot()) {
gate_up_ = a.bank("gate_up");
gu_scale_ = a.bank("gate_up_scale");
down_ = a.bank("down");
dn_scale_ = a.bank("down_scale");
act_block_ = static_cast<int>(a.option("act_block"));
input_prequant_ = a.option("input_prequant", 0) != 0;
const size_t tokens = p.max_tokens, routes = (size_t)p.max_tokens * p.top_k;
xq_.resize(tokens * p.H);
xe_.resize(tokens * (p.H / 2));
xo_.resize(tokens * (p.H / 2));
ge_.resize(routes * (p.I / 2));
go_.resize(routes * (p.I / 2));
for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];
for (int i = 0; i < 256; ++i) e8m0_lut_[i] = std::ldexp(1.0f, std::min(i, 254) - 127);
}
std::string isa_name() const override { return tier_name(pick_isa()); }
// DSV4 FP8-round-trips the input before the gate_up GEMV, unless the GPU already did.
void prep_input(int tok, const bf16_t* x) override {
const bf16_t* src = x;
if (!input_prequant_) {
bf16_t* xq = xq_.data() + (size_t)tok * p.H;
fp8_roundtrip_bf16(x, xq, p.H, act_block_);
src = xq;
}
deinterleave_bf16_f32(src, xe_.data() + (size_t)tok * (p.H / 2), xo_.data() + (size_t)tok * (p.H / 2), p.H);
}
bool preps_inter() const override { return true; }
void prep_inter(int64_t r, bf16_t* g) override {
fp8_roundtrip_bf16(g, g, p.I, act_block_);
deinterleave_bf16_f32(g, ge_.data() + (size_t)r * (p.I / 2), go_.data() + (size_t)r * (p.I / 2), p.I);
}
void gate_up(int layer, int e, int tok, int i0, int i1, const bf16_t* /*x*/, float /*w_in*/,
bf16_t* g) const override {
const int H = p.H, I = p.I;
const uint8_t* gu_packed_l = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));
const uint8_t* gu_scale_l = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));
const int N2 = 2 * I, Hh = H / 2, Hs = H / 32;
const float* xe = xe_.data() + (size_t)tok * (H / 2);
const float* xo = xo_.data() + (size_t)tok * (H / 2);
const uint8_t* gp = gu_packed_l + (size_t)e * N2 * Hh;
const uint8_t* gs = gu_scale_l + (size_t)e * N2 * Hs;
const float lim = p.swiglu_limit;
for (int i = i0; i < i1; ++i) {
// gate_up is stored bf16 by the reference GEMV before swiglu; round to match.
float gate = bf16_to_f32(f32_to_bf16(
dot_(gp + (size_t)i * Hh, gs + (size_t)i * Hs, xe, xo, H, e2m1_lut_, e8m0_lut_)));
float up = bf16_to_f32(f32_to_bf16(dot_(
gp + (size_t)(I + i) * Hh, gs + (size_t)(I + i) * Hs, xe, xo, H, e2m1_lut_, e8m0_lut_)));
if (lim > 0.0f) {
if (gate > lim) gate = lim;
if (up > lim) up = lim;
else if (up < -lim) up = -lim;
}
const float glu = gate / (1.0f + std::exp(-gate)); // silu(gate)
g[i] = f32_to_bf16(glu * up);
}
}
void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {
const int H = p.H, I = p.I, Ih = I / 2, Is = I / 32;
const uint8_t* dn_packed_l = static_cast<const uint8_t*>(tbl_at(down_, layer));
const uint8_t* dn_scale_l = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));
for (int h = h0; h < h1; ++h) {
for (int j = 0; j < n; ++j) {
const size_t e = (size_t)routes[j].e, r = (size_t)routes[j].r;
const uint8_t* dp = dn_packed_l + e * (size_t)H * Ih + (size_t)h * Ih;
const uint8_t* ds = dn_scale_l + e * (size_t)H * Is + (size_t)h * Is;
out[j * kRowBlock + h - h0] = dot_(dp, ds, ge_.data() + r * Ih, go_.data() + r * Ih, I, e2m1_lut_, e8m0_lut_);
}
}
}
// The reference rounds each route's weighted down output to bf16 before the fp32 sum over routes.
bool rounds_routes() const override { return true; }
private:
dsdot_fn dot_;
// e2m1 rows [E,2I,H/2] with e8m0 scales per 32 along K [E,2I,H/32]; down likewise with K = I.
const uint64_t* gate_up_ = nullptr;
const uint64_t* gu_scale_ = nullptr;
const uint64_t* down_ = nullptr;
const uint64_t* dn_scale_ = nullptr;
// Activation FP8 round-trip block, the checkpoint's fp8 block.
int act_block_ = 128;
// The caller already FP8-round-tripped the input activations on the GPU
// (same reference grid), so prep_input must not repeat it on the host-callback
// thread. That scalar per-element pass is single-threaded ON THE DECODE CRITICAL
// PATH (~0.3ms/layer at H=4096, every worker and the GPU waiting on it); moving
// it to a captured GPU elementwise kernel removes it while keeping the official
// W4A8 numerics bit-exact (see cpu_executor.py).
bool input_prequant_ = false;
float e2m1_lut_[16];
float e8m0_lut_[256];
AlignedVector<bf16_t> xq_;
AlignedVector<float> xe_, xo_, ge_, go_;
};
} // namespace
std::unique_ptr<MoeFormat> make_ds_fp4_format(const MoeParams& p, const FormatArgs& a) {
return std::make_unique<DsFp4Format>(p, a);
}
} // namespace freetoken::cpu_moe