mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor(cpu-moe): split the executor into weight formats and gpu sync backends
- move each expert weight format into kernels/<fmt>.cpp behind the MoeFormat
interface in format.h; the executor keeps the threads, the work split and
the pass 2 route sum
- pass each format its banks and integer options by name; MoeParams keeps only
the shared shape and activation fields
- key the formats by name in one table that weight_formats() exports, replacing
the integer format ids and the fp8_block_isa_name hook
- replace the per-(layer, batch size) flag slots, pinned IO buffers and task
table with one doorbell word MoeTask{layer, num_tokens} and one shared pinned
IO block of max_tokens rows
- move the GPU/CPU handshake behind GpuSync with a CUDA and a HIP backend picked
by setup.py; HIP waits on signal memory and adds explicit batch-memop graph
nodes where the runtime does not record the native memops under capture
- raise on graph capture when the handshake is off and host functions are not
replay-safe
- allocate format scratch on 64-byte boundaries
- size max_tokens by the explicit cuda_graph_bs as well
Co-authored-by: zihaomu <zihaomu@outlook.com>
* perf(cpu-moe): skip the worker pool for tasks without routed experts
* perf(cpu-moe): publish and combine the cpu handshake with kernels
* fix(cpu-moe): clean up failed hip capability probes
---------
Co-authored-by: zihaomu <zihaomu@outlook.com>
248 lines
11 KiB
C++
248 lines
11 KiB
C++
#include "../format.h"
|
|
|
|
namespace freetoken::cpu_moe {
|
|
namespace {
|
|
|
|
// ------------------------- mxfp4 (gpt-oss) GEMV -----------------------------
|
|
// Transposed split-K layout: blk[Kpairs, N2] (N innermost), scl[Kpairs/16, N2]
|
|
// e8m0 per 32-K. Computes out[c] = sum_kb (E2M1[lo]*x[2kb] + E2M1[hi]*x[2kb+1])
|
|
// * 2^(e8m0-127) for a contiguous column tile (blk/scl already offset to col 0 of
|
|
// the tile). Vectorized over N (16 columns / __m512), K stays the outer (cache-
|
|
// sequential) loop. Used by both gate_up (K=H) and down (K=I).
|
|
using mxgemv_fn = void (*)(float*, const uint8_t*, const uint8_t*, const bf16_t*, int, int,
|
|
int, const float*, const float*);
|
|
|
|
void mxfp4_gemv_scalar(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,
|
|
int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {
|
|
for (int c = 0; c < ncol; ++c) out[c] = 0.0f;
|
|
for (int kb = 0; kb < Kpairs; ++kb) {
|
|
const uint8_t* w = blk + (size_t)kb * N2;
|
|
const uint8_t* s = scl + (size_t)(kb >> 4) * N2;
|
|
const float xl = bf16_to_f32(x[2 * kb]);
|
|
const float xh = bf16_to_f32(x[2 * kb + 1]);
|
|
for (int c = 0; c < ncol; ++c) {
|
|
const uint8_t byte = w[c];
|
|
out[c] += (e2m1[byte & 0xF] * xl + e2m1[byte >> 4] * xh) * e8m0[s[c]];
|
|
}
|
|
}
|
|
}
|
|
|
|
#if CPU_MOE_X86
|
|
CPU_MOE_TARGET("avx512f")
|
|
void mxfp4_gemv_avx512(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,
|
|
int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {
|
|
(void)e8m0; // e8m0[c]=2^(c-127) computed via bit construction (no gather)
|
|
const __m512 lut = _mm512_loadu_ps(e2m1);
|
|
const __m512i loma = _mm512_set1_epi32(0xF);
|
|
// K-outer / N-inner: each kb cache line is read once and all live column chunks
|
|
// (up to 4 -> 64 cols) accumulate from registers, so DRAM/L2 stream the tile once.
|
|
int c0 = 0;
|
|
for (; c0 + 16 <= ncol; c0 += 64) {
|
|
const int nchunk = std::min(4, (ncol - c0) / 16);
|
|
__m512 acc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_setzero_ps();
|
|
for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = 32 K = one scale row
|
|
__m512 sc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) {
|
|
__m128i sraw = _mm_loadu_si128(reinterpret_cast<const __m128i*>(
|
|
scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 16));
|
|
sc[ci] = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu8_epi32(sraw), 23));
|
|
}
|
|
__m512 blk_acc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm512_setzero_ps();
|
|
for (int kk = 0; kk < 16; ++kk) {
|
|
const int kb = kblk + kk;
|
|
const uint8_t* wbase = blk + (size_t)kb * N2 + c0;
|
|
// The transposed layout strides K by N2 bytes; prefetch ahead so the strided
|
|
// reads are not exposed to DRAM latency (the HW streamer misses big strides).
|
|
constexpr int PFD = 8;
|
|
if (kb + PFD < Kpairs)
|
|
_mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),
|
|
_MM_HINT_T0);
|
|
const __m512 xl = _mm512_set1_ps(bf16_to_f32(x[2 * kb]));
|
|
const __m512 xh = _mm512_set1_ps(bf16_to_f32(x[2 * kb + 1]));
|
|
for (int ci = 0; ci < nchunk; ++ci) {
|
|
__m512i wi = _mm512_cvtepu8_epi32(
|
|
_mm_loadu_si128(reinterpret_cast<const __m128i*>(wbase + ci * 16)));
|
|
__m512 vlo = _mm512_permutexvar_ps(_mm512_and_si512(wi, loma), lut);
|
|
__m512 vhi = _mm512_permutexvar_ps(_mm512_and_si512(_mm512_srli_epi32(wi, 4), loma), lut);
|
|
blk_acc[ci] = _mm512_fmadd_ps(vlo, xl, blk_acc[ci]);
|
|
blk_acc[ci] = _mm512_fmadd_ps(vhi, xh, blk_acc[ci]);
|
|
}
|
|
}
|
|
for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm512_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);
|
|
}
|
|
for (int ci = 0; ci < nchunk; ++ci) _mm512_storeu_ps(out + c0 + ci * 16, acc[ci]);
|
|
}
|
|
for (int c = c0; c < ncol; ++c) { // tail columns (< 16)
|
|
float o = 0.0f;
|
|
for (int kb = 0; kb < Kpairs; ++kb) {
|
|
const uint8_t byte = blk[(size_t)kb * N2 + c];
|
|
uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;
|
|
float sc;
|
|
std::memcpy(&sc, &bits, 4);
|
|
o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) +
|
|
e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;
|
|
}
|
|
out[c] = o;
|
|
}
|
|
}
|
|
|
|
CPU_MOE_TARGET("avx2,fma")
|
|
void mxfp4_gemv_avx2(float* out, const uint8_t* blk, const uint8_t* scl, const bf16_t* x,
|
|
int Kpairs, int N2, int ncol, const float* e2m1, const float* e8m0) {
|
|
(void)e8m0; // e8m0[s]=2^(s-127) built via s<<23 (no gather)
|
|
const __m256 mag8 = _mm256_loadu_ps(e2m1);
|
|
int c0 = 0;
|
|
for (; c0 + 8 <= ncol; c0 += 32) { // up to 4 chunks of 8 = 32 cols
|
|
const int nchunk = std::min(4, (ncol - c0) / 8);
|
|
__m256 acc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_setzero_ps();
|
|
for (int kblk = 0; kblk < Kpairs; kblk += 16) { // 16 K-pairs = one scale row
|
|
__m256 sc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) {
|
|
__m128i sraw = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(
|
|
scl + (size_t)(kblk >> 4) * N2 + c0 + ci * 8));
|
|
sc[ci] = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu8_epi32(sraw), 23));
|
|
}
|
|
__m256 blk_acc[4];
|
|
for (int ci = 0; ci < nchunk; ++ci) blk_acc[ci] = _mm256_setzero_ps();
|
|
for (int kk = 0; kk < 16; ++kk) {
|
|
const int kb = kblk + kk;
|
|
const uint8_t* wbase = blk + (size_t)kb * N2 + c0;
|
|
constexpr int PFD = 8;
|
|
if (kb + PFD < Kpairs)
|
|
_mm_prefetch(reinterpret_cast<const char*>(blk + (size_t)(kb + PFD) * N2 + c0),
|
|
_MM_HINT_T0);
|
|
const __m256 xl = _mm256_set1_ps(bf16_to_f32(x[2 * kb]));
|
|
const __m256 xh = _mm256_set1_ps(bf16_to_f32(x[2 * kb + 1]));
|
|
for (int ci = 0; ci < nchunk; ++ci) {
|
|
__m256i wi = _mm256_cvtepu8_epi32(
|
|
_mm_loadl_epi64(reinterpret_cast<const __m128i*>(wbase + ci * 8)));
|
|
__m256 vlo = e2m1_decode8(_mm256_and_si256(wi, _mm256_set1_epi32(0xF)), mag8);
|
|
__m256 vhi = e2m1_decode8(_mm256_srli_epi32(wi, 4), mag8);
|
|
blk_acc[ci] = _mm256_fmadd_ps(vlo, xl, blk_acc[ci]);
|
|
blk_acc[ci] = _mm256_fmadd_ps(vhi, xh, blk_acc[ci]);
|
|
}
|
|
}
|
|
for (int ci = 0; ci < nchunk; ++ci) acc[ci] = _mm256_fmadd_ps(blk_acc[ci], sc[ci], acc[ci]);
|
|
}
|
|
for (int ci = 0; ci < nchunk; ++ci) _mm256_storeu_ps(out + c0 + ci * 8, acc[ci]);
|
|
}
|
|
for (int c = c0; c < ncol; ++c) { // tail columns (< 8); none when ncol%8==0
|
|
float o = 0.0f;
|
|
for (int kb = 0; kb < Kpairs; ++kb) {
|
|
const uint8_t byte = blk[(size_t)kb * N2 + c];
|
|
uint32_t bits = (uint32_t)scl[(size_t)(kb >> 4) * N2 + c] << 23;
|
|
float sc;
|
|
std::memcpy(&sc, &bits, 4);
|
|
o += (e2m1[byte & 0xF] * bf16_to_f32(x[2 * kb]) +
|
|
e2m1[byte >> 4] * bf16_to_f32(x[2 * kb + 1])) * sc;
|
|
}
|
|
out[c] = o;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
mxgemv_fn select_mxgemv() {
|
|
const IsaTier t = pick_isa();
|
|
#if CPU_MOE_X86
|
|
if (t >= ISA_AVX512) return mxfp4_gemv_avx512;
|
|
if (t >= ISA_AVX2) return mxfp4_gemv_avx2;
|
|
#endif
|
|
(void)t;
|
|
return mxfp4_gemv_scalar;
|
|
}
|
|
|
|
// ----------------------------- mxfp4 (gpt-oss) -----------------------------
|
|
// Transposed split-K layout (N innermost), so the GEMV streams K and accumulates
|
|
// a contiguous block of N output columns -> cache-line-efficient, no repack, no
|
|
// extra host memory. Pass 1 fuses gate_up + clamped-swiglu(+bias); pass 2 adds the
|
|
// down bias, and the executor applies the router weight and sums the routes.
|
|
//
|
|
// Dequant: w = E2M1[code] * 2^(e8m0_scale - 127); two codes per byte (low nibble
|
|
// first), one e8m0 scale per 32 contiguous K. Matches kernel/triton/mxfp4_moe.py.
|
|
class Mxfp4Format final : public MoeFormat {
|
|
public:
|
|
Mxfp4Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), gemv_(select_mxgemv()) {
|
|
gate_up_ = a.bank("gate_up");
|
|
gu_scale_ = a.bank("gate_up_scale");
|
|
gu_bias_ = a.bank("gate_up_bias");
|
|
down_ = a.bank("down");
|
|
dn_scale_ = a.bank("down_scale");
|
|
dn_bias_ = a.bank("down_bias");
|
|
for (int i = 0; i < 16; ++i) e2m1_lut_[i] = kE2M1[i];
|
|
// e8m0 (mxfp4 block scale) = 2^(s-127); the GPU GEMV clamps s to [0,254].
|
|
for (int i = 0; i < 256; ++i) e8m0_lut_[i] = std::ldexp(1.0f, std::min(i, 254) - 127);
|
|
}
|
|
|
|
std::string isa_name() const override { return tier_name(pick_isa()); }
|
|
|
|
void gate_up(int layer, int e, int /*tok*/, int i0, int i1, const bf16_t* x, float /*w_in*/,
|
|
bf16_t* g) const override {
|
|
const int H = p.H, I = p.I;
|
|
const uint8_t* gu_packed_l = static_cast<const uint8_t*>(tbl_at(gate_up_, layer));
|
|
const uint8_t* gu_scale_l = static_cast<const uint8_t*>(tbl_at(gu_scale_, layer));
|
|
const bf16_t* gu_bias_l = static_cast<const bf16_t*>(tbl_at(gu_bias_, layer));
|
|
const int N2 = 2 * I; // gate_up output width (gate/up interleaved)
|
|
const int Hh = H / 2; // packed-K rows
|
|
const int nunit = i1 - i0; // intermediate units owned by this tile
|
|
const int col0 = 2 * i0; // first gate_up column
|
|
const int ncol = 2 * nunit; // gate_up columns owned by this tile
|
|
const uint8_t* blk_e = gu_packed_l + (size_t)e * Hh * N2;
|
|
const uint8_t* scl_e = gu_scale_l + (size_t)e * (size_t)(H / 32) * N2;
|
|
float gu[2 * kRowBlock];
|
|
gemv_(gu, blk_e + col0, scl_e + col0, x, Hh, N2, ncol, e2m1_lut_, e8m0_lut_);
|
|
const bf16_t* bias_e = gu_bias_l + (size_t)e * N2 + col0;
|
|
const float lim = p.swiglu_limit, alpha = p.swiglu_alpha;
|
|
for (int j = 0; j < nunit; ++j) {
|
|
float gate = gu[2 * j] + bf16_to_f32(bias_e[2 * j]);
|
|
float up = gu[2 * j + 1] + bf16_to_f32(bias_e[2 * j + 1]);
|
|
if (gate > lim) gate = lim;
|
|
if (up > lim) up = lim;
|
|
else if (up < -lim) up = -lim;
|
|
const float glu = gate / (1.0f + std::exp(-gate * alpha)); // gate * sigmoid(alpha*gate)
|
|
g[i0 + j] = f32_to_bf16(glu * (up + 1.0f));
|
|
}
|
|
}
|
|
|
|
// The transposed layout already streams a whole column block per call, so the routes go one after another.
|
|
void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {
|
|
const int H = p.H, I = p.I;
|
|
const int Ih = I / 2; // packed-K rows
|
|
const int nh = h1 - h0;
|
|
const uint8_t* dn_packed_l = static_cast<const uint8_t*>(tbl_at(down_, layer));
|
|
const uint8_t* dn_scale_l = static_cast<const uint8_t*>(tbl_at(dn_scale_, layer));
|
|
const bf16_t* dn_bias_l = static_cast<const bf16_t*>(tbl_at(dn_bias_, layer));
|
|
for (int j = 0; j < n; ++j) {
|
|
const size_t e = (size_t)routes[j].e;
|
|
const uint8_t* blk_e = dn_packed_l + e * Ih * H;
|
|
const uint8_t* scl_e = dn_scale_l + e * (size_t)(I / 32) * H;
|
|
float* part = out + j * kRowBlock;
|
|
gemv_(part, blk_e + h0, scl_e + h0, routes[j].g, Ih, H, nh, e2m1_lut_, e8m0_lut_);
|
|
const bf16_t* bias_e = dn_bias_l + e * H + h0;
|
|
for (int c = 0; c < nh; ++c) part[c] += bf16_to_f32(bias_e[c]);
|
|
}
|
|
}
|
|
|
|
private:
|
|
mxgemv_fn gemv_;
|
|
// gpt-oss split-K layout, N innermost: blocks [E,H/2,2I], e8m0 scales [E,H/32,2I], bf16 bias [E,2I]; down likewise with K = I, N = H.
|
|
const uint64_t* gate_up_ = nullptr;
|
|
const uint64_t* gu_scale_ = nullptr;
|
|
const uint64_t* gu_bias_ = nullptr;
|
|
const uint64_t* down_ = nullptr;
|
|
const uint64_t* dn_scale_ = nullptr;
|
|
const uint64_t* dn_bias_ = nullptr;
|
|
float e2m1_lut_[16];
|
|
float e8m0_lut_[256];
|
|
};
|
|
|
|
} // namespace
|
|
|
|
std::unique_ptr<MoeFormat> make_mxfp4_format(const MoeParams& p, const FormatArgs& a) {
|
|
return std::make_unique<Mxfp4Format>(p, a);
|
|
}
|
|
|
|
} // namespace freetoken::cpu_moe
|