mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* refactor(cpu-moe): split the executor into weight formats and gpu sync backends
- move each expert weight format into kernels/<fmt>.cpp behind the MoeFormat
interface in format.h; the executor keeps the threads, the work split and
the pass 2 route sum
- pass each format its banks and integer options by name; MoeParams keeps only
the shared shape and activation fields
- key the formats by name in one table that weight_formats() exports, replacing
the integer format ids and the fp8_block_isa_name hook
- replace the per-(layer, batch size) flag slots, pinned IO buffers and task
table with one doorbell word MoeTask{layer, num_tokens} and one shared pinned
IO block of max_tokens rows
- move the GPU/CPU handshake behind GpuSync with a CUDA and a HIP backend picked
by setup.py; HIP waits on signal memory and adds explicit batch-memop graph
nodes where the runtime does not record the native memops under capture
- raise on graph capture when the handshake is off and host functions are not
replay-safe
- allocate format scratch on 64-byte boundaries
- size max_tokens by the explicit cuda_graph_bs as well
Co-authored-by: zihaomu <zihaomu@outlook.com>
* perf(cpu-moe): skip the worker pool for tasks without routed experts
* perf(cpu-moe): publish and combine the cpu handshake with kernels
* fix(cpu-moe): clean up failed hip capability probes
---------
Co-authored-by: zihaomu <zihaomu@outlook.com>
160 lines
6.6 KiB
C++
160 lines
6.6 KiB
C++
#include "../format.h"
|
|
|
|
namespace freetoken::cpu_moe {
|
|
namespace {
|
|
|
|
// ------------------------------- dot products -------------------------------
|
|
// dot(weight[bf16], act[bf16], n) -> fp32. The selected impl is a function
|
|
// pointer chosen at runtime; the per-row call overhead is negligible vs n.
|
|
|
|
using dot_fn = float (*)(const bf16_t*, const bf16_t*, int);
|
|
|
|
float dot_scalar(const bf16_t* w, const bf16_t* x, int n) {
|
|
float acc = 0.0f;
|
|
for (int i = 0; i < n; ++i) acc += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);
|
|
return acc;
|
|
}
|
|
|
|
#if CPU_MOE_X86
|
|
CPU_MOE_TARGET("avx512f")
|
|
float dot_avx512f(const bf16_t* w, const bf16_t* x, int n) {
|
|
// 4 independent accumulators -> more in-flight loads (memory-level parallelism),
|
|
// which is what lifts a bandwidth-bound GEMV toward peak.
|
|
__m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
|
|
__m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
|
|
int i = 0;
|
|
for (; i + 64 <= n; i += 64) {
|
|
_mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);
|
|
for (int j = 0; j < 64; j += 16) {
|
|
__m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i + j));
|
|
__m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i + j));
|
|
__m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));
|
|
__m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));
|
|
__m512& acc = (j == 0) ? a0 : (j == 16) ? a1 : (j == 32) ? a2 : a3;
|
|
acc = _mm512_fmadd_ps(wf, xf, acc);
|
|
}
|
|
}
|
|
for (; i + 16 <= n; i += 16) {
|
|
__m256i wi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(w + i));
|
|
__m256i xi = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(x + i));
|
|
__m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16));
|
|
__m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16));
|
|
a0 = _mm512_fmadd_ps(wf, xf, a0);
|
|
}
|
|
float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));
|
|
for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);
|
|
return s;
|
|
}
|
|
|
|
#ifdef CPU_MOE_HAS_AVX512BF16
|
|
CPU_MOE_TARGET("avx512bf16,avx512f")
|
|
static inline __m512bh load_bh(const bf16_t* p) {
|
|
__m512i raw = _mm512_loadu_si512(reinterpret_cast<const void*>(p));
|
|
__m512bh out;
|
|
std::memcpy(&out, &raw, sizeof(out));
|
|
return out;
|
|
}
|
|
|
|
CPU_MOE_TARGET("avx512bf16,avx512f")
|
|
float dot_avx512bf16(const bf16_t* w, const bf16_t* x, int n) {
|
|
// 4 accumulators (128 bf16/iter) for memory-level parallelism + a prefetch nudge.
|
|
__m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
|
|
__m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
|
|
int i = 0;
|
|
for (; i + 128 <= n; i += 128) {
|
|
_mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);
|
|
a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));
|
|
a1 = _mm512_dpbf16_ps(a1, load_bh(w + i + 32), load_bh(x + i + 32));
|
|
a2 = _mm512_dpbf16_ps(a2, load_bh(w + i + 64), load_bh(x + i + 64));
|
|
a3 = _mm512_dpbf16_ps(a3, load_bh(w + i + 96), load_bh(x + i + 96));
|
|
}
|
|
for (; i + 32 <= n; i += 32) {
|
|
a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i));
|
|
}
|
|
float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3)));
|
|
for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);
|
|
return s;
|
|
}
|
|
#endif // avx512bf16 available
|
|
|
|
// --------- AVX2 + FMA (256-bit fallback for CPUs without AVX-512) ----------
|
|
// Covers Intel 12-14th gen / Arrow Lake (AVX-512 fused off) and AMD Zen<4. The
|
|
// bf16->fp32 widen is a zero-extend + <<16; with 4 independent accumulators the
|
|
// GEMV is memory-bandwidth bound, same as the AVX-512 path (just half the width).
|
|
CPU_MOE_TARGET("avx2,fma")
|
|
float dot_avx2(const bf16_t* w, const bf16_t* x, int n) {
|
|
__m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
|
|
__m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
|
|
int i = 0;
|
|
for (; i + 32 <= n; i += 32) {
|
|
_mm_prefetch(reinterpret_cast<const char*>(w + i) + PF_AHEAD, _MM_HINT_T0);
|
|
for (int j = 0; j < 32; j += 8) {
|
|
__m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i + j));
|
|
__m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i + j));
|
|
__m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));
|
|
__m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));
|
|
__m256& acc = (j == 0) ? a0 : (j == 8) ? a1 : (j == 16) ? a2 : a3;
|
|
acc = _mm256_fmadd_ps(wf, xf, acc);
|
|
}
|
|
}
|
|
for (; i + 8 <= n; i += 8) {
|
|
__m128i wi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(w + i));
|
|
__m128i xi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(x + i));
|
|
__m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16));
|
|
__m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16));
|
|
a0 = _mm256_fmadd_ps(wf, xf, a0);
|
|
}
|
|
float s = hsum256(_mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3)));
|
|
for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]);
|
|
return s;
|
|
}
|
|
#endif // CPU_MOE_X86
|
|
|
|
dot_fn select_dot() {
|
|
const IsaTier t = pick_isa();
|
|
#if CPU_MOE_X86
|
|
#ifdef CPU_MOE_HAS_AVX512BF16
|
|
if (t >= ISA_AVX512BF16) return dot_avx512bf16;
|
|
#endif
|
|
if (t >= ISA_AVX512) return dot_avx512f;
|
|
if (t >= ISA_AVX2) return dot_avx2;
|
|
#endif
|
|
(void)t;
|
|
return dot_scalar;
|
|
}
|
|
|
|
class Bf16Format final : public MoeFormat {
|
|
public:
|
|
Bf16Format(const MoeParams& p, const FormatArgs& a)
|
|
: MoeFormat(p), dot_(select_dot()), gate_up_(a.bank("gate_up")), down_(a.bank("down")) {}
|
|
|
|
std::string isa_name() const override { return tier_name(pick_isa()); }
|
|
|
|
void gate_up(int layer, int e, int /*tok*/, int i0, int i1, const bf16_t* x, float w_in,
|
|
bf16_t* g) const override {
|
|
const bf16_t* w = static_cast<const bf16_t*>(tbl_at(gate_up_, layer)) + (size_t)e * (2 * p.I) * p.H;
|
|
gated_rows(p, i0, i1, w_in, g, [&](int row) { return dot_(w + (size_t)row * p.H, x, p.H); });
|
|
}
|
|
|
|
void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override {
|
|
const bf16_t* w = static_cast<const bf16_t*>(tbl_at(down_, layer));
|
|
for (int h = h0; h < h1; ++h)
|
|
for (int j = 0; j < n; ++j)
|
|
out[j * kRowBlock + h - h0] = dot_(w + ((size_t)routes[j].e * p.H + h) * p.I, routes[j].g, p.I);
|
|
}
|
|
|
|
private:
|
|
dot_fn dot_;
|
|
// bf16 rows: gate_up [E, 2I, H], down [E, H, I].
|
|
const uint64_t* gate_up_;
|
|
const uint64_t* down_;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
std::unique_ptr<MoeFormat> make_bf16_format(const MoeParams& p, const FormatArgs& a) {
|
|
return std::make_unique<Bf16Format>(p, a);
|
|
}
|
|
|
|
} // namespace freetoken::cpu_moe
|