#include "../format.h" namespace freetoken::cpu_moe { namespace { // ------------------------------- dot products ------------------------------- // dot(weight[bf16], act[bf16], n) -> fp32. The selected impl is a function // pointer chosen at runtime; the per-row call overhead is negligible vs n. using dot_fn = float (*)(const bf16_t*, const bf16_t*, int); float dot_scalar(const bf16_t* w, const bf16_t* x, int n) { float acc = 0.0f; for (int i = 0; i < n; ++i) acc += bf16_to_f32(w[i]) * bf16_to_f32(x[i]); return acc; } #if CPU_MOE_X86 CPU_MOE_TARGET("avx512f") float dot_avx512f(const bf16_t* w, const bf16_t* x, int n) { // 4 independent accumulators -> more in-flight loads (memory-level parallelism), // which is what lifts a bandwidth-bound GEMV toward peak. __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); int i = 0; for (; i + 64 <= n; i += 64) { _mm_prefetch(reinterpret_cast(w + i) + PF_AHEAD, _MM_HINT_T0); for (int j = 0; j < 64; j += 16) { __m256i wi = _mm256_loadu_si256(reinterpret_cast(w + i + j)); __m256i xi = _mm256_loadu_si256(reinterpret_cast(x + i + j)); __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16)); __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16)); __m512& acc = (j == 0) ? a0 : (j == 16) ? a1 : (j == 32) ? a2 : a3; acc = _mm512_fmadd_ps(wf, xf, acc); } } for (; i + 16 <= n; i += 16) { __m256i wi = _mm256_loadu_si256(reinterpret_cast(w + i)); __m256i xi = _mm256_loadu_si256(reinterpret_cast(x + i)); __m512 wf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(wi), 16)); __m512 xf = _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(xi), 16)); a0 = _mm512_fmadd_ps(wf, xf, a0); } float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3))); for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]); return s; } #ifdef CPU_MOE_HAS_AVX512BF16 CPU_MOE_TARGET("avx512bf16,avx512f") static inline __m512bh load_bh(const bf16_t* p) { __m512i raw = _mm512_loadu_si512(reinterpret_cast(p)); __m512bh out; std::memcpy(&out, &raw, sizeof(out)); return out; } CPU_MOE_TARGET("avx512bf16,avx512f") float dot_avx512bf16(const bf16_t* w, const bf16_t* x, int n) { // 4 accumulators (128 bf16/iter) for memory-level parallelism + a prefetch nudge. __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps(); __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps(); int i = 0; for (; i + 128 <= n; i += 128) { _mm_prefetch(reinterpret_cast(w + i) + PF_AHEAD, _MM_HINT_T0); a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i)); a1 = _mm512_dpbf16_ps(a1, load_bh(w + i + 32), load_bh(x + i + 32)); a2 = _mm512_dpbf16_ps(a2, load_bh(w + i + 64), load_bh(x + i + 64)); a3 = _mm512_dpbf16_ps(a3, load_bh(w + i + 96), load_bh(x + i + 96)); } for (; i + 32 <= n; i += 32) { a0 = _mm512_dpbf16_ps(a0, load_bh(w + i), load_bh(x + i)); } float s = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(a0, a1), _mm512_add_ps(a2, a3))); for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]); return s; } #endif // avx512bf16 available // --------- AVX2 + FMA (256-bit fallback for CPUs without AVX-512) ---------- // Covers Intel 12-14th gen / Arrow Lake (AVX-512 fused off) and AMD Zen<4. The // bf16->fp32 widen is a zero-extend + <<16; with 4 independent accumulators the // GEMV is memory-bandwidth bound, same as the AVX-512 path (just half the width). CPU_MOE_TARGET("avx2,fma") float dot_avx2(const bf16_t* w, const bf16_t* x, int n) { __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps(); __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps(); int i = 0; for (; i + 32 <= n; i += 32) { _mm_prefetch(reinterpret_cast(w + i) + PF_AHEAD, _MM_HINT_T0); for (int j = 0; j < 32; j += 8) { __m128i wi = _mm_loadu_si128(reinterpret_cast(w + i + j)); __m128i xi = _mm_loadu_si128(reinterpret_cast(x + i + j)); __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16)); __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16)); __m256& acc = (j == 0) ? a0 : (j == 8) ? a1 : (j == 16) ? a2 : a3; acc = _mm256_fmadd_ps(wf, xf, acc); } } for (; i + 8 <= n; i += 8) { __m128i wi = _mm_loadu_si128(reinterpret_cast(w + i)); __m128i xi = _mm_loadu_si128(reinterpret_cast(x + i)); __m256 wf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(wi), 16)); __m256 xf = _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(xi), 16)); a0 = _mm256_fmadd_ps(wf, xf, a0); } float s = hsum256(_mm256_add_ps(_mm256_add_ps(a0, a1), _mm256_add_ps(a2, a3))); for (; i < n; ++i) s += bf16_to_f32(w[i]) * bf16_to_f32(x[i]); return s; } #endif // CPU_MOE_X86 dot_fn select_dot() { const IsaTier t = pick_isa(); #if CPU_MOE_X86 #ifdef CPU_MOE_HAS_AVX512BF16 if (t >= ISA_AVX512BF16) return dot_avx512bf16; #endif if (t >= ISA_AVX512) return dot_avx512f; if (t >= ISA_AVX2) return dot_avx2; #endif (void)t; return dot_scalar; } class Bf16Format final : public MoeFormat { public: Bf16Format(const MoeParams& p, const FormatArgs& a) : MoeFormat(p), dot_(select_dot()), gate_up_(a.bank("gate_up")), down_(a.bank("down")) {} std::string isa_name() const override { return tier_name(pick_isa()); } void gate_up(int layer, int e, int /*tok*/, int i0, int i1, const bf16_t* x, float w_in, bf16_t* g) const override { const bf16_t* w = static_cast(tbl_at(gate_up_, layer)) + (size_t)e * (2 * p.I) * p.H; gated_rows(p, i0, i1, w_in, g, [&](int row) { return dot_(w + (size_t)row * p.H, x, p.H); }); } void down(int layer, int h0, int h1, const DownRoute* routes, int n, float* out) const override { const bf16_t* w = static_cast(tbl_at(down_, layer)); for (int h = h0; h < h1; ++h) for (int j = 0; j < n; ++j) out[j * kRowBlock + h - h0] = dot_(w + ((size_t)routes[j].e * p.H + h) * p.I, routes[j].g, p.I); } private: dot_fn dot_; // bf16 rows: gate_up [E, 2I, H], down [E, H, I]. const uint64_t* gate_up_; const uint64_t* down_; }; } // namespace std::unique_ptr make_bf16_format(const MoeParams& p, const FormatArgs& a) { return std::make_unique(p, a); } } // namespace freetoken::cpu_moe