// CPU-compute MoE executor for the cpu and hybrid MoE strategies, and its Python binding. // // Decode ships activations to the CPU, computes the routed experts here (reading // the pinned host expert banks at full RAM bandwidth), and ships the results back. // One task (one MoE layer) is in flight at a time: submit() wakes the persistent worker // pool, the pool runs pass 1 (gate_up) and pass 2 (down), and sync() waits for it. // The GPU rings a doorbell to submit and waits on it to sync (GpuSync, gpu_sync.h), so the // whole decode step stays in one CUDA graph; cudaLaunchHostFunc nodes are the fallback. // // Weight formats live in kernels/, one file each behind the MoeFormat interface (format.h). // Compute is FP32-accumulate; the intermediate is stored bf16 to match the GPU decode path. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "format.h" #include "gpu_sync.h" #include "gpu_sync_io.h" #include "threads.h" namespace freetoken::cpu_moe { std::unique_ptr make_bf16_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_nvfp4_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_mxfp4_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_ds_fp4_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_fp8_block_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_gguf_format(const MoeParams& p, const FormatArgs& a); std::unique_ptr make_gguf_scaled_format(const MoeParams& p, const FormatArgs& a); void set_ggml_cpu(uintptr_t addr); void bind_locked_fetch(pybind11::module_& m); namespace { constexpr int kMaxTopK = 64; // One MoE layer on the first num_tokens rows of the pinned IO. // It is also the doorbell word the GPU writes, a constant baked into each captured graph; num_tokens >= 1 keeps it nonzero, since 0 means no doorbell. struct MoeTask { uint64_t layer : 32; uint64_t num_tokens : 32; }; static_assert(sizeof(MoeTask) == sizeof(uint64_t), "MoeTask must pack into one 64-bit word"); inline uint64_t to_word(MoeTask t) { uint64_t w; std::memcpy(&w, &t, sizeof(w)); return w; } inline MoeTask from_word(uint64_t w) { MoeTask t; std::memcpy(&t, &w, sizeof(t)); return t; } // Every weight format this module serves, keyed by the name cpu_executor.py passes; weight_formats() exports the names. const std::pair (*)(const MoeParams&, const FormatArgs&)> kFormats[] = { {"bf16", make_bf16_format}, {"nvfp4", make_nvfp4_format}, {"mxfp4_triton", make_mxfp4_format}, {"ds_fp4", make_ds_fp4_format}, {"fp8_block", make_fp8_block_format}, {"gguf", make_gguf_format}, {"gguf_scaled", make_gguf_scaled_format}, }; std::unique_ptr make_format(const std::string& name, const MoeParams& p, const FormatArgs& a) { for (const auto& [n, make] : kFormats) if (name == n) return make(p, a); throw std::invalid_argument("unknown CPU MoE weight format " + name); } // One format instance as cpu_executor.py passes it: name, bank tables by role, integer options. using FormatSpec = std::tuple, std::map>; struct CpuMoeExecutor { // A host-function callback gets one pointer, so it carries the executor with the task. struct HostTask { CpuMoeExecutor* exec; MoeTask task; }; const int num_layers, num_experts; const bool apply_on_input; MoeParams params; // the formats keep a reference; declared before fmts std::vector> fmts; std::vector layer_fmt; // [num_layers], index into fmts int n_iblk = 0, n_hblk = 0; // kRowBlock blocks per intermediate row and per output row // Pinned IO shared by every layer and batch size (one task in flight): [max_tokens, H] x and y, [max_tokens, top_k] ids and w. const bf16_t* io_x; const int32_t* io_ids; const float* io_w; bf16_t* io_y; AlignedVector g_scratch; // [max_tokens * top_k * I] intermediate // Worker pool. int num_threads; std::vector core_ids; // worker tid -> logical CPU to pin to (may be empty) std::vector workers; std::mutex task_mtx; std::condition_variable task_cv; std::mutex sync_mtx; std::condition_variable sync_cv; std::atomic stop{false}; // spinning workers read it without the lock uint64_t cur_gen = 0; MoeTask cur_task{}; // Separate cache lines: idle workers poll `submitted` while busy workers hit p1_next/p2_next. alignas(64) std::atomic submitted{0}; alignas(64) std::atomic completed{0}; // Spin-then-park waits. Off unless the Python wrapper finds spare CPUs for the spinners. static constexpr auto kSpinWindow = std::chrono::milliseconds(50); std::atomic spin_wait{false}; bool coord_pinned = false; // Work split of the current task. alignas(64) std::atomic p1_next{0}; std::atomic p2_next{0}; std::atomic prt_next{0}; // intermediate-row prep phase (prep_inter) int64_t p1_total = 0, p2_total = 0, prt_total = 0; // Hybrid hands the CPU a few routes of a step: only these get work items, since a claim that finds nothing still costs a contended atomic. std::vector active_routes; // intermediate rows tok * top_k + k whose route names an expert std::vector active_tokens; // tokens with at least one such route // Pass 1 splits the work by expert: one item computes a block for every token on that expert, so the block is read from DRAM once. // Only a step where some expert has two or more routes is grouped; otherwise pass 1 item u is active route u. bool grouped = false; std::vector expert_routes; // active_routes grouped by expert, larger groups first, each group in token order std::vector expert_start; // expert_routes[expert_start[u], expert_start[u + 1]) is group u int p2_tokens = 1; // active tokens per pass 2 item // Grouping scratch over the experts a step uses; expert_slot is -1 for every expert between steps. std::vector expert_slot, slot_expert, slot_count, slot_order, size_at; std::vector slot_fill; std::atomic done_count{0}; std::atomic bar_count{0}; std::atomic bar_sense{0}; // ---- Flag-based GPU<->CPU handshake (replaces the per-layer cudaLaunchHostFunc pair) ---- // The GPU rings the doorbell with a task word at submit; this coordinator thread busy-polls it, runs the task on the worker pool, and completes it, which the GPU waits on at sync. // This removes the ~2x30-50us host-func dispatch round trips per MoE layer per decode step that otherwise idle the GPU (~6 ms/step on a 75-layer model). std::unique_ptr gpu_sync; std::unique_ptr kernel_io; // replaces the copies around the doorbell once enable_kernel_io() runs std::thread coord_thread; std::atomic coord_stop{false}; std::atomic flag_served{0}; // completed doorbells (tests, watchdog) std::map host_tasks; // host-function userData, keyed by doorbell word (graph-stable) CpuMoeExecutor(int num_threads_, int num_layers_, int num_experts_, int top_k_, int hidden_size, int inter_size, int max_tokens_, int activation_id, int apply_router_weight_on_input, std::vector formats, std::vector layer_formats, double swiglu_alpha_, double swiglu_limit_, std::vector core_ids_, uintptr_t x_ptr, uintptr_t ids_ptr, uintptr_t w_ptr, uintptr_t y_ptr) : num_layers(num_layers_), num_experts(num_experts_), apply_on_input(apply_router_weight_on_input != 0), params{top_k_, max_tokens_, hidden_size, inter_size, activation_id, static_cast(swiglu_alpha_), static_cast(swiglu_limit_)}, io_x(reinterpret_cast(x_ptr)), io_ids(reinterpret_cast(ids_ptr)), io_w(reinterpret_cast(w_ptr)), io_y(reinterpret_cast(y_ptr)), num_threads(num_threads_ > 0 ? num_threads_ : 1), core_ids(std::move(core_ids_)) { if (params.top_k > kMaxTopK) throw std::invalid_argument("CPU MoE supports top_k <= " + std::to_string(kMaxTopK)); for (auto& [name, banks, options] : formats) fmts.push_back(make_format(name, params, FormatArgs{name, std::move(banks), std::move(options)})); if (static_cast(layer_formats.size()) != num_layers) throw std::invalid_argument("CPU MoE needs the format of every layer"); for (const int f : layer_formats) if (f < 0 || f >= static_cast(fmts.size())) throw std::out_of_range("CPU MoE layer format index out of range"); layer_fmt = std::move(layer_formats); n_iblk = (params.I + kRowBlock - 1) / kRowBlock; n_hblk = (params.H + kRowBlock - 1) / kRowBlock; g_scratch.assign(static_cast(params.max_tokens) * params.top_k * params.I, 0); active_routes.reserve(static_cast(params.max_tokens) * params.top_k); active_tokens.reserve(params.max_tokens); const size_t max_routes = static_cast(params.max_tokens) * params.top_k; expert_routes.reserve(max_routes); expert_start.reserve(max_routes + 1); expert_slot.assign(num_experts, -1); slot_expert.assign(max_routes, 0); slot_count.assign(max_routes, 0); slot_order.assign(max_routes, 0); size_at.assign(max_routes + 1, 0); slot_fill.assign(max_routes, 0); for (int t = 0; t < num_threads; ++t) workers.emplace_back([this, t] { worker_loop(t); }); } void pin_self(int tid) { if (!core_ids.empty()) pin_this_thread(core_ids[tid % static_cast(core_ids.size())]); } ~CpuMoeExecutor() { coord_stop.store(true); if (coord_thread.joinable()) coord_thread.join(); { std::lock_guard lk(task_mtx); stop = true; } task_cv.notify_all(); for (auto& th : workers) if (th.joinable()) th.join(); } MoeTask task(int layer_id, int num_tokens) const { if (layer_id < 0 || layer_id >= num_layers || num_tokens < 1 || num_tokens > params.max_tokens) throw std::out_of_range("CPU MoE task (layer " + std::to_string(layer_id) + ", " + std::to_string(num_tokens) + " tokens) is outside the executor's layers or IO"); return MoeTask{static_cast(layer_id), static_cast(num_tokens)}; } std::string isa_name() const { std::vector names; for (const auto& f : fmts) if (const std::string n = f->isa_name(); std::find(names.begin(), names.end(), n) == names.end()) names.push_back(n); std::string out; for (const auto& n : names) out += (out.empty() ? "" : ",") + n; return out; } const MoeFormat& format_of(const MoeTask& t) const { return *fmts[layer_fmt[t.layer]]; } // Whether any route of the task names an expert; submit gives the others no work. bool has_routes(const MoeTask& t) const { const int64_t n = static_cast(t.num_tokens) * params.top_k; for (int64_t i = 0; i < n; ++i) if (io_ids[i] >= 0 && io_ids[i] < num_experts) return true; return false; } void barrier(int& local_sense) { local_sense ^= 1; if (bar_count.fetch_add(1) + 1 == num_threads) { bar_count.store(0); bar_sense.store(local_sense); } else { while (bar_sense.load() != local_sense) cpu_relax(); } } // Pass 1 item p = one kRowBlock block of one expert's intermediate rows, for each active route on that expert (one if // ungrouped). The routes run back to back on one worker, so every token after the first reads the block from cache. void do_pass1(const MoeTask& t, int64_t p) { const int top_k = params.top_k, H = params.H, I = params.I; const int64_t ib = p % n_iblk; const int64_t u = p / n_iblk; const int i0 = static_cast(ib) * kRowBlock; const int i1 = std::min(I, i0 + kRowBlock); const MoeFormat& f = format_of(t); const int64_t* routes = grouped ? expert_routes.data() + expert_start[u] : active_routes.data() + u; const int64_t n = grouped ? expert_start[u + 1] - expert_start[u] : 1; for (int64_t j = 0; j < n; ++j) { const int64_t r = routes[j]; const int tok = static_cast(r / top_k); const float w_in = apply_on_input ? io_w[r] : 1.0f; f.gate_up(static_cast(t.layer), io_ids[r], tok, i0, i1, io_x + (size_t)tok * H, w_in, g_scratch.data() + (size_t)r * I); } } // Pass 2 item p = one kRowBlock block of the output rows of p2_tokens active tokens, one token after another. // Tokens that share an expert read its rows of the block from cache after the first. void do_pass2(const MoeTask& t, int64_t p) { const int64_t hb = p % n_hblk; const int64_t first = p / n_hblk * p2_tokens; const int64_t last = std::min(static_cast(active_tokens.size()), first + p2_tokens); for (int64_t i = first; i < last; ++i) down_block(t, active_tokens[i], hb); } // One kRowBlock block of token tok's output row, summed over its routes. void down_block(const MoeTask& t, int tok, int64_t hb) { const int top_k = params.top_k, H = params.H, I = params.I; const int h0 = static_cast(hb) * kRowBlock; const int h1 = std::min(H, h0 + kRowBlock); DownRoute routes[kMaxTopK]; float w_out[kMaxTopK]; int n = 0; for (int k = 0; k < top_k; ++k) { const int e = io_ids[static_cast(tok) * top_k + k]; if (e < 0 || e >= num_experts) continue; const size_t r = (size_t)tok * top_k + k; routes[n] = {e, static_cast(r), g_scratch.data() + r * I}; w_out[n] = apply_on_input ? 1.0f : io_w[r]; ++n; } float part[kMaxTopK * kRowBlock]; const MoeFormat& f = format_of(t); if (n > 0) f.down(static_cast(t.layer), h0, h1, routes, n, part); const bool rounds_routes = f.rounds_routes(); bf16_t* y_row = io_y + (size_t)tok * H; for (int h = h0; h < h1; ++h) { float acc = 0.0f; for (int j = 0; j < n; ++j) { const float v = part[j * kRowBlock + h - h0] * w_out[j]; acc += rounds_routes ? bf16_to_f32(f32_to_bf16(v)) : v; } y_row[h] = f32_to_bf16(acc); } } void run_task_body(const MoeTask& t) { int local_sense = 0; for (;;) { int64_t p = p1_next.fetch_add(1, std::memory_order_relaxed); if (p >= p1_total) break; do_pass1(t, p); } barrier(local_sense); // Prepare the intermediate rows (per token,route) for the down GEMV, see prep_inter. // Needs all of pass1 done (a full row spans every iblk). if (prt_total > 0) { MoeFormat& f = *fmts[layer_fmt[t.layer]]; for (;;) { const int64_t i = prt_next.fetch_add(1, std::memory_order_relaxed); if (i >= prt_total) break; const int64_t r = active_routes[i]; f.prep_inter(r, g_scratch.data() + (size_t)r * params.I); } barrier(local_sense); } for (;;) { int64_t p = p2_next.fetch_add(1, std::memory_order_relaxed); if (p >= p2_total) break; do_pass2(t, p); } } // Tasks arrive once per MoE layer, sub-ms apart. Spinning skips a futex wakeup per worker per layer; parking after kSpinWindow keeps an idle server off the CPU. void worker_loop(int tid) { pin_self(tid); uint64_t my_gen = 0; using wclk = std::chrono::steady_clock; auto last_active = wclk::now(); for (;;) { uint64_t gen = submitted.load(std::memory_order_acquire); if (gen == my_gen && spin_wait.load(std::memory_order_relaxed)) { unsigned polls = 0; while (gen == my_gen) { cpu_relax(); if (stop.load(std::memory_order_relaxed)) return; if ((++polls & 1023u) == 0 && wclk::now() - last_active > kSpinWindow) break; gen = submitted.load(std::memory_order_acquire); } } if (gen == my_gen) { std::unique_lock lk(task_mtx); task_cv.wait(lk, [&] { return stop.load() || cur_gen != my_gen; }); if (stop.load()) return; gen = cur_gen; } // submit() writes cur_task before the release store to `submitted`. my_gen = gen; run_task_body(cur_task); last_active = wclk::now(); if (done_count.fetch_add(1) + 1 == num_threads) { completed.store(my_gen, std::memory_order_release); { std::lock_guard lk(sync_mtx); } sync_cv.notify_all(); } } } // Groups active_routes by expert, larger groups first so heavy pass 1 items do not run last; counts only the experts the // step uses, so submit pays no pass over all experts. A step where no expert has two routes is left ungrouped. void group_routes_by_expert() { int u = 0, most = 0; for (const int64_t r : active_routes) { int& s = expert_slot[io_ids[r]]; if (s < 0) { s = u; slot_expert[u] = io_ids[r]; slot_count[u++] = 0; } most = std::max(most, ++slot_count[s]); } grouped = most > 1; if (!grouped) { for (int s = 0; s < u; ++s) expert_slot[slot_expert[s]] = -1; return; } std::fill(size_at.begin(), size_at.begin() + most + 1, 0); for (int s = 0; s < u; ++s) ++size_at[slot_count[s]]; for (int c = most, first = 0; c >= 1; --c) { const int m = size_at[c]; size_at[c] = first; first += m; } for (int s = 0; s < u; ++s) slot_order[size_at[slot_count[s]]++] = s; expert_start.clear(); int64_t at = 0; for (int i = 0; i < u; ++i) { expert_start.push_back(at); slot_fill[slot_order[i]] = at; at += slot_count[slot_order[i]]; } expert_start.push_back(at); expert_routes.resize(active_routes.size()); for (const int64_t r : active_routes) expert_routes[slot_fill[expert_slot[io_ids[r]]]++] = r; for (int s = 0; s < u; ++s) expert_slot[slot_expert[s]] = -1; } void submit(MoeTask t) { const int num_tokens = static_cast(t.num_tokens); MoeFormat& fmt = *fmts[layer_fmt[t.layer]]; active_routes.clear(); active_tokens.clear(); for (int tok = 0; tok < num_tokens; ++tok) { const size_t before = active_routes.size(); for (int k = 0; k < params.top_k; ++k) { const int64_t r = static_cast(tok) * params.top_k + k; if (io_ids[r] >= 0 && io_ids[r] < num_experts) active_routes.push_back(r); } if (active_routes.size() > before) active_tokens.push_back(tok); else std::memset(io_y + static_cast(tok) * params.H, 0, sizeof(bf16_t) * params.H); } group_routes_by_expert(); // When some expert serves several tokens, pass 2 puts as many tokens in an item as still leaves each worker about // four items; otherwise there is nothing to reuse and an item stays one token, as finer items balance better. const int64_t n_tok = static_cast(active_tokens.size()); const int64_t groups = std::min(n_tok, (4 * static_cast(num_threads) + n_hblk - 1) / n_hblk); p2_tokens = grouped && groups > 0 ? static_cast((n_tok + groups - 1) / groups) : 1; const int64_t p1_items = grouped ? static_cast(expert_start.size()) - 1 : static_cast(active_routes.size()); p1_total = p1_items * n_iblk; p2_total = (n_tok + p2_tokens - 1) / p2_tokens * n_hblk; prt_total = fmt.preps_inter() ? static_cast(active_routes.size()) : 0; p1_next.store(0, std::memory_order_relaxed); p2_next.store(0, std::memory_order_relaxed); prt_next.store(0, std::memory_order_relaxed); done_count.store(0, std::memory_order_relaxed); bar_count.store(0, std::memory_order_relaxed); bar_sense.store(0, std::memory_order_relaxed); // Prepare the per-token input once, up front. // Single-threaded: tiny for decode, and done before the workers are woken below. for (const int tok : active_tokens) fmt.prep_input(tok, io_x + (size_t)tok * params.H); { std::lock_guard lk(task_mtx); cur_task = t; ++cur_gen; submitted.store(cur_gen, std::memory_order_release); } task_cv.notify_all(); } // Only a coordinator on its own core may spin: host-func callbacks and eager calls share cores with the workers. void sync(bool spin = false) { const uint64_t target = submitted.load(std::memory_order_acquire); if (spin) { using sclk = std::chrono::steady_clock; const auto t0 = sclk::now(); unsigned polls = 0; while (completed.load(std::memory_order_acquire) < target) { cpu_relax(); if ((++polls & 1023u) == 0 && sclk::now() - t0 > kSpinWindow) break; } if (completed.load(std::memory_order_acquire) >= target) return; } std::unique_lock lk(sync_mtx); sync_cv.wait(lk, [&] { return completed.load(std::memory_order_acquire) >= target; }); } void submit_with_cuda_stream(uintptr_t stream, int layer_id, int num_tokens) { const MoeTask t = task(layer_id, num_tokens); HostTask& h = host_tasks.try_emplace(to_word(t), HostTask{this, t}).first->second; cudaLaunchHostFunc(reinterpret_cast(stream), &CpuMoeExecutor::submit_cb, &h); } void sync_with_cuda_stream(uintptr_t stream) { cudaLaunchHostFunc(reinterpret_cast(stream), &CpuMoeExecutor::sync_cb, this); } // Start the busy-poll coordinator over the doorbell. ``pin_core`` >= 0 // pins the coordinator to that logical CPU (the worker auto-sizing reserves it), so its // polling never migrates onto / contends with a GEMV worker's core. void start_flag_coordinator(int pin_core) { gpu_sync = std::make_unique(); coord_stop.store(false); coord_pinned = CPU_MOE_HAS_AFFINITY && pin_core >= 0; coord_thread = std::thread([this, pin_core] { if (pin_core >= 0) pin_this_thread(pin_core); coordinator_loop(); }); } void flag_signal(uintptr_t stream, int layer_id, int num_tokens) { gpu_sync->gpu_submit(stream, to_word(task(layer_id, num_tokens))); } void flag_wait(uintptr_t stream) { gpu_sync->gpu_wait(stream); } void enable_kernel_io() { kernel_io = std::make_unique(gpu_sync->ready_word(), gpu_sync->done_word(), io_x, io_ids, io_w, io_y, params.H, params.top_k); } void flag_publish(uintptr_t stream, uintptr_t x, uintptr_t ids, uintptr_t w, int layer_id, int num_tokens) { kernel_io->publish(stream, reinterpret_cast(x), reinterpret_cast(ids), reinterpret_cast(w), num_tokens, to_word(task(layer_id, num_tokens))); } void flag_combine(uintptr_t stream, uintptr_t out, int num_tokens, bool accumulate) { kernel_io->combine(stream, reinterpret_cast(out), num_tokens, accumulate); } bool flag_pending() const { return gpu_sync && gpu_sync->pending(); } // Complete a doorbell the coordinator never answered so the stream can make progress. void poison_flag() { if (gpu_sync) gpu_sync->cpu_done(); } void coordinator_loop() { // Idle backoff (edge-device friendliness): spin hot only while decode traffic is // flowing, with a TIME-based hot window (an iteration count varies 2-6x with pause // cost, so some configs dozed every token). 50 ms since the last // flag comfortably covers intra- and inter-token gaps -- including prefill bursts and // --moe-cpu-layers gaps -- so steady decode never // sleeps and keeps the ~us-level wakeup. Past it, i.e. the engine is actually idle, // escalate to timed sleeps capped at 2 ms: a dozing coordinator costs <0.1% of a // core instead of 100%, and the only price is a <=2 ms discovery delay on the FIRST // MoE layer after an idle period (irrelevant next to prefill). The clock is sampled // every 1024 empty polls (~0.1-1 ms) to keep the hot loop cheap. using coord_clock = std::chrono::steady_clock; constexpr auto kHotWindow = std::chrono::milliseconds(50); constexpr int64_t kSleepCapUs = 2000; auto last_active = coord_clock::now(); unsigned empty_polls = 0; // unsigned: the hot-phase ++ must not overflow into UB int64_t sleep_us = 100; bool dozing = false; while (!coord_stop.load(std::memory_order_relaxed)) { const uint64_t word = gpu_sync->cpu_poll(); if (word != 0) { const MoeTask t = from_word(word); if (has_routes(t)) { submit(t); sync(coord_pinned && spin_wait.load(std::memory_order_relaxed)); } else { // What the pool would write for a task without routes, minus its fork and join. std::memset(io_y, 0, sizeof(bf16_t) * t.num_tokens * params.H); } // Release: the workers' y stores are visible before the GPU sees done. gpu_sync->cpu_done(); flag_served.fetch_add(1, std::memory_order_relaxed); last_active = coord_clock::now(); empty_polls = 0; sleep_us = 100; dozing = false; continue; } // Hot phase: pause-spin, consulting the clock only every 1024 empty polls (the // amortization must gate the CLOCK, not the sleep -- gating the sleep decision // let 1023/1024 idle iterations hot-poll). // Doze phase: sleep on EVERY iteration until traffic returns; one wake-up scan // (~0.5 us) per 2 ms sleep is ~0.03% duty. if (!dozing) { if ((++empty_polls & 1023u) != 0 || coord_clock::now() - last_active < kHotWindow) { cpu_relax(); continue; } dozing = true; } std::this_thread::sleep_for(std::chrono::microseconds(sleep_us)); sleep_us = std::min(sleep_us * 2, kSleepCapUs); } // Teardown: complete any in-flight wait so a replay caught mid-shutdown does not stall // until the watchdog fires. Runs before the destructor's join() returns, while the doorbell is still alive. gpu_sync->cpu_done(); } // Eager (non-graph) path: run one task to completion on the pool. void run(int layer_id, int num_tokens) { submit(task(layer_id, num_tokens)); sync(); } static void CUDART_CB submit_cb(void* ud) { HostTask* h = reinterpret_cast(ud); h->exec->submit(h->task); } static void CUDART_CB sync_cb(void* ud) { reinterpret_cast(ud)->sync(); } }; } // namespace } // namespace freetoken::cpu_moe PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { namespace py = pybind11; using freetoken::cpu_moe::CpuMoeExecutor; using freetoken::cpu_moe::GpuSync; py::class_(m, "CpuMoeExecutor") .def(py::init, std::vector, double, double, std::vector, uintptr_t, uintptr_t, uintptr_t, uintptr_t>(), py::arg("num_threads"), py::arg("num_layers"), py::arg("num_experts"), py::arg("top_k"), py::arg("hidden_size"), py::arg("inter_size"), py::arg("max_tokens"), py::arg("activation_id"), py::arg("apply_router_weight_on_input"), py::arg("formats"), py::arg("layer_formats"), py::arg("swiglu_alpha"), py::arg("swiglu_limit"), py::arg("core_ids"), py::arg("x_ptr"), py::arg("ids_ptr"), py::arg("w_ptr"), py::arg("y_ptr")) .def("submit_with_cuda_stream", &CpuMoeExecutor::submit_with_cuda_stream, py::arg("stream"), py::arg("layer_id"), py::arg("num_tokens"), py::call_guard()) .def("sync_with_cuda_stream", &CpuMoeExecutor::sync_with_cuda_stream, py::arg("stream"), py::call_guard()) .def("run", &CpuMoeExecutor::run, py::arg("layer_id"), py::arg("num_tokens"), py::call_guard()) .def("flag_served", [](const CpuMoeExecutor& e) { return e.flag_served.load(); }) .def("start_flag_coordinator", &CpuMoeExecutor::start_flag_coordinator, py::arg("pin_core")) .def("flag_signal", &CpuMoeExecutor::flag_signal, py::arg("stream"), py::arg("layer_id"), py::arg("num_tokens")) .def("flag_wait", &CpuMoeExecutor::flag_wait, py::arg("stream")) .def("flag_pending", &CpuMoeExecutor::flag_pending) .def("enable_kernel_io", &CpuMoeExecutor::enable_kernel_io) .def("flag_publish", &CpuMoeExecutor::flag_publish, py::arg("stream"), py::arg("x"), py::arg("ids"), py::arg("w"), py::arg("layer_id"), py::arg("num_tokens")) .def("flag_combine", &CpuMoeExecutor::flag_combine, py::arg("stream"), py::arg("out"), py::arg("num_tokens"), py::arg("accumulate")) .def("poison_flag", &CpuMoeExecutor::poison_flag) .def("isa_name", &CpuMoeExecutor::isa_name) .def("set_spin_wait", [](CpuMoeExecutor& e, bool v) { e.spin_wait.store(v, std::memory_order_relaxed); }, py::arg("enabled")) .def("get_spin_wait", [](const CpuMoeExecutor& e) { return e.spin_wait.load(); }); freetoken::cpu_moe::bind_locked_fetch(m); m.def("memops_probe", &GpuSync::probe, py::arg("stream")); m.def("kernel_io_probe", &freetoken::cpu_moe::KernelIo::probe, py::arg("stream")); m.def("host_func_capture_safe", &GpuSync::host_func_capture_safe); // ABI capability marker: the highest ActKind this build implements in the // GENERIC epilogue. CpuMoeExecutor.__init__ probes it before requesting an act // id the epilogue must handle -- a prebuilt .so from before ACT_SWIGLUOAI // accepts id 3 without error and silently computes the wrong activation // (act_apply falls through to gelu_tanh); the probe turns a stale extension // into a loud rebuild instruction instead of wrong model outputs. m.def("max_generic_act_id", []() { return static_cast(freetoken::cpu_moe::ACT_SWIGLU_CLAMP); }); // The ggml-cpu variant the gguf format calls: the kernel table address that freetoken.kernel.ggml_cpu.select() returns. m.def("set_ggml_cpu", &freetoken::cpu_moe::set_ggml_cpu, py::arg("api")); m.def("weight_formats", []() { py::list names; for (const auto& f : freetoken::cpu_moe::kFormats) names.append(f.first); return names; }); }