Files
Xiaoze Fan c5b5b7243a feat(moe): fetch locked-layer experts through pinned staging (#742)
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit

Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest.

* fix(moe): resize the locked fetch on cache rebuild and gather staging by device address

* feat(moe): log decode moe stats per layer group with --moe-collect-stats

* perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads

* feat(moe): take the locked fetch share from the ft bench bw profile by default

* fix(moe): fuse the locked fetch into publish and gather kernels and address review findings

* refactor(moe): gather the locked staging with fast_index_copy_multi

* refactor(moe): move the locked fetch setup out of the engine and address review findings

* fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
2026-10-20 14:59:05 -07:00

206 lines
8.2 KiB
C++

// Locked-layer fetch for hybrid decode (python/freetoken/moe/locked_fetch.py).
//
// A locked layer's host banks have no device address, so the GPU cannot gather its rows. Per decode step a publish kernel
// writes the fetch list (count + layer-local expert rows) into mapped meta and rings a doorbell (GpuSync's words); the
// coordinator and worker threads memcpy each listed row of every bank into pinned staging, then complete the doorbell,
// and fast_index_copy_multi copies staging row i into the i-th evicted slot after a stream memop wait, or with its own
// spin wait where memops do not work. Everything is kernels and memops, so it captures into the decode graph. Row i of a
// bank sits at i * the layer's own row bytes, so layers of different widths share staging.
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstring>
#include <memory>
#include <stdexcept>
#include <thread>
#include <vector>
#include <torch/extension.h>
#include <freetoken/hip_compat.h>
#include "gpu_sync.h"
#include "locked_fetch_io.h"
#include "threads.h"
namespace freetoken::cpu_moe {
namespace {
using fetch_clock = std::chrono::steady_clock;
// same idle policy as the MoE coordinator: spin while decode traffic flows, sleep once it stops
constexpr auto kHotWindow = std::chrono::milliseconds(50);
constexpr int64_t kChunkBytes = 256 << 10;
struct CopyJob {
char* dst;
const char* src;
int64_t bytes;
};
class LockedFetch {
public:
// src[l][b] / row_bytes[l][b]: host base and row bytes of layer l's bank b (0 for banks or layers it never fetches).
// memops: wait on the doorbell with a stream memop, else the gather spins on done_word().
LockedFetch(std::vector<std::vector<uintptr_t>> src, std::vector<std::vector<int64_t>> row_bytes, std::vector<uintptr_t> staging,
int max_rows, std::vector<int> cpus, bool memops)
: src_(std::move(src)), row_bytes_(std::move(row_bytes)), staging_(std::move(staging)), max_rows_(max_rows), memops_(memops) {
sync_ = std::make_unique<GpuSync>(memops);
void* meta = nullptr;
void* meta_dev = nullptr;
const size_t meta_bytes = sizeof(int64_t) * (1 + (max_rows + 1) / 2);
if (cudaHostAlloc(&meta, meta_bytes, cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess ||
cudaHostGetDevicePointer(&meta_dev, meta, 0) != cudaSuccess) {
(void)cudaGetLastError();
if (meta) (void)cudaFreeHost(meta);
throw std::runtime_error("locked fetch: mapped meta allocation failed");
}
meta_ = meta;
count_ = static_cast<volatile int64_t*>(meta);
rows_ = reinterpret_cast<const int32_t*>(static_cast<int64_t*>(meta) + 1);
count_dev_ = static_cast<int64_t*>(meta_dev);
rows_dev_ = reinterpret_cast<int32_t*>(static_cast<int64_t*>(meta_dev) + 1);
const int coord_cpu = cpus.empty() ? -1 : cpus[0];
for (size_t i = 1; i < cpus.size(); ++i) workers_.emplace_back([this, cpu = cpus[i]] { worker_loop(cpu); });
coord_ = std::thread([this, coord_cpu] { coordinator_loop(coord_cpu); });
}
~LockedFetch() {
stop_.store(true);
coord_.join();
for (auto& t : workers_) t.join();
(void)cudaFreeHost(meta_);
}
LockedFetch(const LockedFetch&) = delete;
LockedFetch& operator=(const LockedFetch&) = delete;
// Hands one layer step's fetch list (the hybrid ensure's num_indices and src_indices) to the copy threads, then waits for
// them on `stream` when memops work; otherwise the gather that follows waits on done_word() itself.
void publish(uintptr_t stream, int layer, uintptr_t num_indices, uintptr_t src_indices) {
locked_publish(stream, reinterpret_cast<const int64_t*>(num_indices), reinterpret_cast<const int32_t*>(src_indices), max_rows_,
count_dev_, rows_dev_, reinterpret_cast<volatile int64_t*>(sync_->ready_word()),
reinterpret_cast<volatile int64_t*>(sync_->done_word()), static_cast<int64_t>(layer) + 1);
if (memops_) sync_->gpu_wait(stream);
}
uintptr_t done_word() const { return sync_->done_word(); }
private:
void run_jobs() {
for (;;) {
const int64_t i = next_.fetch_add(1, std::memory_order_relaxed);
if (i >= static_cast<int64_t>(jobs_.size())) return;
std::memcpy(jobs_[i].dst, jobs_[i].src, static_cast<size_t>(jobs_[i].bytes));
}
}
void worker_loop(int cpu) {
if (cpu >= 0) pin_this_thread(cpu);
uint64_t seen = 0;
auto last_active = fetch_clock::now();
int64_t sleep_us = 20;
while (!stop_.load(std::memory_order_relaxed)) {
const uint64_t gen = gen_.load(std::memory_order_acquire);
if (gen == seen) {
if (fetch_clock::now() - last_active < kHotWindow) {
cpu_relax();
} else {
std::this_thread::sleep_for(std::chrono::microseconds(sleep_us));
sleep_us = std::min<int64_t>(sleep_us * 2, 2000);
}
continue;
}
seen = gen;
run_jobs();
finished_.fetch_add(1, std::memory_order_acq_rel);
last_active = fetch_clock::now();
sleep_us = 20;
}
}
// Splits this step's rows into chunk jobs and runs them with the workers.
void copy_layer(int layer) {
int64_t n = *count_;
n = std::max<int64_t>(0, std::min<int64_t>(n, max_rows_));
jobs_.clear();
const auto& src = src_[layer];
const auto& row_bytes = row_bytes_[layer];
for (int64_t i = 0; i < n; ++i) {
const int64_t row = rows_[i];
for (size_t b = 0; b < staging_.size(); ++b) {
const int64_t rb = row_bytes[b];
if (rb == 0) continue;
char* dst = reinterpret_cast<char*>(staging_[b]) + i * rb;
const char* s = reinterpret_cast<const char*>(src[b]) + row * rb;
for (int64_t off = 0; off < rb; off += kChunkBytes) jobs_.push_back({dst + off, s + off, std::min(kChunkBytes, rb - off)});
}
}
if (jobs_.empty()) return;
next_.store(0, std::memory_order_relaxed);
finished_.store(0, std::memory_order_relaxed);
gen_.fetch_add(1, std::memory_order_acq_rel);
run_jobs();
// on stop a worker may leave before it joins this generation; the coordinator's run_jobs already took every job
while (finished_.load(std::memory_order_acquire) < static_cast<int64_t>(workers_.size()) && !stop_.load(std::memory_order_relaxed))
cpu_relax();
}
void coordinator_loop(int cpu) {
if (cpu >= 0) pin_this_thread(cpu);
auto last_active = fetch_clock::now();
int64_t sleep_us = 20;
while (!stop_.load(std::memory_order_relaxed)) {
const uint64_t task = sync_->cpu_poll();
if (task == 0) {
if (fetch_clock::now() - last_active < kHotWindow) {
cpu_relax();
} else {
std::this_thread::sleep_for(std::chrono::microseconds(sleep_us));
sleep_us = std::min<int64_t>(sleep_us * 2, 2000);
}
continue;
}
copy_layer(static_cast<int>(task - 1));
// release: the staging stores are visible before the GPU sees done
sync_->cpu_done();
last_active = fetch_clock::now();
sleep_us = 20;
}
sync_->cpu_done(); // a replay caught mid-teardown does not stall
}
std::vector<std::vector<uintptr_t>> src_;
std::vector<std::vector<int64_t>> row_bytes_;
std::vector<uintptr_t> staging_;
int64_t max_rows_;
bool memops_;
std::unique_ptr<GpuSync> sync_;
void* meta_ = nullptr;
const volatile int64_t* count_ = nullptr;
const int32_t* rows_ = nullptr;
int64_t* count_dev_ = nullptr;
int32_t* rows_dev_ = nullptr;
std::vector<CopyJob> jobs_;
std::atomic<int64_t> next_{0};
std::atomic<int64_t> finished_{0};
std::atomic<uint64_t> gen_{0};
std::atomic<bool> stop_{false};
std::vector<std::thread> workers_;
std::thread coord_;
};
} // namespace
void bind_locked_fetch(pybind11::module_& m) {
namespace py = pybind11;
py::class_<LockedFetch>(m, "LockedFetch")
.def(py::init<std::vector<std::vector<uintptr_t>>, std::vector<std::vector<int64_t>>, std::vector<uintptr_t>, int, std::vector<int>,
bool>(),
py::arg("src"), py::arg("row_bytes"), py::arg("staging"), py::arg("max_rows"), py::arg("cpus"), py::arg("memops"))
.def("publish", &LockedFetch::publish, py::arg("stream"), py::arg("layer"), py::arg("num_indices"), py::arg("src_indices"))
.def("done_word", &LockedFetch::done_word);
}
} // namespace freetoken::cpu_moe