Files
Xiaoze Fan c5b5b7243a feat(moe): fetch locked-layer experts through pinned staging (#742)
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit

Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest.

* fix(moe): resize the locked fetch on cache rebuild and gather staging by device address

* feat(moe): log decode moe stats per layer group with --moe-collect-stats

* perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads

* feat(moe): take the locked fetch share from the ft bench bw profile by default

* fix(moe): fuse the locked fetch into publish and gather kernels and address review findings

* refactor(moe): gather the locked staging with fast_index_copy_multi

* refactor(moe): move the locked fetch setup out of the engine and address review findings

* fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
2026-10-20 14:59:05 -07:00

167 lines
7.4 KiB
C++

#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <stdexcept>
#include <string>
#if defined(_WIN32)
#include <windows.h>
#else
#include <dlfcn.h>
#endif
#include <cuda_runtime_api.h>
#include "gpu_sync.h"
namespace freetoken::cpu_moe {
namespace {
// =====================================================================================
// CUDA stream memory operations (driver API, resolved via dlopen -- no link-time or
// toolchain dependence). The GPU side of the flag handshake: submit = WRITE_VALUE
// (done=0 then ready=task word), sync = WAIT_VALUE(done >= 1). The wait is
// executed by the GPU front-end (no SM-resident kernel), so GPU "utilization" stays
// truthful during CPU compute windows -- a resident spin kernel pinned it at 99%,
// which laptop CPU/GPU dynamic power schedulers answered by clamping the CPU's max
// frequency (GEMV workers -1.5x: the reported edge regression). Availability is
// probed functionally at startup (memops_probe); anything unsupported (some driver models,
// vGPU, old drivers) falls back to the cudaLaunchHostFunc path.
#if defined(_WIN32)
static void* cumemop_dlopen() { return (void*)::LoadLibraryA("nvcuda.dll"); }
static void* cumemop_dlsym(void* h, const char* n) {
return (void*)::GetProcAddress((HMODULE)h, n);
}
#else
static void* cumemop_dlopen() {
void* h = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL);
if (h == nullptr) h = dlopen("libcuda.so", RTLD_LAZY | RTLD_LOCAL);
return h;
}
static void* cumemop_dlsym(void* h, const char* n) { return dlsym(h, n); }
#endif
using cuMemOp64_fn = int (*)(void* stream, unsigned long long addr, unsigned long long value,
unsigned int flags);
static cuMemOp64_fn g_cu_write64 = nullptr;
static cuMemOp64_fn g_cu_wait64 = nullptr;
static constexpr unsigned int kCuWaitValueGeq = 0x0; // CU_STREAM_WAIT_VALUE_GEQ
static constexpr unsigned int kCuWriteDefault = 0x0; // CU_STREAM_WRITE_VALUE_DEFAULT
static bool cumemop_resolve() {
static bool resolved = [] {
void* h = cumemop_dlopen();
if (h == nullptr) return false;
// 11.7+ made the v2 entry points the default; older drivers export only the v1
// names with the same signature.
g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64_v2"));
if (g_cu_write64 == nullptr)
g_cu_write64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWriteValue64"));
g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64_v2"));
if (g_cu_wait64 == nullptr)
g_cu_wait64 = reinterpret_cast<cuMemOp64_fn>(cumemop_dlsym(h, "cuStreamWaitValue64"));
return g_cu_write64 != nullptr && g_cu_wait64 != nullptr;
}();
return resolved;
}
// GPU side of the flag handshake (see the block comment above): enqueued on the
// caller's (possibly capturing) stream; the WAIT immediate is the constant 1,
// replay-safe under CUDA graphs.
// The startup probe validates EAGER memops; a driver could still reject them at graph
// capture time. Those enqueue errors would otherwise be swallowed here and surface only
// as a later EndCapture failure -- log the first CUresult so triage is one step.
static void cumemop_check(int rc, const char* what) {
static std::atomic<bool> warned{false};
if (rc != 0 && !warned.exchange(true)) {
std::fprintf(stderr,
"[freetoken/cpu_moe] %s failed with CUresult=%d (first occurrence; "
"subsequent errors are not repeated). If this happened during CUDA "
"graph capture, the driver lacks capture support for stream memops -- "
"set FREETOKEN_CPU_MOE_FLAG_SYNC=0.\n",
what, rc);
}
}
} // namespace
// ready and done live on separate cache lines of one mapped-pinned block; with UVA the GPU memops and the coordinator use the same addresses.
struct GpuSync::Impl {
void* block = nullptr;
void* device_block = nullptr;
volatile int64_t* ready = nullptr;
volatile int64_t* done = nullptr;
};
GpuSync::GpuSync(bool require_memops) : impl_(std::make_unique<Impl>()) {
if (require_memops && !cumemop_resolve()) throw std::runtime_error("CPU MoE flag sync: CUDA stream memops are unavailable");
const cudaError_t rc = cudaHostAlloc(&impl_->block, 128, cudaHostAllocPortable | cudaHostAllocMapped);
if (rc != cudaSuccess)
throw std::runtime_error(std::string("CPU MoE flag allocation failed: ") + cudaGetErrorString(rc));
std::memset(impl_->block, 0, 128);
impl_->ready = static_cast<volatile int64_t*>(impl_->block);
impl_->done = impl_->ready + 8;
const cudaError_t dev_rc = cudaHostGetDevicePointer(&impl_->device_block, impl_->block, 0);
if (dev_rc != cudaSuccess) {
cudaFreeHost(impl_->block);
throw std::runtime_error(std::string("CPU MoE flag device address: ") + cudaGetErrorString(dev_rc));
}
}
GpuSync::~GpuSync() { cudaFreeHost(impl_->block); }
// Functional probe on a scratch pinned int64: enqueue WRITE(7) + WAIT(>=7) + sync.
// Returns true only if the whole memop path works on THIS stream/device/driver.
bool GpuSync::probe(uintptr_t stream) {
if (!cumemop_resolve()) return false;
void* scratch = nullptr;
if (cudaHostAlloc(&scratch, sizeof(int64_t), cudaHostAllocPortable | cudaHostAllocMapped) != cudaSuccess)
return false;
*static_cast<int64_t*>(scratch) = 0;
auto* s = reinterpret_cast<void*>(stream);
const auto addr = reinterpret_cast<unsigned long long>(scratch);
const bool ok = g_cu_write64(s, addr, 7ULL, kCuWriteDefault) == 0 && g_cu_wait64(s, addr, 7ULL, kCuWaitValueGeq) == 0 &&
cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)) == cudaSuccess;
// a failed enqueue can leave the WRITE pending on the scratch word
cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream));
cudaFreeHost(scratch);
return ok;
}
bool GpuSync::host_func_capture_safe() { return true; }
void GpuSync::gpu_submit(uintptr_t stream, uint64_t task) {
auto* s = reinterpret_cast<void*>(stream);
// Order matters and is preserved by the front end: reset done BEFORE raising ready,
// so the coordinator's completion write for THIS step can never be wiped.
cumemop_check(g_cu_write64(s, reinterpret_cast<unsigned long long>(impl_->done), 0ULL, kCuWriteDefault),
"cuStreamWriteValue64(done)");
cumemop_check(g_cu_write64(s, reinterpret_cast<unsigned long long>(impl_->ready), task, kCuWriteDefault),
"cuStreamWriteValue64(ready)");
}
void GpuSync::gpu_wait(uintptr_t stream) {
cumemop_check(g_cu_wait64(reinterpret_cast<void*>(stream), reinterpret_cast<unsigned long long>(impl_->done), 1ULL,
kCuWaitValueGeq),
"cuStreamWaitValue64(done)");
}
uint64_t GpuSync::cpu_poll() {
// Acquire: everything the GPU made visible before writing ready -- the D2H input copies -- is visible to the worker pool after this read.
const int64_t task = flag_load_acquire(impl_->ready);
if (task == 0) return 0;
flag_store_release(impl_->ready, 0); // consume this step's doorbell
return static_cast<uint64_t>(task);
}
void GpuSync::cpu_done() { flag_store_release(impl_->done, 1); }
bool GpuSync::pending() const { return flag_load_acquire(impl_->ready) != 0 && flag_load_acquire(impl_->done) == 0; }
uintptr_t GpuSync::ready_word() const { return reinterpret_cast<uintptr_t>(impl_->device_block); }
uintptr_t GpuSync::done_word() const { return reinterpret_cast<uintptr_t>(impl_->device_block) + 8 * sizeof(int64_t); }
} // namespace freetoken::cpu_moe