Files
FreeToken/python/freetoken/kernel/csrc/pinned_tensor.cpp
Xiaoze Fan c5b5b7243a feat(moe): fetch locked-layer experts through pinned staging (#742)
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit

Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest.

* fix(moe): resize the locked fetch on cache rebuild and gather staging by device address

* feat(moe): log decode moe stats per layer group with --moe-collect-stats

* perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads

* feat(moe): take the locked fetch share from the ft bench bw profile by default

* fix(moe): fuse the locked fetch into publish and gather kernels and address review findings

* refactor(moe): gather the locked staging with fast_index_copy_multi

* refactor(moe): move the locked fetch setup out of the engine and address review findings

* fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
2026-10-20 14:59:05 -07:00

145 lines
5.6 KiB
C++

#include <cstdint>
#include <freetoken/hip_compat.h>
#include <torch/extension.h>
namespace {
void free_pinned(void *ptr) {
if (ptr != nullptr) {
cudaFreeHost(ptr);
}
}
torch::Tensor create_pinned_tensor_like(torch::Tensor input) {
TORCH_CHECK(input.device().is_cpu(), "Input tensor must be on CPU");
TORCH_CHECK(input.layout() == torch::kStrided,
"Input tensor must have strided layout");
const auto sizes = input.sizes().vec();
const auto strides = input.strides().vec();
const int64_t itemsize = input.element_size();
TORCH_CHECK(itemsize > 0, "Input tensor element size must be positive");
const bool is_empty = input.numel() == 0;
uint64_t storage_elements = is_empty ? 0 : 1;
for (int64_t i = 0; i < static_cast<int64_t>(sizes.size()); ++i) {
TORCH_CHECK(strides[i] >= 0, "Negative strides are not supported");
if (!is_empty) {
storage_elements += static_cast<uint64_t>(sizes[i] - 1) *
static_cast<uint64_t>(strides[i]);
}
}
const uint64_t nbytes = storage_elements * static_cast<uint64_t>(itemsize);
const size_t alloc_nbytes = static_cast<size_t>(nbytes == 0 ? 1 : nbytes);
void *data_ptr = nullptr;
const cudaError_t alloc_err = cudaMallocHost(&data_ptr, alloc_nbytes);
TORCH_CHECK(alloc_err == cudaSuccess,
"cudaMallocHost failed: ", cudaGetErrorString(alloc_err));
auto options = input.options().device(torch::kCPU).pinned_memory(true);
return torch::from_blob(data_ptr, sizes, strides, free_pinned, options);
}
torch::Tensor alloc_pinned_tensor(std::vector<int64_t> sizes,
at::ScalarType dtype) {
int64_t numel = 1;
for (const int64_t s : sizes) {
TORCH_CHECK(s >= 0, "Sizes must be non-negative");
numel *= s;
}
const uint64_t nbytes =
static_cast<uint64_t>(numel) * c10::elementSize(dtype);
const size_t alloc_nbytes = static_cast<size_t>(nbytes == 0 ? 1 : nbytes);
// Portable + mapped: the offload gather kernel reads these banks straight
// from host memory (zero-copy), which requires device-mapped pinned pages.
void *data_ptr = nullptr;
const cudaError_t alloc_err = cudaHostAlloc(
&data_ptr, alloc_nbytes, cudaHostAllocPortable | cudaHostAllocMapped);
if (alloc_err != cudaSuccess) {
// callers may catch the refusal and go on: leave no error for the next launch
(void)cudaGetLastError();
}
TORCH_CHECK(alloc_err == cudaSuccess,
"cudaHostAlloc failed: ", cudaGetErrorString(alloc_err));
auto options = torch::TensorOptions()
.dtype(dtype)
.device(torch::kCPU)
.pinned_memory(true);
return torch::from_blob(data_ptr, sizes, free_pinned, options);
}
// Pinned host memory is GPU-dereferenceable at its host VA only where UVA identity
// holds (Linux; not Windows/WDDM, where cudaHostRegister'd memory maps to a different
// device address). Zero-copy consumers resolve bank base addresses through these.
bool host_ptr_identity() {
int device = 0;
const cudaError_t err = cudaGetDevice(&device);
TORCH_CHECK(err == cudaSuccess, "cudaGetDevice failed: ", cudaGetErrorString(err));
int uva = 0, reg = 0;
cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, device);
cudaDeviceGetAttribute(&reg, cudaDevAttrCanUseHostPointerForRegisteredMem, device);
return uva == 1 && reg == 1;
}
int64_t host_device_ptr(int64_t host_ptr) {
void *dev_ptr = nullptr;
const cudaError_t err =
cudaHostGetDevicePointer(&dev_ptr, reinterpret_cast<void *>(host_ptr), 0);
TORCH_CHECK(err == cudaSuccess,
"cudaHostGetDevicePointer failed (host memory must be pinned+mapped): ",
cudaGetErrorString(err));
return reinterpret_cast<int64_t>(dev_ptr);
}
void host_register(int64_t addr, int64_t nbytes) {
const cudaError_t err =
cudaHostRegister(reinterpret_cast<void *>(addr), static_cast<size_t>(nbytes),
cudaHostRegisterPortable | cudaHostRegisterMapped);
if (err != cudaSuccess) {
// a refused pin is expected (the pin probe), so clear it before a later launch check reports it
(void)cudaGetLastError();
}
TORCH_CHECK(err == cudaSuccess,
"cudaHostRegister failed: ", cudaGetErrorString(err));
}
void host_unregister(int64_t addr) {
const cudaError_t err = cudaHostUnregister(reinterpret_cast<void *>(addr));
TORCH_CHECK(err == cudaSuccess,
"cudaHostUnregister failed: ", cudaGetErrorString(err));
}
int64_t driver_cuda_version() {
int version = 0; // stays 0 when no driver is installed
const cudaError_t err = cudaDriverGetVersion(&version);
TORCH_CHECK(err == cudaSuccess,
"cudaDriverGetVersion failed: ", cudaGetErrorString(err));
return version;
}
} // namespace
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.def("create_pinned_tensor_like", &create_pinned_tensor_like,
"Create an exact-size CPU pinned tensor with input's size/stride/dtype");
m.def("alloc_pinned_tensor", &alloc_pinned_tensor,
"Allocate an exact-size, uninitialized CPU pinned tensor");
m.def("host_ptr_identity", &host_ptr_identity,
"True if the GPU dereferences pinned host memory at its host VA (UVA identity)");
m.def("host_device_ptr", &host_device_ptr,
"Device-visible alias of a pinned+mapped host address");
m.def("host_register", &host_register,
"cudaHostRegister an existing host range as portable+mapped");
m.def("host_unregister", &host_unregister,
"cudaHostUnregister a range host_register pinned");
m.def("driver_cuda_version", &driver_cuda_version,
"Max CUDA version the installed NVIDIA driver supports (0 if none)");
}