Files
FreeToken/python/freetoken/kernel/pinned.py
Xiaoze Fan c5b5b7243a feat(moe): fetch locked-layer experts through pinned staging (#742)
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit

Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest.

* fix(moe): resize the locked fetch on cache rebuild and gather staging by device address

* feat(moe): log decode moe stats per layer group with --moe-collect-stats

* perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads

* feat(moe): take the locked fetch share from the ft bench bw profile by default

* fix(moe): fuse the locked fetch into publish and gather kernels and address review findings

* refactor(moe): gather the locked staging with fast_index_copy_multi

* refactor(moe): move the locked fetch setup out of the engine and address review findings

* fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
2026-10-20 14:59:05 -07:00

74 lines
2.6 KiB
Python

"""Exact-size pinned host tensors (e.g. offload expert banks).
The offload gather kernel (``fast_index_copy``) reads host memory zero-copy from the
GPU, so allocations must be pinned + device-mapped. We avoid
``torch.empty(pin_memory=True)`` because its caching allocator rounds sizes up to the
next power of two (a 70GB bank would reserve 128GB)."""
from __future__ import annotations
import importlib
from functools import lru_cache
import torch
@lru_cache(maxsize=1)
def _load_pinned_extension():
try:
return importlib.import_module("freetoken.kernel._pinned_tensor")
except ImportError as exc:
raise ImportError(
"freetoken.kernel._pinned_tensor is not installed. Reinstall FreeToken "
"so the pinned tensor CUDA extension is built at install time."
) from exc
def create_pinned_tensor_like(input: torch.Tensor) -> torch.Tensor:
"""Create a CPU pinned tensor with the same size, stride, and dtype as input."""
return _load_pinned_extension().create_pinned_tensor_like(input)
def copy_to_pinned_tensor(input: torch.Tensor) -> torch.Tensor:
"""Copy a CPU tensor into exact-size cudaMallocHost pinned storage."""
output = create_pinned_tensor_like(input)
with torch.no_grad():
output.copy_(input)
return output
def alloc_pinned_tensor(*shape: int, dtype: torch.dtype) -> torch.Tensor:
"""Allocate an exact-size, uninitialized pinned host tensor via cudaHostAlloc."""
return _load_pinned_extension().alloc_pinned_tensor(list(shape), dtype)
def host_register(addr: int, nbytes: int) -> None:
"""cudaHostRegister ``nbytes`` at ``addr`` as portable+mapped (pin-after-fill)."""
_load_pinned_extension().host_register(addr, nbytes)
def host_unregister(addr: int) -> None:
"""cudaHostUnregister a range :func:`host_register` pinned."""
_load_pinned_extension().host_unregister(addr)
@lru_cache(maxsize=1)
def _host_ptr_identity() -> bool:
# cached per process: FreeToken pins one CUDA device per process (set at engine launch)
return bool(_load_pinned_extension().host_ptr_identity())
def device_ptr(t: torch.Tensor) -> int:
"""Base address of ``t`` as the GPU must dereference it.
Equals ``data_ptr()`` on CUDA tensors and wherever pinned host memory is
device-visible at its host VA (Linux/UVA). On Windows/WDDM registered memory maps
to a different device address, so zero-copy consumers must use this, not
``data_ptr()``. Host tensors must be pinned+mapped."""
if t.is_cuda or _host_ptr_identity():
return t.data_ptr()
return _load_pinned_extension().host_device_ptr(t.data_ptr())