Files
FreeToken/python/freetoken/kernel/fast_index_copy.py
Xiaoze Fan c5b5b7243a feat(moe): fetch locked-layer experts through pinned staging (#742)
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit

Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest.

* fix(moe): resize the locked fetch on cache rebuild and gather staging by device address

* feat(moe): log decode moe stats per layer group with --moe-collect-stats

* perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads

* feat(moe): take the locked fetch share from the ft bench bw profile by default

* fix(moe): fuse the locked fetch into publish and gather kernels and address review findings

* refactor(moe): gather the locked staging with fast_index_copy_multi

* refactor(moe): move the locked fetch setup out of the engine and address review findings

* fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
2026-10-20 14:59:05 -07:00

214 lines
7.6 KiB
Python

from __future__ import annotations
import math
import os
from functools import lru_cache
from typing import TYPE_CHECKING
import torch
from .utils import load_jit, make_cpp_args
if TYPE_CHECKING:
from tvm_ffi import Module
DEFAULT_NUM_BLOCKS = 4
SKIP_FAST_INDEX_COPY_ENV = "FREETOKEN_SKIP_FAST_INDEX_COPY"
_TRUE_VALUES = {"1", "true", "yes", "on"}
def _skip_fast_index_copy_enabled() -> bool:
return os.getenv(SKIP_FAST_INDEX_COPY_ENV, "").strip().lower() in _TRUE_VALUES
@lru_cache(maxsize=None)
def _jit_update_flag_module() -> Module:
return load_jit(
"fast_index_copy_update_flag",
cuda_files=["fast_index_copy.cuh"],
cuda_wrappers=[("update_copy_flag", "update_copy_flag")],
)
@lru_cache(maxsize=None)
def _jit_fast_index_copy_module(
*,
feature_size: int,
worker_threads: int,
worker_feature_size: int,
num_block: int,
) -> Module:
args = make_cpp_args(
feature_size,
worker_threads,
worker_feature_size,
1024,
num_block,
1,
)
return load_jit(
"fast_index_copy",
*args,
cuda_files=["fast_index_copy.cuh"],
cuda_wrappers=[
("launch", f"&FastIndexCopyKernel<{args}>::run"),
("launch_high", f"&FastIndexCopyKernel<{args}>::run_high"),
("launch_normal", f"&FastIndexCopyKernel<{args}>::run_normal"),
],
)
def _default_worker_threads(feature_size: int) -> int:
if feature_size <= 1024:
return 8
if feature_size <= 2048:
return 16
return 32
def _shrink_worker_feature_size(feature_size: int, worker_feature_size: int) -> int:
if feature_size < worker_feature_size:
worker_feature_size = feature_size
while feature_size % worker_feature_size != 0 and worker_feature_size > 128:
worker_feature_size //= 2
return worker_feature_size
def default_worker_args(feature_size: int) -> tuple[int, int, int]:
"""(worker_threads, worker_feature_size, num_block) of the default call path.
The AOT spec builder (kernel/aot.py) compiles with exactly these values, so a
default ``fast_index_copy_jit`` call hits the prebuilt cache by name.
"""
return (
_default_worker_threads(feature_size),
_shrink_worker_feature_size(feature_size, 2048),
DEFAULT_NUM_BLOCKS,
)
def fast_index_copy_jit(
dst: torch.Tensor,
dst_indices: torch.Tensor,
src: torch.Tensor,
src_indices: torch.Tensor,
num_indices: torch.Tensor | None = None,
*,
worker_threads: int | None = None,
worker_feature_size: int = 2048,
num_block: int | None = None,
priority: str | None = None,
sync_flag: torch.Tensor | None = None,
) -> None:
num_dst_feature = math.prod(dst.shape[1:])
num_src_feature = math.prod(src.shape[1:])
assert num_src_feature == num_dst_feature
# Debug/perf ablation: keep miss bookkeeping intact, but make the copy free to
# approximate a zero-copy-miss runtime. Outputs are only meaningful if callers
# already have valid cache contents for the requested indices.
if _skip_fast_index_copy_enabled():
return
dst = dst.as_strided(size=(dst.size(0), num_dst_feature), stride=(num_dst_feature, 1))
src = src.as_strided(size=(src.size(0), num_src_feature), stride=(num_src_feature, 1))
feature_size = dst.size(-1) * dst.element_size()
num_block = num_block or DEFAULT_NUM_BLOCKS
worker_threads = worker_threads or _default_worker_threads(feature_size)
worker_feature_size = _shrink_worker_feature_size(feature_size, worker_feature_size)
assert worker_threads in (8, 16, 32)
assert feature_size % worker_feature_size == 0
module = _jit_fast_index_copy_module(
feature_size=feature_size,
worker_threads=worker_threads,
worker_feature_size=worker_feature_size,
num_block=num_block,
)
if priority is None:
module.launch(dst, dst_indices, src, src_indices, num_indices)
return
assert priority in ("high", "normal")
assert sync_flag is not None
if priority == "high":
module.launch_high(dst, dst_indices, src, src_indices, num_indices, sync_flag)
return
module.launch_normal(dst, dst_indices, src, src_indices, num_indices, sync_flag)
@lru_cache(maxsize=None)
def _jit_fast_index_copy_multi_module(*, num_threads: int, blocks_per_bank: int) -> Module:
args = make_cpp_args(num_threads, blocks_per_bank)
return load_jit(
"fast_index_copy_multi",
*args,
cuda_files=["fast_index_copy.cuh"],
cuda_wrappers=[("launch", f"&MultiIndexCopyKernel<{args}>::run")],
)
def default_blocks_per_bank(device: torch.device) -> int:
"""H2D gather grid width per bank. The link's bandwidth-delay product sets how many
16B loads must be in flight: PCIe (~31 GB/s) saturates at ~4K threads/bank, NVLink-C2C
(GH200, ~450 GB/s) needs ~32K (measured: 8x1024 -> 222, 32x1024 -> 412 GB/s)."""
env = os.environ.get("FREETOKEN_H2D_BLOCKS_PER_BANK")
if env is not None:
blocks_per_bank = int(
env
)
assert blocks_per_bank > 0, (
"FREETOKEN_H2D_BLOCKS_PER_BANK must be greater than 0, "
f"got {blocks_per_bank}"
)
return blocks_per_bank # manual override (validate > 0)
if any(k in torch.cuda.get_device_name(device) for k in ("GH200", "GB200", "GB300")):
return 32
return 8
def fast_index_copy_multi_jit(
dst_ptrs: torch.Tensor,
src_ptrs: torch.Tensor,
feat_bytes: torch.Tensor,
dst_indices: torch.Tensor,
src_indices: torch.Tensor | None,
num_indices: torch.Tensor | None = None,
*,
num_threads: int = 1024,
blocks_per_bank: int = 8,
wait_flag: int = 0,
) -> None:
"""Fused multi-bank index copy: copy the same rows for every bank in ONE launch.
The copy is host->device PCIe-bandwidth bound (~31 GB/s measured, vs ~3 TB/s HBM), so what
saturates the link is the TOTAL in-flight request count, blocks_per_bank*num_threads
(~4096 threads/bank is the knee; blocks and threads are interchangeable -- 8x1024 ==
16x512 == 4x1024 all measure the same across small/medium/large-expert workloads). PCIe
is the bottleneck, not the GPU, so few blocks already feed it; the split only matters at
the extreme (~100k+ threads bumps the zero-copy launch overhead). 8x1024 = 8192/bank sits
~2x past the knee. Launch count is one regardless of grid width, so the zero-copy launch-
overhead win (and a strict >= over the legacy per-bank kernel across all workloads) holds.
``dst_ptrs``/``src_ptrs``/``feat_bytes`` are int64 [num_banks] device tensors built
once by the caller (the per-bank slot-cache base addr, host-source base addr, and
per-row byte size). Every bank's per-row byte size must be a multiple of 16, and the
base addresses 16-byte aligned (true for contiguous torch allocations of these banks).
``src_indices`` None copies source row i to ``dst_indices[i]``. ``wait_flag``, a GPU-visible
int64 address, makes every block first wait until it holds >= 1 (rows a CPU thread writes).
"""
if _skip_fast_index_copy_enabled():
return
module = _jit_fast_index_copy_multi_module(
num_threads=num_threads, blocks_per_bank=blocks_per_bank
)
module.launch(dst_ptrs, src_ptrs, feat_bytes, dst_indices, src_indices, num_indices, wait_flag)
def update_copy_flag_jit(sync_flag: torch.Tensor, delta: int) -> None:
assert sync_flag.is_cuda
assert sync_flag.numel() == 1
assert sync_flag.dtype == torch.int32
_jit_update_flag_module().update_copy_flag(sync_flag, delta)