mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(moe): fetch locked-layer experts through pinned staging and probe the pin limit Under hybrid, layers whose banks the pin quota could not take keep GPU slots of their own pools and fetch part of their misses through pinned staging that CPU threads fill (--moe-locked-fetch, --moe-locked-max-fetch). --moe-cpu-layers auto without FREETOKEN_PIN_BUDGET_GB now loads the banks unpinned, pins the staging, then pins layers middle-out until CUDA refuses one and OS-locks the rest. * fix(moe): resize the locked fetch on cache rebuild and gather staging by device address * feat(moe): log decode moe stats per layer group with --moe-collect-stats * perf(moe): default the locked fetch to max(2, min(4, logical cpus / 4)) copy threads * feat(moe): take the locked fetch share from the ft bench bw profile by default * fix(moe): fuse the locked fetch into publish and gather kernels and address review findings * refactor(moe): gather the locked staging with fast_index_copy_multi * refactor(moe): move the locked fetch setup out of the engine and address review findings * fix(moe): fail the boot when the locked staging cannot be pinned and keep locked layers out of the hybrid share log
214 lines
7.6 KiB
Python
214 lines
7.6 KiB
Python
from __future__ import annotations
|
|
|
|
import math
|
|
import os
|
|
from functools import lru_cache
|
|
from typing import TYPE_CHECKING
|
|
|
|
import torch
|
|
|
|
from .utils import load_jit, make_cpp_args
|
|
|
|
if TYPE_CHECKING:
|
|
from tvm_ffi import Module
|
|
|
|
|
|
DEFAULT_NUM_BLOCKS = 4
|
|
SKIP_FAST_INDEX_COPY_ENV = "FREETOKEN_SKIP_FAST_INDEX_COPY"
|
|
_TRUE_VALUES = {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _skip_fast_index_copy_enabled() -> bool:
|
|
return os.getenv(SKIP_FAST_INDEX_COPY_ENV, "").strip().lower() in _TRUE_VALUES
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def _jit_update_flag_module() -> Module:
|
|
return load_jit(
|
|
"fast_index_copy_update_flag",
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[("update_copy_flag", "update_copy_flag")],
|
|
)
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def _jit_fast_index_copy_module(
|
|
*,
|
|
feature_size: int,
|
|
worker_threads: int,
|
|
worker_feature_size: int,
|
|
num_block: int,
|
|
) -> Module:
|
|
args = make_cpp_args(
|
|
feature_size,
|
|
worker_threads,
|
|
worker_feature_size,
|
|
1024,
|
|
num_block,
|
|
1,
|
|
)
|
|
return load_jit(
|
|
"fast_index_copy",
|
|
*args,
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[
|
|
("launch", f"&FastIndexCopyKernel<{args}>::run"),
|
|
("launch_high", f"&FastIndexCopyKernel<{args}>::run_high"),
|
|
("launch_normal", f"&FastIndexCopyKernel<{args}>::run_normal"),
|
|
],
|
|
)
|
|
|
|
|
|
def _default_worker_threads(feature_size: int) -> int:
|
|
if feature_size <= 1024:
|
|
return 8
|
|
if feature_size <= 2048:
|
|
return 16
|
|
return 32
|
|
|
|
|
|
def _shrink_worker_feature_size(feature_size: int, worker_feature_size: int) -> int:
|
|
if feature_size < worker_feature_size:
|
|
worker_feature_size = feature_size
|
|
while feature_size % worker_feature_size != 0 and worker_feature_size > 128:
|
|
worker_feature_size //= 2
|
|
return worker_feature_size
|
|
|
|
|
|
def default_worker_args(feature_size: int) -> tuple[int, int, int]:
|
|
"""(worker_threads, worker_feature_size, num_block) of the default call path.
|
|
|
|
The AOT spec builder (kernel/aot.py) compiles with exactly these values, so a
|
|
default ``fast_index_copy_jit`` call hits the prebuilt cache by name.
|
|
"""
|
|
return (
|
|
_default_worker_threads(feature_size),
|
|
_shrink_worker_feature_size(feature_size, 2048),
|
|
DEFAULT_NUM_BLOCKS,
|
|
)
|
|
|
|
|
|
def fast_index_copy_jit(
|
|
dst: torch.Tensor,
|
|
dst_indices: torch.Tensor,
|
|
src: torch.Tensor,
|
|
src_indices: torch.Tensor,
|
|
num_indices: torch.Tensor | None = None,
|
|
*,
|
|
worker_threads: int | None = None,
|
|
worker_feature_size: int = 2048,
|
|
num_block: int | None = None,
|
|
priority: str | None = None,
|
|
sync_flag: torch.Tensor | None = None,
|
|
) -> None:
|
|
num_dst_feature = math.prod(dst.shape[1:])
|
|
num_src_feature = math.prod(src.shape[1:])
|
|
assert num_src_feature == num_dst_feature
|
|
|
|
# Debug/perf ablation: keep miss bookkeeping intact, but make the copy free to
|
|
# approximate a zero-copy-miss runtime. Outputs are only meaningful if callers
|
|
# already have valid cache contents for the requested indices.
|
|
if _skip_fast_index_copy_enabled():
|
|
return
|
|
|
|
dst = dst.as_strided(size=(dst.size(0), num_dst_feature), stride=(num_dst_feature, 1))
|
|
src = src.as_strided(size=(src.size(0), num_src_feature), stride=(num_src_feature, 1))
|
|
|
|
feature_size = dst.size(-1) * dst.element_size()
|
|
num_block = num_block or DEFAULT_NUM_BLOCKS
|
|
worker_threads = worker_threads or _default_worker_threads(feature_size)
|
|
worker_feature_size = _shrink_worker_feature_size(feature_size, worker_feature_size)
|
|
assert worker_threads in (8, 16, 32)
|
|
assert feature_size % worker_feature_size == 0
|
|
|
|
module = _jit_fast_index_copy_module(
|
|
feature_size=feature_size,
|
|
worker_threads=worker_threads,
|
|
worker_feature_size=worker_feature_size,
|
|
num_block=num_block,
|
|
)
|
|
if priority is None:
|
|
module.launch(dst, dst_indices, src, src_indices, num_indices)
|
|
return
|
|
|
|
assert priority in ("high", "normal")
|
|
assert sync_flag is not None
|
|
if priority == "high":
|
|
module.launch_high(dst, dst_indices, src, src_indices, num_indices, sync_flag)
|
|
return
|
|
module.launch_normal(dst, dst_indices, src, src_indices, num_indices, sync_flag)
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def _jit_fast_index_copy_multi_module(*, num_threads: int, blocks_per_bank: int) -> Module:
|
|
args = make_cpp_args(num_threads, blocks_per_bank)
|
|
return load_jit(
|
|
"fast_index_copy_multi",
|
|
*args,
|
|
cuda_files=["fast_index_copy.cuh"],
|
|
cuda_wrappers=[("launch", f"&MultiIndexCopyKernel<{args}>::run")],
|
|
)
|
|
|
|
def default_blocks_per_bank(device: torch.device) -> int:
|
|
"""H2D gather grid width per bank. The link's bandwidth-delay product sets how many
|
|
16B loads must be in flight: PCIe (~31 GB/s) saturates at ~4K threads/bank, NVLink-C2C
|
|
(GH200, ~450 GB/s) needs ~32K (measured: 8x1024 -> 222, 32x1024 -> 412 GB/s)."""
|
|
env = os.environ.get("FREETOKEN_H2D_BLOCKS_PER_BANK")
|
|
if env is not None:
|
|
blocks_per_bank = int(
|
|
env
|
|
)
|
|
assert blocks_per_bank > 0, (
|
|
"FREETOKEN_H2D_BLOCKS_PER_BANK must be greater than 0, "
|
|
f"got {blocks_per_bank}"
|
|
)
|
|
return blocks_per_bank # manual override (validate > 0)
|
|
if any(k in torch.cuda.get_device_name(device) for k in ("GH200", "GB200", "GB300")):
|
|
return 32
|
|
return 8
|
|
|
|
def fast_index_copy_multi_jit(
|
|
dst_ptrs: torch.Tensor,
|
|
src_ptrs: torch.Tensor,
|
|
feat_bytes: torch.Tensor,
|
|
dst_indices: torch.Tensor,
|
|
src_indices: torch.Tensor | None,
|
|
num_indices: torch.Tensor | None = None,
|
|
*,
|
|
num_threads: int = 1024,
|
|
blocks_per_bank: int = 8,
|
|
wait_flag: int = 0,
|
|
) -> None:
|
|
"""Fused multi-bank index copy: copy the same rows for every bank in ONE launch.
|
|
|
|
The copy is host->device PCIe-bandwidth bound (~31 GB/s measured, vs ~3 TB/s HBM), so what
|
|
saturates the link is the TOTAL in-flight request count, blocks_per_bank*num_threads
|
|
(~4096 threads/bank is the knee; blocks and threads are interchangeable -- 8x1024 ==
|
|
16x512 == 4x1024 all measure the same across small/medium/large-expert workloads). PCIe
|
|
is the bottleneck, not the GPU, so few blocks already feed it; the split only matters at
|
|
the extreme (~100k+ threads bumps the zero-copy launch overhead). 8x1024 = 8192/bank sits
|
|
~2x past the knee. Launch count is one regardless of grid width, so the zero-copy launch-
|
|
overhead win (and a strict >= over the legacy per-bank kernel across all workloads) holds.
|
|
|
|
``dst_ptrs``/``src_ptrs``/``feat_bytes`` are int64 [num_banks] device tensors built
|
|
once by the caller (the per-bank slot-cache base addr, host-source base addr, and
|
|
per-row byte size). Every bank's per-row byte size must be a multiple of 16, and the
|
|
base addresses 16-byte aligned (true for contiguous torch allocations of these banks).
|
|
|
|
``src_indices`` None copies source row i to ``dst_indices[i]``. ``wait_flag``, a GPU-visible
|
|
int64 address, makes every block first wait until it holds >= 1 (rows a CPU thread writes).
|
|
"""
|
|
if _skip_fast_index_copy_enabled():
|
|
return
|
|
module = _jit_fast_index_copy_multi_module(
|
|
num_threads=num_threads, blocks_per_bank=blocks_per_bank
|
|
)
|
|
module.launch(dst_ptrs, src_ptrs, feat_bytes, dst_indices, src_indices, num_indices, wait_flag)
|
|
|
|
|
|
def update_copy_flag_jit(sync_flag: torch.Tensor, delta: int) -> None:
|
|
assert sync_flag.is_cuda
|
|
assert sync_flag.numel() == 1
|
|
assert sync_flag.dtype == torch.int32
|
|
_jit_update_flag_module().update_copy_flag(sync_flag, delta)
|