mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* fix(moe): prevent stale cpu expert inputs in rocm graphs * fix(moe): publish rocm cpu inputs with graph events * fix(moe): qualify rocm kernel io with ordered copy fallback * fix(moe): enable locked hybrid staging on rocm --------- Co-authored-by: LZ-QWQ <lz1830837889@163.com>
236 lines
12 KiB
Python
236 lines
12 KiB
Python
"""Locked-layer fetch for hybrid decode.
|
|
|
|
A locked layer's host banks are OS-locked but not registered with CUDA (the pin quota ran out), so the GPU cannot
|
|
gather their rows. Each decode step a publish kernel hands the fetch list chosen by the hybrid ensure to CPU threads
|
|
(``_cpu_moe.LockedFetch``), which copy each fetched row of every bank into pinned staging, and fast_index_copy_multi
|
|
copies staging row i into ``evict_slots[i]``. The misses it does not fetch go to the CPU executor as on any hybrid layer.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import collections
|
|
import os
|
|
|
|
import torch
|
|
|
|
from freetoken.utils import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def locked_fetch_wanted(config) -> bool:
|
|
"""Whether hybrid layers fetch through pinned staging instead of staying CPU-only."""
|
|
return config.moe_strategy == "hybrid" and config.moe_locked_fetch != 0
|
|
|
|
|
|
def check_locked_fetch_config(config) -> None:
|
|
"""Reject --moe-locked-max-fetch past the experts of a layer before the banks are read (only with --moe-cpu-layers)."""
|
|
num_experts = config.model_config.num_experts
|
|
if config.moe_cpu_layers and locked_fetch_wanted(config) and not 1 <= config.moe_locked_max_fetch <= num_experts:
|
|
raise ValueError(f"--moe-locked-max-fetch {config.moe_locked_max_fetch} must be in [1, {num_experts}] (the experts of a layer)")
|
|
|
|
|
|
def staging_reserve(config, bank_bytes) -> int:
|
|
"""Pin budget the staging takes when --moe-cpu-layers is settled before the load: ``max_fetch`` rows of twice the mean expert.
|
|
|
|
``bank_bytes()`` estimates the banks; it runs only when the staging is wanted."""
|
|
if not (config.moe_cpu_layers and locked_fetch_wanted(config)) or not (total := bank_bytes()):
|
|
return 0
|
|
mc = config.model_config
|
|
return 2 * (total // (mc.num_moe_layers * mc.num_experts)) * config.moe_locked_max_fetch
|
|
|
|
|
|
def settle_banks(config, banks, *, probe: bool, has_locked: bool):
|
|
"""Pin the staging when locked layers may fetch, then with ``probe`` settle the DEFERRED banks (expert_banks.probe_banks).
|
|
|
|
Pinning the staging first keeps the banks from taking its share of the pin budget. Returns the banks, the locked layers
|
|
the probe found (None without ``probe``) and the staging (None when unused); a staging that cannot be pinned raises."""
|
|
from freetoken.moe.expert_banks import probe_banks
|
|
|
|
staging = None
|
|
if locked_fetch_wanted(config) and (probe or has_locked):
|
|
staging = pin_staging(banks.sources, config.moe_locked_max_fetch)
|
|
locked = None
|
|
if probe:
|
|
banks, locked = probe_banks(banks, config.model_config.num_moe_layers)
|
|
if staging is not None and not locked:
|
|
refund_staging(staging) # every bank got pinned
|
|
staging = None
|
|
return banks, locked, staging
|
|
|
|
|
|
def locked_pool_tags(layer_tags: list | None, locked, num_layers: int) -> list:
|
|
"""Bank layer tags with the locked layers marked, so they group into slot pools of their own."""
|
|
return [(t, "locked") if i in locked else t for i, t in enumerate(layer_tags or [None] * num_layers)]
|
|
|
|
|
|
def attach_locked_fetch(cache, config, staging: dict[str, torch.Tensor], layer_rows) -> None:
|
|
"""Set ``cache.locked_fetch`` for ``cache.locked_hybrid``; needs the cache's CPU executor. ``layer_rows`` as in fetch_plan."""
|
|
locked = sorted(cache.locked_hybrid)
|
|
plan = fetch_plan(config.moe_locked_fetch, locked, layer_rows)
|
|
cache.locked_fetch = LockedLayerFetch(cache, locked, plan, config.moe_locked_max_fetch, cache.cpu_executor, staging=staging)
|
|
|
|
|
|
def staging_bytes(sources: dict[str, list[torch.Tensor]], max_fetch: int) -> dict[str, int]:
|
|
"""Per canonical bank role: ``max_fetch`` rows of the bank's widest row over every layer."""
|
|
from freetoken.moe.legacy_format import canonical_role
|
|
|
|
out = {}
|
|
for name, per_layer in sources.items():
|
|
rows = [t[0].numel() * t.element_size() for t in per_layer if t.numel()]
|
|
out[canonical_role(name)] = max([16] + [r * max_fetch for r in rows])
|
|
return out
|
|
|
|
|
|
def alloc_staging(sizes: dict[str, int]) -> dict[str, torch.Tensor]:
|
|
"""Pinned staging per bank role."""
|
|
from freetoken.kernel.pinned import alloc_pinned_tensor
|
|
from freetoken.moe.host_banks import charge_pin, refund_pin
|
|
|
|
charge_pin(sum(sizes.values()))
|
|
try:
|
|
return {role: alloc_pinned_tensor(n, dtype=torch.uint8) for role, n in sizes.items()}
|
|
except BaseException:
|
|
refund_pin(sum(sizes.values()))
|
|
raise
|
|
|
|
|
|
def pin_staging(sources: dict[str, list[torch.Tensor]], max_fetch: int) -> dict[str, torch.Tensor]:
|
|
"""The staging for ``sources``; raises when it cannot be pinned."""
|
|
from freetoken.moe.host_banks import PinFailed
|
|
|
|
sizes = staging_bytes(sources, max_fetch)
|
|
try:
|
|
return alloc_staging(sizes)
|
|
except (PinFailed, RuntimeError) as exc:
|
|
raise RuntimeError(
|
|
f"could not pin the {sum(sizes.values()) / 2**20:.1f} MiB locked-layer staging ({exc}); free pinned host memory, "
|
|
"lower --moe-locked-max-fetch, or pass --moe-locked-fetch 0 to keep locked layers CPU-only"
|
|
) from exc
|
|
|
|
|
|
def refund_staging(staging: dict[str, torch.Tensor]) -> None:
|
|
"""Give the staging's bytes back to the simulated pin cap; the memory itself goes with its last reference."""
|
|
from freetoken.moe.host_banks import refund_pin
|
|
|
|
refund_pin(sum(t.numel() for t in staging.values()))
|
|
|
|
|
|
def fetch_plan(share: float, layers: list[int], layer_rows) -> dict[int, tuple[int, float]]:
|
|
"""(max_fetch, fraction) per locked layer for the hybrid ensure.
|
|
|
|
``share`` > 0 fetches that share of every step's misses; otherwise each layer takes p / (2p + c) from its `ft bench bw`
|
|
row, or a fixed cap of 1 without one. ``layer_rows()`` returns (row key, label) per MoE layer and the profile's rows, or None.
|
|
"""
|
|
if share > 0:
|
|
logger.info_rank0(f"--moe-locked-fetch {share:g}: locked layers fetch this share of each step's misses")
|
|
return {layer: (0, share) for layer in layers}
|
|
from freetoken.moe.bench_profile import locked_fetch_fraction
|
|
|
|
rows = layer_rows()
|
|
keys, labels, rows = rows if rows is not None else ([None] * (max(layers) + 1), [""] * (max(layers) + 1), {})
|
|
fractions = {layer: locked_fetch_fraction(rows.get(keys[layer])) for layer in layers}
|
|
groups = collections.Counter((labels[layer], f) for layer, f in fractions.items())
|
|
if fetched := [f"{f:.1%} on {n} {label} layers" for (label, f), n in groups.items() if f is not None]:
|
|
logger.info_rank0(f"--moe-locked-fetch auto: locked layers fetch this share of each step's misses (benched p / (2p + c)): {'; '.join(fetched)}")
|
|
if capped := [layer for layer, f in fractions.items() if f is None]:
|
|
logger.warning_rank0(
|
|
f"--moe-locked-fetch auto: no `ft bench bw` row for locked layers {capped}; they fetch at most 1 expert per step. "
|
|
"`ft bench bw --model <checkpoint>` benches each layer kind"
|
|
)
|
|
return {layer: (0, f) if f is not None else (1, 0.0) for layer, f in fractions.items()}
|
|
|
|
|
|
def _smt_siblings(cpus: list[int]) -> list[int]:
|
|
out = []
|
|
for cpu in cpus:
|
|
try:
|
|
with open(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list") as f:
|
|
text = f.read().strip()
|
|
except OSError:
|
|
continue
|
|
for part in text.split(","):
|
|
lo, _, hi = part.partition("-")
|
|
out.extend(i for i in range(int(lo), int(hi or lo) + 1) if i != cpu)
|
|
return out
|
|
|
|
|
|
def _copy_threads() -> int:
|
|
"""FREETOKEN_LOCKED_FETCH_THREADS, else max(2, min(4, logical CPUs / 4)): a few threads already saturate memcpy."""
|
|
if env := os.environ.get("FREETOKEN_LOCKED_FETCH_THREADS"):
|
|
return max(1, int(env))
|
|
logical = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
|
|
return max(2, min(4, logical // 4))
|
|
|
|
|
|
def _copy_cpus(executor, count: int) -> list[int]:
|
|
"""CPUs for the copy threads: SMT siblings of the executor's workers, else physical cores the executor leaves free."""
|
|
from freetoken.moe.cpu_executor import physical_core_cpus
|
|
|
|
busy = set(executor.core_ids) | {getattr(executor, "_coord_core", -1)}
|
|
cpus = [c for c in dict.fromkeys(_smt_siblings(executor.core_ids)) if c not in busy]
|
|
cpus = cpus or [c for c in physical_core_cpus() if c not in busy]
|
|
return cpus[:count]
|
|
|
|
|
|
class LockedLayerFetch:
|
|
"""Staging and copy threads for the locked layers of one ``OffloadMoeCache``."""
|
|
|
|
def __init__(self, cache, layers: list[int], plan: dict[int, tuple[int, float]], max_fetch: int, executor, staging: dict[str, torch.Tensor] | None = None) -> None:
|
|
from freetoken.kernel import _cpu_moe
|
|
from freetoken.kernel.pinned import device_ptr
|
|
from freetoken.moe.legacy_format import canonical_role
|
|
|
|
if not cache._copy_fused_ok:
|
|
raise ValueError("locked layers fetch through the fused multi-bank copy, which this cache could not plan (bank alignment or FREETOKEN_FUSED_COPY=0)")
|
|
# (max_fetch, fraction) of each locked layer for the hybrid ensure; max_fetch caps any of them
|
|
self.plan = plan
|
|
self.max_fetch = max_fetch
|
|
roles = [canonical_role(name) for name in cache.bank_schema]
|
|
nbanks = len(cache.banks)
|
|
src = [[0] * nbanks for _ in range(cache.num_layers)]
|
|
row_bytes = [[0] * nbanks for _ in range(cache.num_layers)]
|
|
need = [16] * nbanks
|
|
for b, (per_layer, _) in enumerate(cache.banks):
|
|
for layer in layers:
|
|
rb = int(cache._row_bytes[cache.layer_layout[layer]][b])
|
|
if per_layer[layer].numel() == 0 or rb == 0:
|
|
continue
|
|
src[layer][b] = per_layer[layer].data_ptr()
|
|
row_bytes[layer][b] = rb
|
|
need[b] = max(need[b], rb * max_fetch)
|
|
if staging is None:
|
|
staging = alloc_staging(dict(zip(roles, need)))
|
|
for b, role in enumerate(roles):
|
|
assert staging[role].numel() >= need[b], (role, staging[role].numel(), need[b])
|
|
self.staging = [staging[role] for role in roles]
|
|
self.staging_ptrs = torch.tensor([device_ptr(t) for t in self.staging], dtype=torch.int64, device=cache.device)
|
|
cpus = _copy_cpus(executor, _copy_threads())
|
|
# the executor's doorbell decision (FREETOKEN_CPU_MOE_FLAG_SYNC and the memop probe); without memops the gather spins on the GPU
|
|
memops = bool(getattr(executor, "_flag_sync", False))
|
|
self._ext = _cpu_moe.LockedFetch(src, row_bytes, [t.data_ptr() for t in self.staging], max_fetch, cpus, memops)
|
|
self._wait_flag = 0 if memops else self._ext.done_word()
|
|
logger.info_rank0(
|
|
f"locked layers {layers}: GPU slots + fetch up to {max_fetch} misses per step through "
|
|
f"{sum(t.numel() for t in self.staging) / 2**20:.1f} MiB of pinned staging, {max(1, len(cpus))} copy thread(s) on cpus {cpus or 'any'}"
|
|
+ ("" if memops else ", the GPU spin-waits on them (no stream memops)")
|
|
)
|
|
|
|
def copy(self, cache, layer_id: int) -> None:
|
|
"""Stage and gather the rows the last hybrid ensure scheduled for ``layer_id``; graph capturable."""
|
|
from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit
|
|
|
|
# the gather reads num_indices rows of the staging, which has max_fetch
|
|
assert cache._pending_fetch_cap == self.max_fetch, (cache._pending_fetch_cap, self.max_fetch)
|
|
self._ext.publish(torch.cuda.current_stream().cuda_stream, layer_id, cache.num_indices.data_ptr(), cache.src_indices.data_ptr())
|
|
fast_index_copy_multi_jit(
|
|
cache._copy_dst_ptrs_pool[cache.pool_of_layer[layer_id]],
|
|
self.staging_ptrs,
|
|
cache._copy_feat_bytes[cache.layer_layout[layer_id]],
|
|
cache.evict_slots,
|
|
None,
|
|
cache.num_indices,
|
|
blocks_per_bank=cache._copy_blocks_per_bank,
|
|
wait_flag=self._wait_flag,
|
|
)
|