Files
FreeToken/python/freetoken/moe/benchbw.py
Xiaoze Fan 3d23551c5b perf(cpu-moe): speed up the nvfp4 W4A8 kernels (#851)
The VNNI dots were compute bound per core; with fewer instructions per weight they reach the DRAM read bandwidth.
The W4A8 kernel now follows FREETOKEN_CPU_MOE_ISA.
2026-11-02 15:05:17 -08:00

1146 lines
50 KiB
Python

"""``ft bench bw``: measure host RAM vs PCIe bandwidth and pick the offload-family
MoE backend for a given model on this machine.
Two layers of measurement:
* Ceilings (hardware-only, model-agnostic): a STREAM-style CPU DRAM read and a
linear pinned<->device copy. Quick upper bounds.
* Real kernels per selected workload + expert dtype: the actual CPU MoE GEMV
(``CpuMoeExecutor``, what hybrid computes on the CPU) and the actual PCIe gather
(``OffloadMoeCache.copy_missing`` / ``fast_index_copy``, what offload streams over
PCIe on a decode miss). These are what the hybrid-vs-offload choice rides on.
When both work, the same pair is also measured *overlapped* (running concurrently,
contending for host DRAM) -- those two achieved bandwidths set the hybrid backend's
per-step fetch split (``bench_profile.fetch_fraction``).
The rule, per (workload, format): recommend ``hybrid`` when the CPU MoE kernel
bandwidth exceeds ``threshold`` x the PCIe gather bandwidth (default 2x), otherwise
``offload``. Formats with no CPU MoE weight path (e.g. block-fp8) can't be computed on
the CPU, so they always resolve to ``offload``.
Each bench writes one row per MoE layer kind: format, options, expert shape, top_k and CPU
worker threads (``bench_profile.layer_kind_key``). ``--model <checkpoint>`` benches every kind
a real checkpoint has, which is what the hybrid fetch split matches per layer; ``--dtype`` and
the ``--model`` presets bench canonical geometries. The rows live in a JSON file per GPU
(default ``$XDG_CACHE_HOME/freetoken/benchbw/<gpu-uuid>.json``); a run replaces the rows it
benched and keeps the others.
ft bench bw # per-dtype tuning bench (default: all formats)
ft bench bw --dtype nvfp4,bf16 # only these formats
ft bench bw --model qwen3.6-moe # per-model detail instead
ft bench bw --model /path/to/model.gguf # every MoE layer kind of a checkpoint
ft bench bw --dtype all --model all # both
ft bench bw --gpu GPU-2f3a... # bench a specific GPU (UUID or nvidia-smi index)
"""
from __future__ import annotations
import argparse
import contextlib
import json
import logging
import math
import os
import socket
import statistics
import subprocess
import threading
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from types import SimpleNamespace
import torch
from freetoken.gpu_select import (
assign_gpu,
bind_assigned_gpu,
gpu_identity,
single_gpu_arg,
)
from freetoken.kernel.pinned import alloc_pinned_tensor
from freetoken.moe.bench_profile import PROFILE_VERSION
from freetoken.moe.cpu_executor import physical_core_cpus, resolve_threads_and_affinity
from freetoken.utils import init_logger
logger = init_logger(__name__)
# Formats the CPU MoE C++ kernel can compute AND this bench can build banks for; anything
# else is offload-only here.
_CPU_MOE_FORMATS = frozenset({"bf16", "nvfp4", "mxfp4_triton", "ds_fp4", "fp8_block", "gguf", "gguf_scaled"})
# Formats this bench can build synthetic (correctly-sized) banks for.
_BUILDABLE_FORMATS = frozenset({"bf16", "nvfp4", "fp8_block", "mxfp4_triton", "ds_fp4"})
# Friendlier CLI/display aliases for the internal quant_format strings.
_FORMAT_ALIASES = {"fp8": "fp8_block", "mxfp4": "mxfp4_triton"}
_FORMAT_DISPLAY = {"fp8_block": "fp8", "mxfp4_triton": "mxfp4"}
# Cap synthetic bank memory per (format) so big models stay feasible; bandwidth is
# per-byte so a smaller expert count doesn't change the measured GB/s (as long as the
# working set still exceeds the LLC, which this budget guarantees).
_SYNTH_BANK_BUDGET = 2 << 30
# CPU MoE ISA tiers, forced via FREETOKEN_CPU_MOE_ISA (the kernel caps down to hw/build
# support, so requesting a higher tier than the CPU has is safe -- it clamps).
_ISA_TIERS = ("scalar", "avx2", "avx512", "avx512bf16")
@contextlib.contextmanager
def _forced_isa(isa: str | None):
"""Force the CPU MoE ISA tier for executors constructed in this block (None = auto)."""
key = "FREETOKEN_CPU_MOE_ISA"
if isa is None:
yield
return
prev = os.environ.get(key)
os.environ[key] = isa
try:
yield
finally:
if prev is None:
os.environ.pop(key, None)
else:
os.environ[key] = prev
@dataclass(frozen=True)
class Workload:
name: str
hidden: int # H
inter: int # moe_intermediate I
experts: int # E
top_k: int
formats: tuple[str, ...] # native expert quant_format(s) to bench
activation: str = "silu"
swiglu_alpha: float = 1.702
swiglu_limit: float | None = None
act_block: int | None = None # ds_fp4 W4A8 activation quant block
options: tuple[tuple[str, int], ...] = () # CPU executor options of the format, e.g. the GGUF types
def format_options(self, fmt: str) -> dict[str, int]:
opts = dict(self.options)
if self.act_block is not None and fmt == "ds_fp4":
opts.setdefault("act_block", self.act_block)
return opts
# Preset workloads. Dims from the model configs / benchmarks/bench_offload_cache_copy.py.
# E/top_k/H/I are what drive the per-expert byte size and thus the bandwidths.
WORKLOADS: dict[str, Workload] = {
"qwen3.6-moe": Workload("qwen3.6-moe", 2048, 512, 256, 8, ("bf16", "nvfp4", "fp8_block")),
"qwen3-30b": Workload("qwen3-30b", 2048, 768, 128, 8, ("bf16",)),
"gemma4-26b": Workload("gemma4-26b", 2816, 704, 128, 8, ("bf16",), activation="gelu_tanh"),
"gpt-oss-120b": Workload("gpt-oss-120b", 2880, 2880, 128, 4, ("mxfp4_triton",),
activation="gpt_oss_swiglu", swiglu_limit=7.0),
"gpt-oss-20b": Workload("gpt-oss-20b", 2880, 2880, 32, 4, ("mxfp4_triton",),
activation="gpt_oss_swiglu", swiglu_limit=7.0),
"dsv4": Workload("dsv4", 4096, 2048, 256, 6, ("ds_fp4",), swiglu_limit=7.0, act_block=128),
"dsv4.1-flash": Workload("dsv4.1-flash", 5120, 2304, 384, 6, ("ds_fp4",), swiglu_limit=10.0, act_block=32),
"glm4.7-nvfp4": Workload("glm4.7-nvfp4", 5120, 1536, 160, 8, ("nvfp4",)),
"glm5.3-flash-nvfp4": Workload(
"glm5.3-flash-nvfp4", 4096, 2048, 288, 8, ("nvfp4",),
activation="swiglu_clamp", swiglu_alpha=1.0, swiglu_limit=10.0,
),
"minimax-m2.5": Workload("minimax-m2.5", 3072, 1536, 256, 8, ("nvfp4",)),
}
# Per-dtype canonical geometries for the TUNING bench (`--dtype`): one representative geometry per
# expert format, the set the Desktop "detect" flow runs. A model's layers match these rows only when
# their kind is the same; `--model <checkpoint>` benches a model's own kinds. Geometries mirror the
# model families that ship each format (mxfp4 keeps gpt-oss's large experts -- the reason that
# format sits near the 2x threshold).
DTYPE_WORKLOADS: dict[str, Workload] = {
"bf16": Workload("dtype:bf16", 2048, 768, 128, 8, ("bf16",)),
"nvfp4": Workload("dtype:nvfp4", 3072, 1536, 128, 8, ("nvfp4",)),
"fp8_block": Workload("dtype:fp8", 2048, 512, 128, 8, ("fp8_block",)),
"mxfp4_triton": Workload("dtype:mxfp4", 2880, 2880, 128, 4, ("mxfp4_triton",),
activation="gpt_oss_swiglu", swiglu_limit=7.0),
"ds_fp4": Workload("dtype:ds_fp4", 4096, 2048, 128, 6, ("ds_fp4",), swiglu_limit=7.0, act_block=128),
}
def default_out_path(gpu_uuid: str | None = None) -> str:
# Single source of truth with the (torch-free) reader the engine consults.
from freetoken.moe.bench_profile import default_profile_path
return default_profile_path(gpu_uuid)
def _cgroup_mem_headroom() -> int | None:
"""Free bytes under this process's cgroup v2 memory limit, or None if unlimited."""
try:
with open("/sys/fs/cgroup/memory.max") as f:
raw = f.read().strip()
if raw == "max":
return None
limit = int(raw)
with open("/sys/fs/cgroup/memory.current") as f:
used = int(f.read().strip())
return max(0, limit - used)
except (OSError, ValueError):
return None
def _available_ram_bytes() -> int:
"""Free host RAM, clamped to the cgroup memory limit so a container isn't over-estimated."""
try:
host = os.sysconf("SC_AVPHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")
except (ValueError, OSError, AttributeError):
host = 8 << 30
cg = _cgroup_mem_headroom()
return min(host, cg) if cg is not None else host
# ============================== ceilings (hardware) ==============================
def measure_cpu_mem_bw(
num_threads: int = 0,
iters: int = 8,
min_total_bytes: int = 2 << 30,
per_core_bytes: int = 128 << 20,
) -> dict:
"""Aggregate host RAM read bandwidth (GB/s) across the physical cores (STREAM read).
One worker per physical core (``resolve_threads_and_affinity``), each pinned,
first-touching then streaming a private slice with a single-threaded reduction,
all overlapped. The buffer is sized past the last-level cache so every pass reads
DRAM, capped at half the available RAM (dropping worker count rather than under-
sizing a slice) so the fill can't OOM the box.
"""
n_threads, cores = resolve_threads_and_affinity(num_threads)
total_bytes = max(min_total_bytes, n_threads * per_core_bytes)
cap = _available_ram_bytes() // 2
if cap > 0 and total_bytes > cap:
max_fit = max(1, cap // per_core_bytes)
if max_fit < n_threads:
n_threads = max_fit
cores = cores[:n_threads]
total_bytes = min(max(min_total_bytes, n_threads * per_core_bytes), cap)
n_elems = (total_bytes // 4) // n_threads * n_threads # float32, evenly split
per = n_elems // n_threads
total_bytes = n_elems * 4
buf = torch.empty(n_elems, dtype=torch.float32) # workers first-touch their own slice
results: list[float] = [] # keeps the reductions reachable (no dead-code elimination)
prev_threads = torch.get_num_threads()
torch.set_num_threads(1) # each Python worker runs a single-threaded reduction
release = threading.Barrier(n_threads + 1)
done = threading.Barrier(n_threads + 1)
def worker(idx: int, core: int) -> None:
try:
os.sched_setaffinity(0, {core})
except (AttributeError, OSError):
pass
s = buf.narrow(0, idx * per, per)
s.fill_(1.0) # first-touch on this core's NUMA node + fault every page
acc = float(s.sum()) # warmup
release.wait()
for _ in range(iters):
acc += float(s.sum())
done.wait()
results.append(acc)
threads = [threading.Thread(target=worker, args=(i, c)) for i, c in enumerate(cores)]
for t in threads:
t.start()
try:
release.wait()
t0 = time.perf_counter()
done.wait()
wall = time.perf_counter() - t0
finally:
for t in threads:
t.join()
torch.set_num_threads(prev_threads)
return {
"bw_gbs": (total_bytes * iters) / wall / 1e9,
"threads": n_threads,
"buffer_bytes": total_bytes,
"iters": iters,
}
def measure_pcie_bw(device: torch.device, nbytes: int = 256 << 20, iters: int = 30) -> dict:
"""Linear pinned<->device copy bandwidth (GB/s) for H2D and D2H, via CUDA events."""
n = max(1, nbytes // 2) # bf16 elements
moved = n * 2
with torch.cuda.device(device):
src = alloc_pinned_tensor(n, dtype=torch.bfloat16)
dst = torch.empty(n, dtype=torch.bfloat16, device=device)
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
def _timed(copy) -> float:
for _ in range(3):
copy()
torch.cuda.synchronize(device)
start.record()
for _ in range(iters):
copy()
end.record()
end.synchronize()
return (moved * iters) / (start.elapsed_time(end) / 1e3) / 1e9
h2d = _timed(lambda: dst.copy_(src, non_blocking=True))
d2h = _timed(lambda: src.copy_(dst, non_blocking=True))
return {"h2d_gbs": h2d, "d2h_gbs": d2h, "bytes": moved, "iters": iters}
# ============================== bank geometry ==============================
def _gguf_bank_shapes(fmt: str, opts: dict, H: int, I: int) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
"""Per-expert (row shape, dtype) of the GGUF banks: ggml rows as bytes, float rows in their dtype."""
from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes
def rows(ggml_type: int, n: int, k: int):
dtype = float_dtype(ggml_type)
return ((n, k), dtype) if dtype is not None else ((n, row_bytes(ggml_type, k)), torch.uint8)
shapes = {"gate_up": rows(opts["gate_up_type"], 2 * I, H), "down": rows(opts["down_type"], H, I)}
if fmt == "gguf_scaled":
shapes["global"] = ((4,), torch.float32)
return shapes
def _offload_bank_specs(fmt: str, H: int, I: int, opts: dict | None = None) -> dict[str, tuple[int, torch.dtype]]:
"""Per-bank (elems_per_expert, dtype) for the flat offload host banks (the gather)."""
u8, bf16, f32, f8 = torch.uint8, torch.bfloat16, torch.float32, torch.float8_e4m3fn
if fmt in ("gguf", "gguf_scaled"):
return {name: (math.prod(shape), dtype) for name, (shape, dtype) in _gguf_bank_shapes(fmt, opts, H, I).items()}
if fmt == "bf16":
return {"gate_up": (2 * I * H, bf16), "down": (H * I, bf16)}
if fmt == "fp8_block":
return {
"gate_up": (2 * I * H, f8), "gate_up_scale": ((2 * I // 128) * (H // 128), bf16),
"down": (H * I, f8), "down_scale": ((H // 128) * (I // 128), bf16),
}
if fmt == "nvfp4":
return {
"gate_up_packed": (2 * I * (H // 2), u8), "gate_up_scale": (2 * I * (H // 16), u8),
"gate_up_global": (2 * I, f32), "down_packed": (H * (I // 2), u8),
"down_scale": (H * (I // 16), u8), "down_global": (H, f32),
}
if fmt == "mxfp4_triton":
return {
"gate_up_blocks": (2 * I * (H // 2), u8), "gate_up_scales": (2 * I * (H // 32), u8),
"gate_up_bias": (2 * I, bf16), "down_blocks": (H * (I // 2), u8),
"down_scales": (H * (I // 32), u8), "down_bias": (H, bf16),
}
if fmt == "ds_fp4":
return {
"gate_up_packed": (2 * I * (H // 2), u8), "gate_up_scale": (2 * I * (H // 32), u8),
"down_packed": (H * (I // 2), u8), "down_scale": (H * (I // 32), u8),
}
raise NotImplementedError(fmt)
def _expert_bytes(fmt: str, H: int, I: int, opts: dict | None = None) -> int:
return sum(elems * dt.itemsize for elems, dt in _offload_bank_specs(fmt, H, I, opts).values())
def _synth_experts(E: int, expert_bytes: int) -> int:
"""Expert count for the synthetic banks: as many as fit the memory budget (bandwidth is
per-byte, so fewer experts don't change the GB/s), capped at the model's E. The budget
(2 GiB) far exceeds any LLC, so the working set stays DRAM-sized whenever E is large; for
huge-expert models each expert already exceeds the LLC, so a handful still defeats it."""
return min(E, max(1, _SYNTH_BANK_BUDGET // max(1, expert_bytes)))
def _cpu_moe_bank_sources(fmt: str, H: int, I: int, E: int, opts: dict | None = None) -> dict:
"""3D pinned banks in the exact layout ``CpuMoeExecutor`` expects for ``fmt``.
Scale banks that decode through an e8m0 exponent (mxfp4/ds_fp4) are set to a unit
exponent so no weight lands in the float32 denormal range (which would slow the CPU
GEMV and skew the timing); packed e2m1 codes are finite for any byte, so they're left
as-is. Values otherwise don't affect the kernel's work.
"""
def pin(*shape, dtype):
return alloc_pinned_tensor(*shape, dtype=dtype)
if fmt in ("gguf", "gguf_scaled"):
from freetoken.third_party.gguf import GGML_QUANT_SIZES, GGMLQuantizationType
b = {}
for name, (shape, dtype) in _gguf_bank_shapes(fmt, opts, H, I).items():
b[name] = pin(E, *shape, dtype=dtype)
if name == "global":
b[name].fill_(1.0)
elif dtype == torch.uint8:
b[name].zero_() # zero ggml blocks: every fp16 block scale is 0, never denormal
if opts["gate_up_type" if name == "gate_up" else "down_type"] == GGMLQuantizationType.MXFP4:
# each block starts with an e8m0 scale byte; 0 decodes to 2^-127, a denormal that slows the CPU kernel
b[name].view(E, -1, GGML_QUANT_SIZES[GGMLQuantizationType.MXFP4][1])[..., 0] = 127
else:
b[name].fill_(0.02)
return b
if fmt == "bf16":
gate_up, down = pin(E, 2 * I, H, dtype=torch.bfloat16), pin(E, H, I, dtype=torch.bfloat16)
gate_up.fill_(0.02) # uninitialized bf16 can be denormal -> x86 FP slowdown
down.fill_(0.02)
return {"gate_up": gate_up, "down": down}
if fmt == "fp8_block":
def nb(n):
return (n + 127) // 128
b = {
"gate_up": pin(E, 2 * I, H, dtype=torch.float8_e4m3fn),
"gate_up_scale": pin(E, nb(2 * I), nb(H), dtype=torch.bfloat16),
"down": pin(E, H, I, dtype=torch.float8_e4m3fn),
"down_scale": pin(E, nb(H), nb(I), dtype=torch.bfloat16),
}
# e4m3 0x00 is a clean zero and the scales are unit-ish, so nothing lands in
# the fp32 denormal range (which would slow the GEMV and skew the timing).
b["gate_up"].view(torch.uint8).fill_(0x38) # 1.0
b["down"].view(torch.uint8).fill_(0x38)
b["gate_up_scale"].fill_(1.0)
b["down_scale"].fill_(1.0)
return b
if fmt == "nvfp4":
b = {
"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),
"gate_up_scale": pin(E, 2 * I, H // 16, dtype=torch.uint8),
"gate_up_global": pin(E, 2 * I, dtype=torch.float32),
"down_packed": pin(E, H, I // 2, dtype=torch.uint8),
"down_scale": pin(E, H, I // 16, dtype=torch.uint8),
"down_global": pin(E, H, dtype=torch.float32),
}
b["gate_up_global"].fill_(1.0)
b["down_global"].fill_(1.0) # e4m3 scales decode to normal float32
return b
if fmt == "mxfp4_triton": # transposed split-K layout
b = {
"gate_up_blocks": pin(E, H // 2, 2 * I, dtype=torch.uint8),
"gate_up_scales": pin(E, H // 32, 2 * I, dtype=torch.uint8),
"gate_up_bias": pin(E, 2 * I, dtype=torch.bfloat16),
"down_blocks": pin(E, I // 2, H, dtype=torch.uint8),
"down_scales": pin(E, I // 32, H, dtype=torch.uint8),
"down_bias": pin(E, H, dtype=torch.bfloat16),
}
b["gate_up_scales"].fill_(127) # e8m0 127 -> 2^0 = 1 (avoid denormal weights)
b["down_scales"].fill_(127)
b["gate_up_bias"].zero_()
b["down_bias"].zero_()
return b
if fmt == "ds_fp4":
b = {
"gate_up_packed": pin(E, 2 * I, H // 2, dtype=torch.uint8),
"gate_up_scale": pin(E, 2 * I, H // 32, dtype=torch.uint8),
"down_packed": pin(E, H, I // 2, dtype=torch.uint8),
"down_scale": pin(E, H, I // 32, dtype=torch.uint8),
}
b["gate_up_scale"].fill_(127) # e8m0 unit exponent
b["down_scale"].fill_(127)
return b
raise NotImplementedError(fmt)
# ========================= real kernels (per workload) =========================
def _build_gather_rig(fmt: str, wl: Workload, device: torch.device):
"""Production ``copy_missing`` rig: an ``OffloadMoeCache`` over synthetic pinned banks,
staged so every ``copy_missing()`` refills a full layer (all synth experts, scattered
sources). Returns ``(cache, bytes_per_copy)``; shared by the standalone PCIe gather
bench and the overlapped CPU+PCIe bench."""
from freetoken.moe.offload_cache import OffloadMoeCache
H, I, opts = wl.hidden, wl.inter, wl.format_options(fmt)
E = _synth_experts(wl.experts, _expert_bytes(fmt, H, I, opts))
specs = _offload_bank_specs(fmt, H, I, opts)
cache = OffloadMoeCache(num_layers=1, num_experts=E, cache_size=E, device=device, quant_format=fmt)
total_bytes = 0
sources = {}
for name, (elems, dtype) in specs.items():
sources[name] = [alloc_pinned_tensor(E, elems, dtype=dtype)] # cudaHostAlloc (mapped) like the loaders
total_bytes += E * elems * dtype.itemsize
cache.set_bank_sources(sources) # builds the fused copy plan (production default) when aligned
cache.evict_slots[:E].copy_(torch.arange(E, dtype=torch.int32, device=device))
cache.src_indices[:E].copy_(torch.randperm(E, device=device).to(torch.int32))
cache.num_indices.fill_(E) # int64, allocated by the cache -> matches the kernel contract
# copy_missing resolves the per-layer source through this (normally set by
# ensure_experts/materialize_layer, bypassed here since this bench pokes the
# evict/src/num_indices state directly).
cache._pending_src_layer = 0
return cache, total_bytes
def measure_pcie_gather_bw(fmt: str, wl: Workload, device: torch.device, iters: int = 20) -> dict:
"""Real PCIe gather bandwidth (GB/s): pinned host banks -> GPU slot cache.
Drives the production ``OffloadMoeCache.copy_missing`` (fused multi-bank
``fast_index_copy`` when 16-byte aligned, else per-bank), refilling a full layer
(all synth experts, scattered sources). Reusing the cache gives the kernel-correct
int64 ``num_indices``. Timed with CUDA events.
"""
eb = _expert_bytes(fmt, wl.hidden, wl.inter, wl.format_options(fmt))
cache, total_bytes = _build_gather_rig(fmt, wl, device)
E = cache.num_experts
for _ in range(3):
cache.copy_missing()
torch.cuda.synchronize(device)
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
if hasattr(torch.cuda, "_sleep"):
torch.cuda._sleep(10**7) # keep the CPU from running the launches ahead of the timer
start.record()
for _ in range(iters):
cache.copy_missing()
end.record()
end.synchronize()
ms = start.elapsed_time(end) / iters
return {
"bw_gbs": total_bytes / (ms / 1e3) / 1e9,
"expert_bytes": eb,
"synth_experts": E,
"fused": bool(cache._copy_fused_ok),
}
def _build_cpu_moe_executor(fmt: str, wl: Workload, banks: dict, num_threads: int, E: int):
"""A production ``CpuMoeExecutor`` over synthetic banks (via a minimal cache stand-in)."""
from freetoken.moe.cpu_executor import CpuMoeExecutor
cache = SimpleNamespace(
quant_format=fmt,
bank_sources={name: [t] for name, t in banks.items()},
num_layers=1, num_experts=E,
decode_target="cpu", cpu_executor=None,
)
return CpuMoeExecutor(
cache, top_k=wl.top_k, activation=wl.activation,
apply_router_weight_on_input=False, num_threads=num_threads, max_tokens=1,
device=torch.device("cuda"), swiglu_alpha=wl.swiglu_alpha,
swiglu_limit=wl.swiglu_limit, layer_formats=[(fmt, wl.format_options(fmt))] * cache.num_layers,
)
def _cpu_moe_step_fns(ex, wl: Workload, E: int):
"""``(set_ids, run)`` closures for bs=1 decode steps on a constructed executor.
``set_ids(i)`` routes step ``i`` to a rotating id window so consecutive steps touch
disjoint experts (working set >> LLC) -- the DRAM-bound regime hybrid decode actually
runs in; ``run()`` executes one step (releases the GIL inside the extension)."""
bs, top_k = 1, wl.top_k
io = ex._io
io["x"].copy_(torch.randn(bs, wl.hidden, dtype=torch.bfloat16) * 0.1)
io["w"].copy_(torch.rand(bs, top_k, dtype=torch.float32))
base = torch.arange(top_k, dtype=torch.int32)
def set_ids(i: int) -> None:
io["ids"].copy_((((i * top_k) + base) % E).view(bs, top_k))
def run() -> None:
ex._ext.run(0, bs)
return set_ids, run
def _time_cpu_moe(ex, wl: Workload, eb: int, E: int, iters: int) -> float:
"""bs=1 CPU MoE decode bandwidth (GB/s) for a constructed executor, median-timed."""
set_ids, run = _cpu_moe_step_fns(ex, wl, E)
warmup = 8
for i in range(warmup):
set_ids(i)
run()
samples = []
for i in range(iters):
set_ids(warmup + i) # continue the sweep so timed steps don't re-touch warmed experts
t0 = time.perf_counter()
run()
samples.append(time.perf_counter() - t0)
ms = statistics.median(samples) * 1e3
return (wl.top_k * eb) / (ms / 1e3) / 1e9
def measure_cpu_moe_bw(fmt: str, wl: Workload, iters: int = 64, num_threads: int = 0,
isas: list[str] | None = None) -> dict:
"""Real CPU MoE GEMV bandwidth (GB/s) at bs=1, reading experts from pinned host banks.
``bw_gbs``/``isa`` are ALWAYS the kernel's auto-picked (best supported) tier -- what
production actually runs -- so the recommendation never depends on a restricted ``--isa``
subset. ``isas`` (a list) additionally forces each ``FREETOKEN_CPU_MOE_ISA`` tier over
the SAME banks (built once) to populate the diagnostic ``isa_sweep`` breakdown; tiers
above the CPU's support clamp down and dedupe by the resulting isa name.
"""
H, I, opts = wl.hidden, wl.inter, wl.format_options(fmt)
eb = _expert_bytes(fmt, H, I, opts)
E = _synth_experts(wl.experts, eb)
banks = _cpu_moe_bank_sources(fmt, H, I, E, opts)
def run(isa: str | None) -> tuple[str, float, list[int]]:
with _forced_isa(isa):
ex = _build_cpu_moe_executor(fmt, wl, banks, num_threads, E)
bw = round(_time_cpu_moe(ex, wl, eb, E, iters), 2)
label, cores = ex.isa, list(ex.core_ids)
del ex
return label, bw, cores
auto_isa, auto_bw, cores = run(None) # recommendation basis: the tier production would pick
sweep = {auto_isa: auto_bw}
for isa in isas or []:
label, bw, _ = run(isa)
sweep[label] = max(sweep.get(label, 0.0), bw) # dedupe clamped tiers, keep best
return {"bw_gbs": auto_bw, "isa": auto_isa, "isa_sweep": sweep, "cores": cores,
"expert_bytes": eb, "synth_experts": E}
def measure_overlap_bw(fmt: str, wl: Workload, device: torch.device,
num_threads: int = 0, seconds: float = 2.0) -> dict:
"""Concurrent achieved bandwidths (GB/s): the CPU MoE GEMV and the PCIe gather running
at the same time -- the contention regime hybrid decode's overlap actually lives in.
The standalone numbers cannot predict this split. Assuming full DRAM contention (CPU
keeps ``cpu_bw - pcie_bw`` under DMA) over-penalizes a CPU kernel that never saturated
DRAM to begin with -- the DMA then mostly rides the leftover bandwidth; assuming no
contention ignores it entirely. Measuring the contended pair directly gives the hybrid
backend its bandwidth-matched fetch split: fetched : cpu-computed = pcie_ov : cpu_ov.
Both sides hammer their own pinned banks flat out for ``seconds`` from a shared
barrier: a worker thread loops bs=1 CPU decode steps (``run`` releases the GIL)
while the main thread loops full-layer ``copy_missing`` gathers (synchronized per copy
so the DMA is really in flight, not just enqueued). Each side reports bytes / its own
elapsed; the two windows differ by at most one CPU step + one gather.
"""
H, I, opts = wl.hidden, wl.inter, wl.format_options(fmt)
eb = _expert_bytes(fmt, H, I, opts)
E = _synth_experts(wl.experts, eb)
ex = _build_cpu_moe_executor(fmt, wl, _cpu_moe_bank_sources(fmt, H, I, E, opts), num_threads, E)
set_ids, run_step = _cpu_moe_step_fns(ex, wl, E)
cache, gather_bytes = _build_gather_rig(fmt, wl, device)
for i in range(8): # warm both sides (JIT, page faults, clocks)
set_ids(i)
run_step()
cache.copy_missing()
torch.cuda.synchronize(device)
start = threading.Barrier(2)
stop = threading.Event()
cpu_out: dict = {}
def cpu_side() -> None:
steps = 0
start.wait()
t0 = time.perf_counter()
while not stop.is_set():
set_ids(8 + steps)
run_step()
steps += 1
cpu_out["gbs"] = steps * wl.top_k * eb / (time.perf_counter() - t0) / 1e9
worker = threading.Thread(target=cpu_side)
worker.start()
start.wait()
t0 = time.perf_counter()
copies = 0
while time.perf_counter() - t0 < seconds:
cache.copy_missing()
torch.cuda.synchronize(device)
copies += 1
pcie_gbs = copies * gather_bytes / (time.perf_counter() - t0) / 1e9
stop.set()
worker.join()
del ex
return {"cpu_gbs": cpu_out["gbs"], "pcie_gbs": pcie_gbs}
def recommend(cpu_bw_gbs: float, pcie_bw_gbs: float, threshold: float = 2.0) -> str:
"""``hybrid`` iff CPU bandwidth exceeds ``threshold`` x PCIe bandwidth, else ``offload``."""
return "hybrid" if cpu_bw_gbs > threshold * pcie_bw_gbs else "offload"
# ================================ orchestration ================================
def _note(entry: dict, msg: str) -> None:
entry["note"] = f"{entry['note']} | {msg}" if entry.get("note") else msg
def _bench_format(fmt: str, wl: Workload, device: torch.device, threshold: float,
cpu_threads: int, cpu_iters: int, pcie_iters: int,
isas: list[str] | None = None) -> dict:
"""Per (workload, format): real PCIe gather + real CPU MoE GEMV + hybrid/offload verdict.
Expected degradations (extension not built, OOM, JIT failure) are caught per bench so
one format can't kill the run; an unexpected exception is left to propagate.
"""
entry: dict = {
"cpu_moe_gbs": None, "cpu_moe_isa": None,
"cpu_cores": None, "isa_sweep": None, "pcie_gather_gbs": None,
"cpu_moe_overlap_gbs": None, "pcie_gather_overlap_gbs": None,
"recommended": None,
"note": None,
}
try:
g = measure_pcie_gather_bw(fmt, wl, device, pcie_iters)
entry["pcie_gather_gbs"] = round(g["bw_gbs"], 2)
except (ImportError, RuntimeError) as e:
logger.warning(f"benchbw: PCIe gather bench failed for {wl.name}/{fmt}: {e}")
_note(entry, f"pcie gather unavailable ({e})")
finally:
torch.cuda.empty_cache()
if fmt not in _CPU_MOE_FORMATS:
_note(entry, f"CPU MoE has no {fmt} weight path; hybrid unavailable")
else:
try:
c = measure_cpu_moe_bw(fmt, wl, cpu_iters, cpu_threads, isas=isas)
entry["cpu_moe_gbs"] = c["bw_gbs"] # auto/best tier (what production runs)
entry["cpu_moe_isa"] = c["isa"]
entry["cpu_cores"] = c["cores"]
if isas: # a sweep was requested -> keep the per-tier breakdown
entry["isa_sweep"] = c["isa_sweep"]
# The tier label comes from the bf16 dot chooser, so it can over-report tiers
# the actual per-format kernel doesn't distinguish. Disclose the real story.
if fmt == "nvfp4": # nvfp4 is the only use_vnni format
_note(entry, "isa tier labels are nominal: avx512bf16 == avx512f here "
"(avx512 runs W4A8 with AVX512-VNNI, avx2 with AVX-VNNI; "
"a tier without its VNNI runs fp32 W4A16)")
elif fmt in ("mxfp4_triton", "ds_fp4", "fp8_block"):
_note(entry, "isa tier labels are nominal: avx512bf16 == avx512f here "
"(3 real kernels: scalar/avx2/avx512, no bf16/VNNI variant)")
elif fmt in ("gguf", "gguf_scaled"):
_note(entry, "isa tier labels are nominal: GGUF rows run on the ggml-cpu variant "
"picked at load, regardless of the forced tier")
except (ImportError, RuntimeError) as e:
logger.warning(f"benchbw: CPU MoE bench failed for {wl.name}/{fmt}: {e}")
_note(entry, f"cpu moe unavailable ({e})")
cpu_g, pcie_g = entry["cpu_moe_gbs"], entry["pcie_gather_gbs"]
if cpu_g is not None and pcie_g: # pcie_g truthy also rules out a div-by-zero
entry["recommended"] = recommend(cpu_g, pcie_g, threshold)
# Both sides work standalone -> also measure them contended (concurrently). This
# pair sets the hybrid backend's fetch split (bench_profile.fetch_fraction); the
# hybrid-vs-offload verdict above stays on the standalone numbers.
try:
o = measure_overlap_bw(fmt, wl, device, cpu_threads)
entry["cpu_moe_overlap_gbs"] = round(o["cpu_gbs"], 2)
entry["pcie_gather_overlap_gbs"] = round(o["pcie_gbs"], 2)
except (ImportError, RuntimeError) as e:
logger.warning(f"benchbw: overlap bench failed for {wl.name}/{fmt}: {e}")
_note(entry, f"overlap bench unavailable ({e})")
finally:
torch.cuda.empty_cache()
else:
# No CPU-vs-PCIe pair to compare (no CPU path, or a bench failed): offload is the
# backend that always works, so it's the safe call absent evidence for hybrid.
entry["recommended"] = "offload"
return entry
def _kind(fmt: str, wl: Workload) -> tuple:
"""A layer kind without the CPU thread count, which the row key adds once the executor has sized its pool."""
return (fmt, tuple(sorted(wl.format_options(fmt).items())), wl.hidden, wl.inter, wl.top_k)
def checkpoint_layer_kinds(model_path: str, device: torch.device) -> dict[tuple, Workload]:
"""Each distinct MoE layer kind of a checkpoint as hybrid decode would serve it: kind -> workload."""
from freetoken.distributed import DistributedInfo, set_tp_info, try_get_tp_info
from freetoken.engine.config import EngineConfig
from freetoken.engine.engine import meta_offload_layer_methods
from freetoken.layers import set_rope_device
if try_get_tp_info() is None:
set_tp_info(rank=0, size=1)
tp = try_get_tp_info()
cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(tp.rank, tp.size), dtype=torch.bfloat16, moe_strategy="hybrid")
if not getattr(cfg.model_config, "is_moe", False):
raise RuntimeError(f"{model_path} has no MoE layers")
set_rope_device(device)
methods = meta_offload_layer_methods(cfg, "hybrid")
if methods is None:
raise RuntimeError(f"{model_path}: its MoE layers bind no expert method")
kinds: dict[tuple, Workload] = {}
for m in methods:
fmt, opts = m.cpu_spec()
c = m.cfg
wl = Workload(
f"{os.path.basename(model_path.rstrip('/'))}:{fmt}", c.hidden, c.local_intermediate, c.num_experts, c.top_k, (fmt,),
activation=c.activation, swiglu_alpha=float(c.alpha), swiglu_limit=c.limit, options=tuple(sorted(opts.items())),
)
kinds.setdefault(_kind(fmt, wl), wl)
return kinds
def _freetoken_build() -> dict:
"""FreeToken version and, in a git checkout, its commit: rows benched on older kernels may be stale."""
from freetoken.version import __version__
try:
commit = subprocess.run(
["git", "-C", os.path.dirname(os.path.abspath(__file__)), "rev-parse", "--short=12", "HEAD"],
capture_output=True, text=True, timeout=5, check=True,
).stdout.strip() or None
except (OSError, subprocess.SubprocessError):
commit = None
return {"version": __version__, "commit": commit}
def _cpu_info() -> dict:
model = numa = None
try:
with open("/proc/cpuinfo") as f:
model = next((line.split(":", 1)[1].strip() for line in f if line.startswith("model name")), None)
except OSError:
pass
try:
numa = sum(1 for n in os.listdir("/sys/devices/system/node") if n.startswith("node") and n[4:].isdigit()) or None
except OSError:
pass
return {"model": model, "physical_cores": len(physical_core_cpus()), "numa_nodes": numa}
def _merge_previous(result: dict, path: str) -> None:
"""Keep the rows of the profile already at ``path`` (same GPU and version) that this run did not re-bench; a re-benched row keeps its other sources."""
try:
with open(path) as f:
prev = json.load(f)
except (OSError, ValueError):
return
if not isinstance(prev, dict) or prev.get("version") != result["version"] or (prev.get("gpu") or {}).get("uuid") != result["gpu"]["uuid"]:
return
prev_rows = prev.get("layer_kinds")
if not isinstance(prev_rows, dict):
return
rows = dict(prev_rows)
for key, row in result["layer_kinds"].items():
old = prev_rows.get(key)
if isinstance(old, dict) and isinstance(old.get("sources"), list):
row["sources"] = sorted(set(old["sources"]) | set(row["sources"]))
rows[key] = row
result["layer_kinds"] = rows
def _atomic_write_json(path: str, obj: dict) -> None:
os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True)
tmp = f"{path}.tmp.{os.getpid()}"
with open(tmp, "w") as f:
json.dump(obj, f, indent=2)
f.write("\n")
os.replace(tmp, path)
def run_benchbw(
out_path: str | None = None,
threshold: float = 2.0,
device_index: int = 0,
models: tuple[str, ...] = (),
dtypes: tuple[str, ...] | None = None,
formats: tuple[str, ...] | None = None,
isas: list[str] | None = None,
cpu_threads: int = 0,
cpu_iters: int = 8,
pcie_bytes: int = 256 << 20,
pcie_iters: int = 30,
kernel_cpu_iters: int = 64,
kernel_pcie_iters: int = 20,
checkpoints: tuple[str, ...] = (),
) -> dict:
from freetoken.moe.bench_profile import layer_kind_key
from freetoken.moe.cpu_executor import _format_label
if not torch.cuda.is_available():
raise RuntimeError(
"benchbw needs a CUDA device to measure PCIe bandwidth (both offload and "
"hybrid serve experts to the GPU)."
)
if torch.cuda.device_count() == 0:
raise RuntimeError(
f"no CUDA device visible (CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r})."
)
if not 0 <= device_index < torch.cuda.device_count():
raise RuntimeError(
f"device_index {device_index} out of range (found {torch.cuda.device_count()} CUDA devices)."
)
device = torch.device("cuda", device_index)
torch.cuda.set_device(device)
gpu = gpu_identity(device_index)
# kind -> (workload, sources); checkpoints are read before any timing so a bad one fails fast
units: dict[tuple, tuple[Workload, list[str]]] = {}
def add(fmt: str, wl: Workload, source: str) -> None:
sources = units.setdefault(_kind(fmt, wl), (wl, []))[1]
if source not in sources:
sources.append(source)
for fmt in dtypes or ():
add(fmt, DTYPE_WORKLOADS[fmt], f"dtype:{_FORMAT_DISPLAY.get(fmt, fmt)}")
for mname in models:
for fmt in formats or WORKLOADS[mname].formats:
add(fmt, WORKLOADS[mname], f"preset:{mname}")
for path in checkpoints:
logger.info(f"benchbw: reading the MoE layer kinds of {path} ...")
for kind, wl in checkpoint_layer_kinds(path, device).items():
add(kind[0], wl, path)
# Machine-parseable progress on stdout (opt-in), so the daemon/Desktop can stream feedback
# while the bench runs -- mirrors ft checkpoint's FTCONVERT lines. `done`/`total` count the
# measured units (ceilings + one per layer kind); `label` is what's measuring now.
prog_on = os.environ.get("FREETOKEN_BENCH_PROGRESS") == "1"
total = 1 + len(units)
done = 0
def _prog(label: str) -> None:
if prog_on:
print(f"FTBENCH {done} {total} {label}", flush=True)
_prog("ceilings")
logger.info(f"benchbw: measuring hardware ceilings on cuda:{device_index} ...")
cpu = measure_cpu_mem_bw(num_threads=cpu_threads, iters=cpu_iters)
pcie = measure_pcie_bw(device, nbytes=pcie_bytes, iters=pcie_iters)
done = 1
# CpuMoeExecutor logs an INFO line per construction; with --isa that's several per
# format and buries the report. Silence just that logger for the measurement loop.
exec_log = logging.getLogger("freetoken.moe.cpu_executor")
prev_level = exec_log.level
exec_log.setLevel(logging.WARNING)
now = datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
build = _freetoken_build()
rows: dict = {}
try:
for (fmt, opts, hidden, inter, top_k), (wl, sources) in units.items():
label = f"{_format_label(fmt, opts)} {hidden}x{inter} top{top_k}"
_prog(label)
logger.info(f"benchbw: {label} real kernels ...")
entry = _bench_format(fmt, wl, device, threshold, cpu_threads, kernel_cpu_iters, kernel_pcie_iters, isas)
done += 1
if not entry["cpu_cores"]:
logger.warning(f"benchbw: no CPU MoE result for {label}, so no row: {entry['note']}")
continue
threads = len(entry["cpu_cores"])
rows[layer_kind_key(fmt, dict(opts), hidden, inter, top_k, threads)] = {
"format": fmt, "options": dict(opts), "hidden": hidden, "inter": inter, "top_k": top_k,
"cpu_threads": threads, "activation": wl.activation, "sources": sources,
"threshold": threshold, "measured_at": now, "freetoken": build, **entry,
}
finally:
exec_log.setLevel(prev_level)
_prog("done")
result = {
"version": PROFILE_VERSION,
"timestamp": now,
"epoch": int(time.time()),
"host": socket.gethostname(),
# index is the CUDA ordinal the bench ran on; uuid keys the profile file
"gpu": {"index": device_index, "name": gpu["name"], "uuid": gpu["uuid"]},
"cpu": _cpu_info(),
"ceilings": {
"cpu_stream_read_gbs": round(cpu["bw_gbs"], 2),
"cpu_stream_threads": cpu["threads"],
"pcie_linear_h2d_gbs": round(pcie["h2d_gbs"], 2),
"pcie_linear_d2h_gbs": round(pcie["d2h_gbs"], 2),
},
"layer_kinds": rows,
}
out_path = os.path.expanduser(out_path or default_out_path(gpu["uuid"]))
benched = list(rows)
_merge_previous(result, out_path)
_atomic_write_json(out_path, result)
result["out_path"] = out_path
result["benched"] = benched
if prog_on:
print(f"FTBENCH_OUT {out_path}", flush=True)
return result
# ================================== reporting ==================================
def _gbs(x) -> str:
return f"{x:>8.1f} GB/s" if isinstance(x, (int, float)) else f"{'n/a':>13}"
def _print_kernels(kernels: dict, iw: int) -> None:
print(f" {'format':<8} {'expert':>9} {'CPU-MoE':>13} {'PCIe-gather':>13} "
f"{'CPU/PCIe':>9} backend")
for fmt, e in kernels.items():
disp = _FORMAT_DISPLAY.get(fmt, fmt)
eb = f"{_expert_bytes(fmt, e['hidden'], e['inter'], e['options']) / 2**20:.2f} MB"
cpu_g, pcie_g = e["cpu_moe_gbs"], e["pcie_gather_gbs"]
ratio = f"{cpu_g / pcie_g:.2f}x" if cpu_g is not None and pcie_g else "—"
print(f" {disp:<8} {eb:>9} {_gbs(e['cpu_moe_gbs'])} {_gbs(e['pcie_gather_gbs'])} "
f"{ratio:>9} {e['recommended']}")
c_ov, p_ov = e.get("cpu_moe_overlap_gbs"), e.get("pcie_gather_overlap_gbs")
if c_ov and p_ov:
print(f" overlapped: CPU-MoE {c_ov:.1f} + PCIe {p_ov:.1f} GB/s "
f"-> hybrid fetches {p_ov / (p_ov + c_ov):.1%} of misses")
if e.get("isa_sweep"):
tiers = sorted(e["isa_sweep"].items(), key=lambda kv: -kv[1])
for i, (k, v) in enumerate(tiers):
label = "isa:" if i == 0 else " "
print(f" {label} {k.split('(')[0]:<{iw}} {v:>7.1f} GB/s")
if e.get("note"):
print(f" └─ {e['note']}")
def _print_report(r: dict) -> None:
from freetoken.moe.cpu_executor import _format_label
c, cpu, rows = r["ceilings"], r["cpu"], r["layer_kinds"]
print()
print(f" host {r['host']} gpu cuda:{r['gpu']['index']} ({r['gpu']['name']}) "
f"cpu {cpu['model']} ({cpu['physical_cores']}c, {cpu['numa_nodes']} NUMA)")
print(f" ceilings: CPU STREAM read {c['cpu_stream_read_gbs']:.1f} | "
f"PCIe linear H2D {c['pcie_linear_h2d_gbs']:.1f} D2H {c['pcie_linear_d2h_gbs']:.1f} GB/s")
benched = [rows[key] for key in r["benched"]]
# one global ISA-name width so every sweep's GB/s lands in the same column
iw = max((len(k.split("(")[0]) for e in benched for k in (e.get("isa_sweep") or {})), default=0)
for e in benched:
label = _format_label(e["format"], tuple(sorted(e["options"].items())))
sources = ", ".join(os.path.basename(s.rstrip("/")) for s in e["sources"])
print(f"\n {label} H={e['hidden']} I={e['inter']} top_k={e['top_k']} threads={e['cpu_threads']} "
f" ({sources}; threshold {e['threshold']}x)")
_print_kernels({e["format"]: e}, iw)
print(f"\n saved: {r['out_path']} ({len(benched)} rows benched, {len(rows)} in the profile)\n")
# ================================ CLI plumbing ================================
def _positive_int(s: str) -> int:
v = int(s)
if v <= 0:
raise argparse.ArgumentTypeError("must be a positive integer")
return v
def _nonneg_int(s: str) -> int:
v = int(s)
if v < 0:
raise argparse.ArgumentTypeError("must be >= 0")
return v
def _positive_float(s: str) -> float:
v = float(s)
if v <= 0:
raise argparse.ArgumentTypeError("must be positive")
return v
def _model_list(s: str) -> tuple[str, ...]:
"""Preset names and checkpoint paths; a path is any item that exists on disk."""
if s.strip() == "all":
return tuple(WORKLOADS)
items = [x.strip() for x in s.split(",") if x.strip()]
bad = [x for x in items if x not in WORKLOADS and not os.path.exists(x)]
if bad or not items:
raise argparse.ArgumentTypeError(
f"'all', or a comma-separated list of checkpoint paths and presets from {sorted(WORKLOADS)}, got {s!r}"
)
return tuple(dict.fromkeys(items))
def _dtype_list(s: str) -> tuple[str, ...]:
if s.strip() == "all":
return tuple(DTYPE_WORKLOADS)
items = [_FORMAT_ALIASES.get(x.strip(), x.strip()) for x in s.split(",") if x.strip()]
bad = [x for x in items if x not in DTYPE_WORKLOADS]
if bad or not items:
raise argparse.ArgumentTypeError(
f"'all' or a comma-separated subset of {sorted(DTYPE_WORKLOADS)} "
f"(aliases {sorted(_FORMAT_ALIASES)}), got {s!r}"
)
return tuple(dict.fromkeys(items))
def _format_list(s: str) -> tuple[str, ...]:
items = [_FORMAT_ALIASES.get(x.strip(), x.strip()) for x in s.split(",") if x.strip()]
bad = [x for x in items if x not in _BUILDABLE_FORMATS]
if bad or not items:
raise argparse.ArgumentTypeError(
f"comma-separated subset of {sorted(_BUILDABLE_FORMATS)} (aliases "
f"{sorted(_FORMAT_ALIASES)}), got {s!r}"
)
return tuple(dict.fromkeys(items))
def _isa_list(s: str) -> list[str] | None:
if s.strip() == "auto":
return None
if s.strip() == "all":
return list(_ISA_TIERS)
items = [x.strip() for x in s.split(",") if x.strip()]
bad = [x for x in items if x not in _ISA_TIERS]
if bad or not items:
raise argparse.ArgumentTypeError(f"'auto', 'all', or a subset of {list(_ISA_TIERS)}, got {s!r}")
return list(dict.fromkeys(items))
def main(argv: list[str] | None = None, prog: str = "ft bench bw") -> int:
p = argparse.ArgumentParser(prog=prog, description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--dtype", type=_dtype_list, default=None,
help=f"tuning bench: 'all' (default when no --model), or comma-separated from "
f"{sorted(DTYPE_WORKLOADS)} -- one bench per expert format at a canonical geometry")
p.add_argument("--model", type=_model_list, default=None,
help=f"per-model detail instead of/in addition to --dtype: 'all', or comma-"
f"separated presets from {sorted(WORKLOADS)} and checkpoint paths (a "
f"checkpoint benches each of its MoE layer kinds, which the hybrid fetch "
f"split matches per layer)")
p.add_argument("--formats", type=_format_list, default=None,
help="override a --model's native expert formats (comma-separated)")
p.add_argument("--isa", type=_isa_list, default=None, dest="isas",
help=f"CPU MoE ISA: 'auto' (default, best), 'all', or a subset of "
f"{list(_ISA_TIERS)} to sweep (kernel caps down to hw support)")
p.add_argument("-o", "--out", default=None,
help=f"JSON output path (default {default_out_path('<gpu-uuid>')})")
p.add_argument("--threshold", type=_positive_float, default=2.0,
help="recommend hybrid when CPU BW > threshold x PCIe BW (default 2.0)")
p.add_argument("--gpu", type=single_gpu_arg, default=None,
help="GPU to bench: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or an "
"nvidia-smi index (default: the first visible GPU)")
p.add_argument("--cpu-threads", type=_nonneg_int, default=0,
help="CPU worker threads (0 = one per physical core)")
p.add_argument("--cpu-iters", type=_positive_int, default=8, help="STREAM read passes to time")
p.add_argument("--pcie-mib", type=_positive_int, default=256, help="linear PCIe copy size in MiB")
p.add_argument("--pcie-iters", type=_positive_int, default=30, help="linear PCIe copies to time")
p.add_argument("--kernel-cpu-iters", type=_positive_int, default=64,
help="CPU MoE decode steps to time")
p.add_argument("--kernel-pcie-iters", type=_positive_int, default=20,
help="fast_index_copy gather passes to time")
ns = p.parse_args(argv)
# same as ft serve --gpu: resolve, then bind by UUID at CUDA init
try:
assign_gpu(ns.gpu)
device_index = bind_assigned_gpu().index
except (ValueError, RuntimeError) as e:
p.error(str(e))
models = tuple(m for m in ns.model or () if m in WORKLOADS)
checkpoints = tuple(m for m in ns.model or () if m not in WORKLOADS)
dtypes = ns.dtype
# Default (no selection): the full per-dtype tuning bench. --model alone skips it; --dtype + --model runs both.
if not ns.model and not dtypes:
dtypes = tuple(DTYPE_WORKLOADS)
try:
result = run_benchbw(
out_path=ns.out, threshold=ns.threshold, device_index=device_index, models=models,
dtypes=dtypes, formats=ns.formats, isas=ns.isas, cpu_threads=ns.cpu_threads,
cpu_iters=ns.cpu_iters, pcie_bytes=ns.pcie_mib << 20, pcie_iters=ns.pcie_iters,
kernel_cpu_iters=ns.kernel_cpu_iters, kernel_pcie_iters=ns.kernel_pcie_iters,
checkpoints=checkpoints,
)
except (RuntimeError, OSError) as e:
print(f"error: {e}")
return 1
_print_report(result)
return 0
if __name__ == "__main__":
raise SystemExit(main())