"""Python wrapper around the ``_cpu_moe`` C++ executor (--moe-strategy cpu). Owns the persistent CPU worker pool and one pinned IO block of ``max_tokens`` rows that every layer and batch size shares (one task is in flight at a time). ``decode`` issues the whole CUDA-graph-capturable sequence on the current stream: D2H (hidden, topk_ids, topk_weights -> pinned IO rows [:bs]) -> publish (HIP external event before the memop doorbell) -> submit (doorbell naming (layer, bs), or a cudaLaunchHostFunc host node) -> sync (wait for the CPU pool to finish) -> H2D (pinned expert output -> GPU) With kernel IO, a publish kernel does the D2H and the doorbell, and skips the step when no route goes to the CPU; a combine kernel does the H2D (csrc/cpu_moe/gpu_sync_io.h). """ from __future__ import annotations import os import threading import time import weakref import torch from freetoken.kernel.pinned import alloc_pinned_tensor from freetoken.utils import init_logger logger = init_logger(__name__) # Flag-based GPU<->CPU handshake for hybrid/cpu decode. The default host-func path # (cudaLaunchHostFunc submit+sync per layer) pays ~30-50us of callback dispatch latency # per call with the GPU stream idle -- 2 calls per MoE layer per decode step (~6 ms/step # on a 75-layer model). Instead the GPU raises a mapped-pinned "ready" flag at submit; a # persistent CPU coordinator (in _cpu_moe) polls it, runs the layer, and sets a "done" # flag the GPU waits on at sync -- no host-func round-trip. Both GPU-side operations are # STREAM MEMORY OPERATIONS (cuStreamWriteValue64 / cuStreamWaitValue64 on CUDA, see # gpu_sync.h): they execute on the GPU front end with no SM-resident kernel, so # GPU utilization stays truthful during the CPU compute window. (The first cut used a # spin-wait kernel; that pinned reported utilization at 99% and laptop CPU/GPU dynamic # power schedulers responded by clamping the CPU frequency -- a net decode regression on # power-coupled edge devices.) The doorbell word names the (layer, decode batch size) # task, so one ready/done pair serves every layer and every captured graph (gpu_sync.h). # Where memops are unavailable (some driver models, vGPU, old drivers -- functionally probed at # startup), decode keeps the host-func path (functional, slower). ROCm 7.14 cannot replay # that path from a graph, so capture fails loudly there instead. A Python watchdog thread # turns a wedged coordinator into a loud RuntimeError (via raise_if_unhealthy) instead of # an indefinite stream stall. # Caveat: the coordinator busy-polls one core while decode traffic flows (idle backoff # otherwise); FREETOKEN_CPU_MOE_FLAG_SYNC=0 opts out entirely. _FLAG_SYNC = os.getenv("FREETOKEN_CPU_MOE_FLAG_SYNC", "1") != "0" # Pool threads spin up to 50 ms before they park. FREETOKEN_CPU_MOE_SPIN=0 opts out. _SPIN_WAIT = os.getenv("FREETOKEN_CPU_MOE_SPIN", "1") != "0" # ds_fp4 runs its input FP8 round-trip on the GPU when it can; tests turn this off to compare against the CPU round-trip. _DSFP4_GPU_PREQUANT = True # Activation ids must match ActKind in csrc/cpu_moe/common.h. Id 3 is the # clamped (up + 1) swiglu: "swigluoai" runs it in the generic GEMV epilogue, # "gpt_oss_swiglu" is the same math fused inside the mxfp4 kernel. _ACT_IDS = { "silu": 0, "swish": 0, "gelu": 1, "gelu_tanh": 2, "gelu_pytorch_tanh": 2, "gpt_oss_swiglu": 3, "swigluoai": 3, "swiglu_clamp": 4, } def compiled_extension_supports(activation: str) -> bool: """Whether the compiled ``_cpu_moe`` extension can serve ``activation`` through its generic epilogue. A stale prebuilt .so accepts newer act ids while silently computing the wrong math; the executor hard-errors on that, but the engine's auto offload->hybrid upgrade consults this first so a default boot degrades to offload instead of crashing after weight load.""" if activation not in _ACT_IDS: return False if _ACT_IDS[activation] < 3: return True try: from freetoken.kernel import _cpu_moe except ImportError: return False return _ACT_IDS[activation] <= getattr(_cpu_moe, "max_generic_act_id", lambda: 2)() def physical_core_cpus() -> list[int]: """One logical CPU per physical core, restricted to this process's affinity. MoE decode is memory-bandwidth-bound, so SMT siblings only contend for the same core's load ports without adding bandwidth. Picking one logical CPU per physical core (and pinning to it) gives the best, most stable bandwidth. Falls back to the full affinity set when sysfs topology is unavailable. """ try: allowed = sorted(os.sched_getaffinity(0)) except AttributeError: allowed = list(range(os.cpu_count() or 1)) reps: list[int] = [] seen: set[str] = set() for cpu in allowed: try: with open(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list") as f: key = f.read().strip() except OSError: reps.append(cpu) continue if key not in seen: seen.add(key) reps.append(cpu) return reps or allowed or [0] def _spin_wait_ok(core_ids: list[int], coord_core: int) -> bool: """Spin only if each pool thread owns a distinct CPU and two CPUs stay free for the main and CUDA callback threads.""" if not _SPIN_WAIT or not hasattr(os, "sched_getaffinity"): return False pool = list(core_ids) + ([coord_core] if coord_core >= 0 else []) if len(set(pool)) != len(pool): return False spare = set(os.sched_getaffinity(0)) - set(pool) return len(spare) >= 2 def resolve_threads_and_affinity(requested: int) -> tuple[int, list[int]]: """Return (num_threads, core_ids) for the worker pool. ``requested == 0`` -> one thread per physical core, pinned to it (best for the bandwidth-bound GEMV: SMT siblings only contend for a core's load ports and the spin-barrier degrades badly when oversubscribed). An explicit count is honored, spreading first across physical cores, then across the remaining logical CPUs (so distinct hardware threads are used before any core is doubled up). """ reps = physical_core_cpus() if requested and requested > 0: n = int(requested) try: allowed = sorted(os.sched_getaffinity(0)) except AttributeError: allowed = list(range(os.cpu_count() or 1)) # physical-core reps first, then the rest of the logical CPUs. order = reps + [c for c in allowed if c not in set(reps)] if not order: order = [0] core_ids = [order[i % len(order)] for i in range(n)] return n, core_ids return len(reps), list(reps) def _format_label(name: str, opts: tuple) -> str: """One format instance as the startup log names it, e.g. ``gguf(gate_up=IQ2_XXS, down=IQ4_NL)``.""" if name in ("gguf", "gguf_scaled"): from freetoken.layers.quantization.ggml_types import type_name gate_up, down = dict(opts)["gate_up_type"], dict(opts)["down_type"] return f"{name}(gate_up={type_name(gate_up)}, down={type_name(down)})" return name + (f"({', '.join(f'{k}={v}' for k, v in opts)})" if opts else "") class CpuMoeExecutor: """Decode-time CPU expert compute over an ``OffloadMoeCache``'s host banks (the formats ``_cpu_moe.weight_formats()`` lists — see ``_resolve_banks``).""" def __init__( self, cache, *, top_k: int, activation: str, apply_router_weight_on_input: bool, num_threads: int, max_tokens: int, device: torch.device, activation_dtype: torch.dtype = torch.bfloat16, graph_enabled: bool = True, swiglu_alpha: float = 1.702, swiglu_limit: float | None = None, layer_formats: list[tuple[str, dict[str, int]]] | None = None, ) -> None: from freetoken.kernel import _cpu_moe from freetoken.moe.legacy_format import canonical_role # layer_formats[l] = (format, options) of layer l; None runs every layer in cache.quant_format if layer_formats is None: layer_formats = [(cache.quant_format, {})] * int(cache.num_layers) assert len(layer_formats) == cache.num_layers, (len(layer_formats), cache.num_layers) keys = [(name, tuple(sorted(opts.items()))) for name, opts in layer_formats] specs = list(dict.fromkeys(keys)) names = sorted({name for name, _ in specs}) for f in names: if f not in _cpu_moe.weight_formats(): raise NotImplementedError( f"--moe-strategy cpu/hybrid computes experts on the CPU and supports " f"{sorted(_cpu_moe.weight_formats())} formats, but this checkpoint's experts are " f"{f!r}; use --moe-strategy offload (GPU-side dequant) instead." ) if f in ("gguf", "gguf_scaled"): from freetoken.kernel import ggml_cpu if (variant := ggml_cpu.select()) is None: raise NotImplementedError( "--moe-strategy cpu/hybrid cannot compute GGUF experts here: this build has no ggml-cpu kernels (x86-64 only); " "use --moe-strategy offload instead." ) _cpu_moe.set_ggml_cpu(variant.api) if activation not in _ACT_IDS: raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}") if apply_router_weight_on_input and (output_only := {"mxfp4_triton", "ds_fp4"} & set(names)): raise NotImplementedError(f"CPU MoE backend: {sorted(output_only)} experts apply the router weight on the output only") # ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without # error and silently computes the wrong activation in the generic # epilogue -- fail loudly with the rebuild instruction instead. (mxfp4 # handles its act inside the kernel and predates the marker.) if _ACT_IDS[activation] >= 3 and names != ["mxfp4_triton"]: supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)() if _ACT_IDS[activation] > supported: raise RuntimeError( f"the compiled _cpu_moe extension predates activation " f"{activation!r} (max generic act id {supported}); rebuild it " "with `python setup.py build_ext --inplace` (or reinstall the " "wheel) before serving this model on the cpu/hybrid backend." ) self.num_layers = int(cache.num_layers) self.num_experts = int(cache.num_experts) self.top_k = int(top_k) self.quant_format = ",".join(names) self.device = device self.max_tokens = int(max_tokens) self.apply_router_weight_on_input = bool(apply_router_weight_on_input) # The per-layer tensors and their pointer tables must outlive the executor # (C++ holds raw addresses into both). self._banks: list[torch.Tensor] = [] by_role = {canonical_role(name): per_layer for name, per_layer in cache.bank_sources.items()} # every format's tables address every layer, but a format reads only its own layers' rows formats, shapes = [], set() for name, opts in specs: banks, options, shape = self._resolve_banks(by_role, name, keys.index((name, opts)), dict(opts)) formats.append((name, banks, {**dict(opts), **options})) shapes.add(shape) if len(shapes) != 1: raise NotImplementedError(f"CPU MoE formats disagree on the expert shape (hidden, intermediate): {sorted(shapes)}") (self.H, self.I), = shapes layer_counts = {spec: keys.count(spec) for spec in specs} # ds_fp4: the reference FP8 activation round-trip is a scalar per-element chain # that the C++ side runs single-threaded on the CUDA host-callback thread -- # straight on the decode critical path (~0.3ms/layer at H=4096, every worker and # the GPU waiting on it). When a GPU is present we run the numerically identical # round-trip as a captured GPU kernel BEFORE the D2H (see decode_submit) and tell # the C++ side to skip its own. Measured on DeepSeek-V4-Flash bs=1 decode: # 12.85 -> 15.65 tok/s, output bit-identical (tests/moe/test_dsfp4_prequant.py). # the W4A8 activation round-trip block follows the checkpoint's fp8 block (128 on V4, 32 on V4.1) blocks = {options.get("act_block") for name, _, options in formats if name == "ds_fp4"} assert None not in blocks, "ds_fp4 experts need the act_block option, the checkpoint's fp8 activation quant block" # the GPU round-trip rewrites the whole input, so only when every layer is ds_fp4 with one block self._gpu_prequant = names == ["ds_fp4"] and len(blocks) == 1 and device.type == "cuda" and _DSFP4_GPU_PREQUANT self._act_block = next(iter(blocks)) if self._gpu_prequant else None for name, _, options in formats: if name == "ds_fp4": options["input_prequant"] = int(self._gpu_prequant) if self._gpu_prequant: logger.info_rank0( "ds_fp4: input FP8 round-trip moved to the GPU " "(bit-identical grid; the CPU-side scalar round-trip is skipped)" ) # Decide the flag handshake up front (env + device + a functional stream-memop # probe): its coordinator needs a core of its own, which the auto thread sizing # below reserves (a coordinator time-slicing against the GEMV workers measurably # destabilizes throughput on fully-subscribed boxes). self._flag_sync = _FLAG_SYNC and device.type == "cuda" if self._flag_sync and not _cpu_moe.memops_probe(torch.cuda.current_stream().cuda_stream): logger.info_rank0( "cpu-moe flag handshake unavailable: stream memory operations failed the " "startup probe; using the cudaLaunchHostFunc sync" ) self._flag_sync = False # ROCm 7.14 returns wrong results when it replays host-function nodes, so there only the doorbell is graph-safe. self._graph_capture_safe = self._flag_sync or _cpu_moe.host_func_capture_safe() nthreads, core_ids = resolve_threads_and_affinity(num_threads) coord_core = -1 if self._flag_sync and num_threads == 0 and nthreads > 2: # Auto sizing: give the coordinator the last physical core instead of # oversubscribing (workers drop from N to N-1). coord_core = core_ids[-1] nthreads -= 1 core_ids = core_ids[:-1] self._coord_core = coord_core alloc_io = ( (lambda *shape, dtype: torch.empty(*shape, dtype=dtype)) if device.type == "cpu" else alloc_pinned_tensor ) self._io = { "x": alloc_io(self.max_tokens, self.H, dtype=torch.bfloat16), "ids": alloc_io(self.max_tokens, self.top_k, dtype=torch.int32), "w": alloc_io(self.max_tokens, self.top_k, dtype=torch.float32), "y": alloc_io(self.max_tokens, self.H, dtype=torch.bfloat16), } self._ext = _cpu_moe.CpuMoeExecutor( num_threads=nthreads, num_layers=self.num_layers, num_experts=self.num_experts, top_k=self.top_k, hidden_size=self.H, inter_size=self.I, max_tokens=self.max_tokens, activation_id=_ACT_IDS[activation], apply_router_weight_on_input=1 if apply_router_weight_on_input else 0, formats=formats, layer_formats=[specs.index(k) for k in keys], swiglu_alpha=float(swiglu_alpha), swiglu_limit=float(swiglu_limit) if swiglu_limit is not None else float("inf"), core_ids=core_ids, x_ptr=self._io["x"].data_ptr(), ids_ptr=self._io["ids"].data_ptr(), w_ptr=self._io["w"].data_ptr(), y_ptr=self._io["y"].data_ptr(), ) self.num_threads = nthreads self.core_ids = core_ids self.isa = self._ext.isa_name() self.spin_wait = _spin_wait_ok(core_ids, coord_core) self._ext.set_spin_wait(self.spin_wait) spare = len(physical_core_cpus()) - nthreads - (1 if coord_core >= 0 else 0) - 1 clamp = max(1, min(torch.get_num_threads(), spare)) if clamp < torch.get_num_threads(): logger.info_rank0( f"torch intra-op threads: {torch.get_num_threads()} -> {clamp} " "(cores reserved for the pinned CPU MoE pool)" ) torch.set_num_threads(clamp) # Flag-based handshake: the extension owns the ready/done words (gpu_sync.h) and a # persistent CPU coordinator that polls the doorbell, runs the task it names on the # pool, and sets done for the GPU. The WATCHDOG thread marks the executor unhealthy # when a doorbell stays unanswered (dead coordinator): it poisons done to unblock the # stream and raise_if_unhealthy() turns the step into a loud error instead of silent # stale output (flag_sync itself was decided above, before thread sizing). self._unhealthy = False self._kernel_io = False if self._flag_sync: self._ext.start_flag_coordinator(self._coord_core) reason = _cpu_moe.kernel_io_probe(torch.cuda.current_stream().cuda_stream) if reason: logger.warning_rank0(f"cpu-moe kernel IO unavailable ({reason}); using D2H/H2D copies around the doorbell") else: self._ext.enable_kernel_io() self._kernel_io = True self._watchdog_stop = False # The thread target holds a WEAKREF and re-derefs it each tick: a bound # method would strong-reference the executor forever (the loop never ends # on its own), pinning the C++ worker pool and the pinned banks against GC # in build-many-engines scenarios. NB: the stop flag / weakref death is # observed only between 2 s sleeps, so teardown of the THREAD can lag up to # ~2 s -- it is a daemon, so neither GC of the executor (weakref breaks the # cycle) nor process exit waits on it. self._watchdog = threading.Thread( target=_watchdog_main, args=(weakref.ref(self),), name="cpu-moe-flag-watchdog", daemon=True, ) self._watchdog.start() self._graph_safe_dtypes = {activation_dtype} if torch.version.hip is not None: self._graph_capture_safe = self._flag_sync if self._flag_sync: reason = self._probe_graph_io(activation_dtype, capture_graph=graph_enabled) if reason and self._kernel_io and activation_dtype == torch.bfloat16: logger.warning_rank0(f"cpu-moe kernel IO probe failed ({reason}); checking event + memcpy IO") # Python selects the copy path; the native KernelIo allocation lives until executor teardown. self._kernel_io = False reason = self._probe_graph_io(activation_dtype, capture_graph=graph_enabled) if reason: if not self._eager_io_safe: raise RuntimeError(f"CPU MoE eager IO probe failed for {activation_dtype}: {reason}") logger.warning_rank0(f"cpu-moe graph IO probe failed for {activation_dtype} ({reason}); using eager decode") self._graph_capture_safe = graph_enabled and not reason fmt_label = "; ".join(f"{_format_label(name, opts)}: {n} layers" for (name, opts), n in layer_counts.items()) logger.info_rank0( f"CPU MoE executor ready: threads={nthreads} (pinned to cores " f"{core_ids[0]}..{core_ids[-1]}) isa={self.isa} " f"fmt={fmt_label} " f"H={self.H} I={self.I} experts={self.num_experts} layers={self.num_layers} " f"top_k={self.top_k} act={activation} max_tokens={self.max_tokens} " f"spin_wait={self.spin_wait} " f"handshake={'kernel' if self._kernel_io else 'event-memop' if self._flag_sync and torch.version.hip is not None else 'memop' if self._flag_sync else 'host-func'}" ) def graph_capture_safe(self, dtype: torch.dtype) -> bool: return self._graph_capture_safe and ( torch.version.hip is None or dtype in self._graph_safe_dtypes ) def _probe_graph_io(self, dtype: torch.dtype, *, capture_graph: bool = True) -> str: """Qualify IO against synchronous CPU execution, including hybrid accumulation.""" self._eager_io_safe = False stream = torch.cuda.Stream() stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(stream): bs = min(2, self.max_tokens) hidden = torch.empty(3, bs, self.H, device="cuda", dtype=dtype) ids = torch.empty(3, bs, self.top_k, device="cuda", dtype=torch.int32) weights = torch.empty(3, bs, self.top_k, device="cuda", dtype=torch.float32) add = torch.empty_like(hidden) base = torch.arange(hidden.numel(), dtype=torch.float32).reshape(hidden.shape) routes = torch.arange(ids.numel()).reshape(ids.shape) def inputs(step): hidden.copy_(((base + step * 7) % 31 - 15) / 64) cpu_ids = ((routes + step) % self.num_experts).int() cpu_ids[step % 3].fill_(-1) cpu_ids[(step + 1) % 3, :, -1] = -1 ids.copy_(cpu_ids) weights.copy_(((routes + step) % 7 + 1).float() / (7 * self.top_k)) add.copy_(((base + step * 3) % 17 - 8) / 32) def forward(): outputs = [] for i in range(3): pending = self.decode_submit(i % self.num_layers, hidden[i], weights[i], ids[i]) outputs.append(self.decode_sync(pending, add_to=add[i].clone() if i == 1 else None)) return outputs def reference(): # No asynchronous IO or notification: both paths can otherwise share the same bug. x = hidden if self._gpu_prequant: from freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_roundtrip x = torch.stack([act_quant_fp8_roundtrip(row, block=self._act_block) for row in hidden]) host_x, host_ids, host_w, host_add = x.cpu(), ids.cpu(), weights.cpu(), add.cpu() expected = [] for i in range(3): self._io["x"][:bs].copy_(host_x[i]) self._io["ids"][:bs].copy_(host_ids[i]) self._io["w"][:bs].copy_(host_w[i]) self._ext.run(i % self.num_layers, bs) # Match the IO rounding boundary before hybrid addition; the BF16 combine kernel # also adds already-rounded BF16 operands in FP32, then rounds back to BF16. out = self._io["y"][:bs].to(dtype).clone() expected.append(out + host_add[i] if i == 1 else out) return expected def matches(actual, expected): return all(torch.equal(got.cpu(), want) for got, want in zip(actual, expected)) inputs(0) eager = forward() stream.synchronize() if not matches(eager, reference()): return "eager output differs from synchronous CPU reference" self._eager_io_safe = True if not capture_graph: return "" graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, stream=stream): captured = forward() for step in range(1, 4): inputs(step) graph.replay() stream.synchronize() self.raise_if_unhealthy() # Read the reference only after replay, so it cannot prefill stale graph IO. if not matches(captured, reference()): return "graph output differs from synchronous CPU reference" return "" def _make_table(self, layers: list[torch.Tensor]) -> torch.Tensor: """Build a CPU int64 tensor of per-layer base addresses for one bank. ``layers`` is one ``[num_experts, ...]`` tensor per layer (the per-layer host bank contract). The C++ side stores this table's pointer and indexes ``tbl[layer_id]`` at call time; both the table and the layer tensors are kept on ``self._banks`` (GC guard) so the raw pointers stay valid for the executor's lifetime. """ assert len(layers) == self.num_layers, (len(layers), self.num_layers) table = torch.tensor([t.data_ptr() for t in layers], dtype=torch.int64) self._banks.append(table) self._banks.extend(layers) return table def _tables(self, **roles: list[torch.Tensor]) -> dict[str, int]: """Per-layer table addresses keyed by canonical bank role.""" return {role: self._make_table(layers).data_ptr() for role, layers in roles.items()} def _resolve_fp8_block_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]: """Block-FP8 schema: row-major e4m3 + one bf16 scale per 128x128 block; the scale banks pad their last dim, so C++ gets the real row stride.""" gu, gus = banks["gate_up"], banks["gate_up_scale"] dn, dns = banks["down"], banks["down_scale"] assert gu[rep].element_size() == 1 and dn[rep].element_size() == 1, (gu[rep].dtype, dn[rep].dtype) assert gus[rep].dtype == torch.bfloat16 and dns[rep].dtype == torch.bfloat16, (gus[rep].dtype, dns[rep].dtype) I = int(gu[rep].shape[1] // 2) H = int(gu[rep].shape[2]) assert gu[rep].shape[1] == 2 * I assert tuple(dn[rep].shape[1:]) == (H, I), (dn[rep].shape, H, I) assert gus[rep].shape[1] == (2 * I + 127) // 128 and gus[rep].shape[2] >= (H + 127) // 128, (gus[rep].shape, H, I) assert dns[rep].shape[1] == (H + 127) // 128 and dns[rep].shape[2] >= (I + 127) // 128, (dns[rep].shape, H, I) banks = self._tables(gate_up=gu, down=dn, gate_up_scale=gus, down_scale=dns) options = dict(gu_scale_stride=int(gus[rep].stride(1)), dn_scale_stride=int(dns[rep].stride(1))) return banks, options, (H, I) def _resolve_banks(self, banks: dict, fmt: str, rep: int = 0, opts: dict | None = None) -> tuple[dict, dict, tuple[int, int]]: """Return (banks, options, (H, I)) for the C++ ctor: banks maps each canonical role the format reads to its per-layer table's address, options holds its integer settings. ``banks[name]`` is a list of ``num_layers`` ``[num_experts, ...]`` tensors (the per-layer host bank contract); shapes are read from the first layer so per-partition (TP) sizes are exact. Every bank address is a per-layer table (see ``_make_table``), not a single bank's -- the C++ side resolves ``tbl[layer_id]`` per task. """ if fmt in ("gguf", "gguf_scaled"): return self._resolve_gguf_banks(banks, rep, int(opts["gate_up_type"]), int(opts["down_type"]), scaled=fmt == "gguf_scaled") if fmt == "bf16": gate_up = banks["gate_up"] down = banks["down"] if gate_up[rep].dtype != torch.bfloat16 or down[rep].dtype != torch.bfloat16: raise NotImplementedError( f"bf16 CPU MoE requires bf16 banks, got {gate_up[rep].dtype}/{down[rep].dtype}" ) H = int(gate_up[rep].shape[2]) I = int(gate_up[rep].shape[1] // 2) assert gate_up[rep].shape[1] == 2 * I assert tuple(down[rep].shape[1:]) == (H, I), (down[rep].shape, H, I) banks = self._tables(gate_up=gate_up, down=down) return banks, {}, (H, I) if fmt == "mxfp4_triton": return self._resolve_mxfp4_banks(banks, rep) if fmt == "ds_fp4": return self._resolve_dsfp4_banks(banks, rep) if fmt == "fp8_block": return self._resolve_fp8_block_banks(banks, rep) # nvfp4: packed e2m1 (2/byte) + fp8-e4m3 per-16 block scales + fp32 row globals. gup, gus, gug = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_global"] dnp, dns, dng = banks["down"], banks["down_scale"], banks["down_global"] assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype) assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte" assert gug[rep].dtype == torch.float32 and dng[rep].dtype == torch.float32, (gug[rep].dtype, dng[rep].dtype) I = int(gup[rep].shape[1] // 2) H = int(gup[rep].shape[2] * 2) assert gup[rep].shape[1] == 2 * I assert H % 16 == 0 and I % 16 == 0, (H, I) assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I) assert tuple(gus[rep].shape[1:]) == (2 * I, H // 16), (gus[rep].shape, I, H) assert tuple(dns[rep].shape[1:]) == (H, I // 16), (dns[rep].shape, H, I) assert tuple(gug[rep].shape[1:]) == (2 * I,) and tuple(dng[rep].shape[1:]) == (H,) banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, gate_up_global=gug, down_scale=dns, down_global=dng) return banks, {}, (H, I) def _resolve_gguf_banks(self, banks: dict, rep: int, gate_up_type: int, down_type: int, scaled: bool = False) -> tuple[dict, dict, tuple[int, int]]: """GGUF rows read in place: gate_up [E, 2I, row_bytes(H)] (gate rows, then up rows) and down [E, H, row_bytes(I)], one ggml type each; float types never reach here (cpu_spec sends them to the bf16 format, whose banks load bf16). ``scaled`` adds global [E, 4] fp32, each expert's (gate, up, down, 1) NVFP4 weight_scale_2.""" from freetoken.layers.quantization.ggml_types import float_dtype, row_bytes, type_name gate_up, down = banks["gate_up"], banks["down"] H, I = int(down[rep].shape[1]), int(gate_up[rep].shape[1] // 2) for role, t, ggml_type, k in (("gate_up", gate_up[rep], gate_up_type, H), ("down", down[rep], down_type, I)): dtype = float_dtype(ggml_type) if float_dtype(ggml_type) is not None else torch.uint8 if t.dtype != dtype or t.shape[-1] * t.element_size() != row_bytes(ggml_type, k): raise ValueError(f"MoE layer {rep} {role} bank {tuple(t.shape)} {t.dtype} is not {type_name(ggml_type)} rows of {k} values") types = {"gate_up_type": gate_up_type, "down_type": down_type} if not scaled: return self._tables(gate_up=gate_up, down=down), types, (H, I) scales = banks["global"] if scales[rep].dtype != torch.float32 or tuple(scales[rep].shape) != (gate_up[rep].shape[0], 4) or not scales[rep].is_contiguous(): raise ValueError(f"MoE layer {rep} global bank {tuple(scales[rep].shape)} {scales[rep].dtype} is not contiguous [experts, 4] fp32") return self._tables(gate_up=gate_up, down=down, **{"global": scales}), types, (H, I) def _resolve_mxfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]: """gpt-oss mxfp4 ``mxfp4_triton`` schema: transposed split-K blocks/scales (N innermost) + per-output-row biases. The C++ kernel streams K and accumulates a contiguous N-block, so the GPU-tiled layout is read in place (no repack, no extra host memory). Block scales are e8m0 (1 byte / 32 K).""" gub, gus, gob = banks["gate_up"], banks["gate_up_scale"], banks["gate_up_bias"] dnb, dns, dob = banks["down"], banks["down_scale"], banks["down_bias"] assert gub[rep].dtype == torch.uint8 and dnb[rep].dtype == torch.uint8, (gub[rep].dtype, dnb[rep].dtype) assert gus[rep].dtype == torch.uint8 and dns[rep].dtype == torch.uint8, (gus[rep].dtype, dns[rep].dtype) assert gob[rep].dtype == torch.bfloat16 and dob[rep].dtype == torch.bfloat16, (gob[rep].dtype, dob[rep].dtype) # gate_up_blocks [E, H//2, 2I]; down_blocks [E, I//2, H] H = int(gub[rep].shape[1] * 2) I = int(gub[rep].shape[2] // 2) assert gub[rep].shape[2] == 2 * I assert H % 32 == 0 and I % 32 == 0, (H, I) assert tuple(dnb[rep].shape[1:]) == (I // 2, H), (dnb[rep].shape, H, I) assert tuple(gus[rep].shape[1:]) == (H // 32, 2 * I), (gus[rep].shape, H, I) assert tuple(dns[rep].shape[1:]) == (I // 32, H), (dns[rep].shape, H, I) assert tuple(gob[rep].shape[1:]) == (2 * I,) and tuple(dob[rep].shape[1:]) == (H,) banks = self._tables(gate_up=gub, down=dnb, gate_up_scale=gus, down_scale=dns, gate_up_bias=gob, down_bias=dob) return banks, {}, (H, I) def _resolve_dsfp4_banks(self, banks: dict, rep: int = 0) -> tuple[dict, dict, tuple[int, int]]: """DeepSeek-V4 ``ds_fp4`` schema: row-major e2m1 (2/byte) + e8m0 per-32 block scales, no global, no bias. Layout matches nvfp4 (K contiguous per output row), so the C++ GEMV reads it in place. The kernel additionally FP8-round-trips the activations (the checkpoint's fp8 block) to match the DeepSeek W4A8 reference.""" gup, gus = banks["gate_up"], banks["gate_up_scale"] dnp, dns = banks["down"], banks["down_scale"] assert gup[rep].dtype == torch.uint8 and dnp[rep].dtype == torch.uint8, (gup[rep].dtype, dnp[rep].dtype) assert gus[rep].element_size() == 1 and dns[rep].element_size() == 1, "block scales must be 1 byte" I = int(gup[rep].shape[1] // 2) H = int(gup[rep].shape[2] * 2) assert gup[rep].shape[1] == 2 * I assert H % 32 == 0 and I % 32 == 0, (H, I) # e8m0 scales per 32 along K assert tuple(dnp[rep].shape[1:]) == (H, I // 2), (dnp[rep].shape, H, I) assert tuple(gus[rep].shape[1:]) == (2 * I, H // 32), (gus[rep].shape, I, H) assert tuple(dns[rep].shape[1:]) == (H, I // 32), (dns[rep].shape, H, I) banks = self._tables(gate_up=gup, down=dnp, gate_up_scale=gus, down_scale=dns) return banks, {}, (H, I) def decode( self, layer_id: int, hidden_states: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, ) -> torch.Tensor: """One MoE layer of decode on the CPU. Returns a GPU [bs, H] tensor. All ops go on the current CUDA stream so the whole thing is captured into the active CUDA graph (the two host nodes carry the data dependency on the pinned buffers, which hold this step's real routing on replay).""" pending = self.decode_submit(layer_id, hidden_states, topk_weights, topk_ids) return self.decode_sync(pending) def decode_submit( self, layer_id: int, hidden_states: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, ) -> tuple: """Issue the D2H copies + the CPU-pool submit host node (or the publish kernel), then return without waiting. Lets a caller (the hybrid backend) enqueue GPU work between this and :meth:`decode_sync` so the CPU compute overlaps the GPU GEMM / PCIe fetch. ``topk_ids`` may carry ``-1`` entries (the C++ kernel skips them), so the CPU computes only the routes assigned to it. Returns an opaque handle to pass to :meth:`decode_sync`. The output tensor is allocated here so it stays live (and distinct from the interleaved GPU work) across the overlap window.""" bs = hidden_states.shape[0] if bs > self.max_tokens: raise ValueError(f"CPU MoE decode batch {bs} exceeds the executor's max_tokens {self.max_tokens}") if not self.graph_capture_safe(hidden_states.dtype) and torch.cuda.is_current_stream_capturing(): raise RuntimeError( "CPU MoE decode cannot be captured into a CUDA graph here: the IO/handshake " "path is not replay-safe for this runtime and dtype; run with --cuda-graph-max-bs 0" ) if self._gpu_prequant: # DSV4: apply the reference FP8 round-trip on the GPU (the same kernel the # GPU W4A8 path uses -> bit-identical grid) so the CPU side reads # pre-quantized activations and skips its serial scalar pass. from freetoken.kernel.triton.dsv4.fp8_linear import act_quant_fp8_roundtrip hidden_states = act_quant_fp8_roundtrip(hidden_states, block=self._act_block) stream = torch.cuda.current_stream().cuda_stream if self._kernel_io and hidden_states.dtype == torch.bfloat16: x = hidden_states.contiguous() ids = topk_ids.to(torch.int32).contiguous() w = topk_weights.to(torch.float32).contiguous() assert x.shape == (bs, self.H) and ids.shape == w.shape == (bs, self.top_k), (x.shape, ids.shape, w.shape) self._ext.flag_publish(stream, x.data_ptr(), ids.data_ptr(), w.data_ptr(), layer_id, bs) return (bs, torch.empty_like(x), True) # D2H: ship this step's activations + routing to pinned host memory. io = self._io io["x"][:bs].copy_(hidden_states, non_blocking=True) io["ids"][:bs].copy_(topk_ids.to(torch.int32), non_blocking=True) io["w"][:bs].copy_(topk_weights.to(torch.float32), non_blocking=True) out = torch.empty_like(hidden_states) if self._flag_sync: # Front-end memops: reset done, then ring the doorbell with (layer, bs). # HIP records the input-publication event before these writes. self._ext.flag_signal(stream, layer_id, bs) else: self._ext.submit_with_cuda_stream(stream, layer_id, bs) return (bs, out, False) def decode_sync(self, pending: tuple, add_to: torch.Tensor | None = None) -> torch.Tensor: """Issue the CPU-pool sync + the H2D result copy for a prior :meth:`decode_submit`, and return the GPU output tensor. With flag-sync the wait is a front-end stream memop on done (set by the CPU coordinator); otherwise a cudaLaunchHostFunc. With ``add_to``, return ``add_to`` + the CPU output; the kernel path adds in place into ``add_to``.""" bs, out, kernel_io = pending stream = torch.cuda.current_stream().cuda_stream if self._flag_sync: # Front-end WAIT(done >= 1): blocks this stream's later nodes without # occupying an SM, so GPU utilization stays truthful during the CPU window. self._ext.flag_wait(stream) else: self._ext.sync_with_cuda_stream(stream) if kernel_io: if add_to is not None and add_to.dtype == torch.bfloat16 and add_to.is_contiguous() and add_to.shape == out.shape: self._ext.flag_combine(stream, add_to.data_ptr(), bs, True) return add_to self._ext.flag_combine(stream, out.data_ptr(), bs, False) return out if add_to is None else add_to + out out.copy_(self._io["y"][:bs], non_blocking=True) return out if add_to is None else add_to + out def _watchdog_tick(self, suspect: dict) -> None: """One watchdog sampling round (called every 2 s by ``_watchdog_main``). The doorbell is only declared dead when THREE things hold across >=10 s: it is still pending (ready set, done==0), it was already pending when first suspected, and the coordinator has served NOTHING since (flag_served unchanged). The served-count criterion kills the false-positive window: two point samples can land on (independent, us-scale) pending windows under heavy external load, but a coordinator that made progress in between is alive by definition. ``suspect`` holds (first_seen, served_at_first_sight) and persists across ticks.""" if not self._ext.flag_pending(): suspect.clear() return now = time.monotonic() served = self._ext.flag_served() if not suspect or suspect["served"] != served: suspect.update(first_seen=now, served=served) # new suspect, or alive-but-loaded: rearm return if now - suspect["first_seen"] < 10.0: return logger.error( "cpu-moe flag watchdog: doorbell unanswered for >10s with no coordinator " "progress (wedged/dead); poisoning done and failing the next step" ) self._unhealthy = True self._ext.poison_flag() # after the mark: unblock the stream into a checked failure suspect.clear() def raise_if_unhealthy(self) -> None: """Raise if the flag watchdog fired (a doorbell stayed unanswered because the coordinator never responded). Called by the engine once per forward -- a single flag read -- so a dead coordinator surfaces as a loud error on the next step instead of silently shipping stale expert outputs.""" if self._unhealthy: raise RuntimeError( "CPU MoE flag-handshake watchdog fired: a decode step's doorbell was " "never answered by the coordinator thread (its outputs cannot be " "trusted). This indicates a wedged/killed coordinator; restart the " "engine, or set FREETOKEN_CPU_MOE_FLAG_SYNC=0 to use the " "cudaLaunchHostFunc sync." ) def _watchdog_main(executor_ref) -> None: """Watchdog daemon body: weakref-deref per tick so the thread never keeps a dead executor alive (see the start site in ``CpuMoeExecutor.__init__``).""" suspect: dict = {} while True: time.sleep(2.0) executor = executor_ref() if executor is None or executor._watchdog_stop: return try: executor._watchdog_tick(suspect) finally: del executor # drop the strong ref before the next sleep