mirror of
https://github.com/FlashML-org/FreeToken.git
synced 2026-11-21 01:52:30 +08:00
* feat(spec): hand the pending token to Proposer.propose and return a SpecProposal * feat(engine): Sampler.probs and sparse-q rejection * feat(spec): let a proposer declare slot states riding the GDN slots * feat(models): export layer outputs to the feature sink in the qwen3_5 and qwen4_exp families * feat(attention): record the source slot of each tracked request * feat(kernel): DFlash2 draft kernels * feat(models): DFlash2 draft family * feat(spec): DFlashWindow * feat(spec): DFlashProposer * docs(cli): add dflash to the speculative decoding section * refactor(engine): share the sampling backend choice and drop the extra renorm in Sampler.probs * fix(models): build the DFlash rope table uncached and refuse draft scalings it does not apply * refactor(spec): share the draft path check between the MTP and DFlash proposers * test(e2e): compare rebuilt outputs exactly only without hybrid MoE * refactor(spec): keep proposer slot states on EngineConfig instead of replacing model_config * perf(spec): compute the verify distribution once for sparse-q rejection * perf(models): let the feature sink add the residual on the rows it keeps * perf(models): share the DFlash draft K/V weights with the ring projection * fix(kernel): keep the dflash selector off zero-probability candidates * test(spec): drop the DFlash tests that the draft forward and Monte Carlo tests already cover * fix(spec): default and cap DFlash steps at the draft's block_size - 1 * fix(models): reject DFlash draft configs the draft forward cannot serve * test(e2e): read the MoE strategy from the rebuild test's extra args * docs(spec): say that slot-state owners write their own prefill track snapshot * test(models): move the DFlash draft parity tests under tests/models/dflash * refactor(spec): drop the no-op verify handler of the DFlash window * refactor(models): fold the DFlash candidate top-k into propose * refactor(spec): declare the draft step limits through Proposer.num_steps only * refactor(kernel): make the DFlash ring attention stateless and keep its workspace on the ring * refactor(models): name the DFlash draft submodules like the other model families * refactor(spec): move SpecProposal next to DraftContext in spec/proposer.py * refactor(spec): hand the verify the draft distribution over the whole vocabulary * refactor(spec): call the DFlashRing object dflash_ring and its length ring_len * refactor(models): split the DFlash draft's propose into compute_candidates and sample_path * refactor(spec): keep the draft distribution on its support in SpecProposal * fix(spec): drop the step's proposal once the verify has read it * refactor(spec): name the DFlash feature sinks DFlashPrefillSink and DFlashVerifySink * refactor(models): read the declared Batch.features directly instead of through getattr * test(dflash): check the window rebuild inside the block forward test * test(spec): share the plain run of the verify contract and run the DFlash draft once per family * test(kvcache): drop the window cases from the pool tests and check a layerless state's bytes * test(spec): check the verify probs of each row against a torch top-k and top-p reference * test(e2e): add an accept-len floor and a prefill chunk knob to the AIME gate * refactor(scheduler): let callers take the speculative counts from the status reporter * test(spec): keep the shared plain runs of the verify contract in their own temp dirs * test(models): give the ricursive test's stand-in batch a features field
2236 lines
117 KiB
Python
2236 lines
117 KiB
Python
from __future__ import annotations
|
|
|
|
import collections
|
|
import contextlib
|
|
import errno
|
|
import gc
|
|
import math
|
|
import os
|
|
from datetime import timedelta
|
|
from typing import Any, Dict, Iterable, NamedTuple, Tuple
|
|
|
|
import torch
|
|
from freetoken.attention import AttnType, attention_backend_info, create_attention_backend
|
|
from freetoken.attention.linear import LinearAttnBackend, gdn_slot
|
|
from freetoken.core import Batch, Context, Req, set_global_ctx
|
|
from freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info
|
|
from freetoken.gpu_select import gpu_identity
|
|
from freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpoint
|
|
from freetoken.layers import OffloadMoELayer, VocabParallelEmbedding, iter_moe_layers, iter_ops, set_rope_device
|
|
from freetoken.layers.quantization import LayerKind, QuantBackend, QuantKind, finalize_quant, set_quant_backend
|
|
from freetoken.moe.offload_cache import iter_offload_moe_layers
|
|
from freetoken.mm.config import ENCODER_SECTIONS
|
|
from freetoken.models import create_model, load_weight
|
|
from freetoken.models.weight import ftw_lacks_vision
|
|
from freetoken.moe import is_offload_moe_strategy
|
|
from freetoken.moe.expert_banks import attach_resident_banks, load_expert_banks, pin_probe_enabled
|
|
from freetoken.moe.host_banks import PinFailed
|
|
from freetoken.moe.locked_fetch import attach_locked_fetch, check_locked_fetch_config, locked_pool_tags, settle_banks, staging_reserve
|
|
from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cache, bank_layouts
|
|
from freetoken.moe.pool_policy import group_layers, make_pools, pool_of_layer, resolve_profile, split_slots
|
|
from freetoken.utils import align_ceil, init_logger, is_sm90_family, is_sm100_family, mem_GB, torch_dtype
|
|
|
|
from .config import EngineConfig
|
|
from .graph import GraphRunner, _determine_cuda_graph_bs, get_free_memory
|
|
from .sample import BatchSamplingArgs, Sampler
|
|
from .spec_protocol import SpecRuntime
|
|
from freetoken.kvcache import create_kv_pool, resolve_pool_class
|
|
from freetoken.kvcache.base import CacheRebuildRejected
|
|
from freetoken.kvcache.cache_status import _supports_swa_ratio
|
|
from freetoken.kvcache.linear_state_pool import (
|
|
_linear_pool_min_slots, _linear_pool_num_slots, state_pool_bytes,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
def _disable_unsafe_cpu_moe_graphs(config: EngineConfig, executor, dtype, tp_cpu_group) -> None:
|
|
safe = torch.tensor([executor.graph_capture_safe(dtype)], dtype=torch.int32)
|
|
if config.tp_info.size > 1:
|
|
torch.distributed.all_reduce(safe, op=torch.distributed.ReduceOp.MIN, group=tp_cpu_group)
|
|
if not safe.item():
|
|
logger.warning_rank0("CPU MoE IO/handshake is not graph-safe on every TP rank for this dtype; disabling decode graphs")
|
|
# Clear explicit sizes too; GraphRunner gives them priority over max_bs, including on rebuild.
|
|
object.__setattr__(config, "cuda_graph_bs", [])
|
|
object.__setattr__(config, "cuda_graph_max_bs", 0)
|
|
|
|
|
|
def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) -> None:
|
|
"""Clamp the decode batch so every GPU-decoded layer's routed experts fit in its pool."""
|
|
num_experts = config.model_config.num_experts
|
|
small = [s for s in pool_sizes if s < num_experts]
|
|
if not small:
|
|
return
|
|
# a decode step routes every row of every request, decode_rows_per_req rows each
|
|
limit = min(small) // (config.model_config.num_experts_per_tok * config.decode_rows_per_req)
|
|
if config.max_running_req > limit:
|
|
logger.warning_rank0(
|
|
f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "
|
|
f"{config.max_running_req} -> {limit} so a decode step fits in it"
|
|
)
|
|
object.__setattr__(config, "max_running_req", limit)
|
|
if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > limit:
|
|
object.__setattr__(config, "cuda_graph_max_bs", limit)
|
|
if config.cuda_graph_bs is not None:
|
|
object.__setattr__(config, "cuda_graph_bs", [bs for bs in config.cuda_graph_bs if bs <= limit])
|
|
|
|
|
|
def _require_offload_cache_size(cache_size: int, num_experts: int) -> None:
|
|
"""The offload MoE cache needs at least one slot per expert per layer. A too-small size
|
|
(e.g. a bare offload run with moe_cache_size unset and auto disabled) must fail loudly."""
|
|
if cache_size < num_experts:
|
|
raise ValueError(
|
|
f"moe_cache_size={cache_size} is too small: need at least num_experts={num_experts} "
|
|
f"slots. Pass --moe-cache-size/--moe-cache-rate, or use --moe-cache-auto "
|
|
f"(the default for offload/hybrid backends when no cache-sizing flag is given; "
|
|
f"--moe-strategy cpu always sizes its own fixed two-layer buffer and ignores "
|
|
f"cache-sizing flags)."
|
|
)
|
|
|
|
|
|
def _flashinfer_available() -> bool:
|
|
from freetoken.kernel.backend import is_flashinfer_installed
|
|
|
|
return is_flashinfer_installed()
|
|
|
|
|
|
def _sgl_flash_attn_available() -> bool:
|
|
from freetoken.kernel.backend import is_rocm
|
|
|
|
if is_rocm():
|
|
return False
|
|
try:
|
|
from sgl_kernel.flash_attn import flash_attn_with_kvcache # noqa: F401
|
|
except Exception as exc:
|
|
detail = next((line.strip() for line in str(exc).splitlines() if line.strip()), "")
|
|
logger.warning_rank0(
|
|
"sgl_kernel.flash_attn is unavailable; auto attention backend falls back to fi "
|
|
f"({type(exc).__name__}: {detail})"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def _startup_kv_budget(memory_ratio: float, init_free_memory: int, new_free_memory: int) -> int:
|
|
"""Bytes available to the KV pool at startup: ratio-scaled pre-load free memory minus
|
|
what the resident model consumed. Kept as a pure function so the composition with the
|
|
pool families' ``solve_num_pages`` stays CPU-testable."""
|
|
return int(memory_ratio * init_free_memory) - (init_free_memory - new_free_memory)
|
|
|
|
|
|
def _page_table_width(max_seq_len: int, page_size: int) -> int:
|
|
"""Column count for the page table. ``_write_page_table`` writes WHOLE trailing pages, so the
|
|
highest column touched is ``align_ceil(max_seq_len, page_size) - 1`` -- which the 32-alignment
|
|
alone does not cover once page_size > 32 (an unaligned --max-seq-len-override on DSV4's P=128
|
|
or trtllm's forced 64 would index past the row)."""
|
|
return align_ceil(align_ceil(max_seq_len, page_size), 32)
|
|
|
|
|
|
def _seq_limits(config, num_tokens: int) -> tuple[int, int]:
|
|
"""(longest request, page-table width). A decode step may widen a request by decode_rows_per_req - 1 provisional rows past its length, so requests get that much less and the table still covers the widened tail."""
|
|
limit = min(config.max_seq_len, num_tokens)
|
|
return limit - (config.decode_rows_per_req - 1), _page_table_width(limit, config.page_size)
|
|
|
|
|
|
def _required_attn_types(model_config) -> frozenset[AttnType]:
|
|
"""Backend-driving attention types of this model, from the group-spec walk
|
|
(single source shared with the pool factory and the KV cost model). getattr
|
|
fallbacks: duck-typed test configs may not implement the spec walk; for those,
|
|
dsv4_args marks DSV4 (the real config declares a DSV4 attention group)."""
|
|
specs_fn = getattr(model_config, "kv_cache_group_specs", None)
|
|
if specs_fn is None:
|
|
if getattr(model_config, "dsv4_args", None) is not None:
|
|
return frozenset({AttnType.DSV4})
|
|
return frozenset({AttnType.FULL})
|
|
types = frozenset(
|
|
spec.attn_type for spec in specs_fn() if spec.attn_type.backend_driven
|
|
)
|
|
return types or frozenset({AttnType.FULL})
|
|
|
|
|
|
def _backend_parts_serve(name: str, required: frozenset[AttnType]) -> bool:
|
|
return all(
|
|
required <= attention_backend_info(part).supported_types
|
|
for part in name.split(",")
|
|
)
|
|
|
|
|
|
def _backend_requirements_met(name: str) -> bool:
|
|
# flashinfer first across ALL parts: the sgl probe logs a "falls back to fi" warning,
|
|
# which would mislead when the candidate is about to fail on flashinfer anyway.
|
|
infos = [attention_backend_info(part) for part in name.split(",")]
|
|
if any(i.requires_flashinfer for i in infos) and not _flashinfer_available():
|
|
return False
|
|
if any(i.requires_sgl_kernel for i in infos) and not _sgl_flash_attn_available():
|
|
return False
|
|
if any(i.requires_sm100 for i in infos) and not is_sm100_family():
|
|
return False
|
|
return True
|
|
|
|
|
|
def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str:
|
|
"""First candidate (in per-type priority order) whose arch condition holds,
|
|
whose packages are installed, and whose every comma part serves ALL required
|
|
types. Reproduces the historical hardware tree for FULL-only models:
|
|
sm_100 -> trtllm, sm_90+sgl_kernel -> "fa,fi", flashinfer -> fi, else triton."""
|
|
candidates: list[tuple[str, bool]] = []
|
|
if AttnType.DSV4 in required:
|
|
candidates.append(("dsv4_sparse", True))
|
|
if AttnType.DSV41 in required:
|
|
candidates.append(("dsv41_sparse", True))
|
|
if required & {AttnType.MLA, AttnType.DSA}:
|
|
candidates.append(("dsa", True))
|
|
if AttnType.BSA in required:
|
|
candidates.append(("m3_sparse", True))
|
|
if AttnType.QSA in required:
|
|
candidates.append(("qsa_sparse", True))
|
|
if AttnType.SWA in required:
|
|
candidates.append(("triton", True))
|
|
if AttnType.FULL in required:
|
|
candidates += [
|
|
("trtllm", is_sm100_family()),
|
|
("fa,fi", is_sm90_family()),
|
|
("fi", True),
|
|
("triton", True),
|
|
]
|
|
for name, arch_ok in candidates:
|
|
if not arch_ok:
|
|
continue
|
|
if not _backend_parts_serve(name, required):
|
|
continue
|
|
if not _backend_requirements_met(name):
|
|
continue
|
|
return name
|
|
raise RuntimeError(
|
|
"No attention backend can serve attention types "
|
|
f"{sorted(t.value for t in required)} on this machine."
|
|
)
|
|
|
|
|
|
def _spec_runner():
|
|
from freetoken.engine.spec_protocol import SPEC_RUNNER
|
|
from freetoken.models.register import _load_attr
|
|
|
|
return _load_attr(*SPEC_RUNNER.split(":"))
|
|
|
|
|
|
def _adjust_spec_config(config, override, windowed_kv: bool) -> None:
|
|
"""Resolve k and the method's own limits, then check what every speculative method needs of the target."""
|
|
runner = _spec_runner()
|
|
runner.adjust_config(config, override)
|
|
override("spec_slot_states", runner.slot_states(config))
|
|
if windowed_kv:
|
|
raise ValueError(
|
|
"speculative decoding does not support targets with sliding-window or DSV4 window-tier KV yet: "
|
|
"their window frees would release slots a verify step still reads"
|
|
)
|
|
# at config time: the state pool that holds the handlers is built only after the weights load
|
|
if unverified := [s.name for s in config.model_config.slot_states if s.verify is None]:
|
|
raise ValueError(f"speculative decoding does not support per-request state {unverified} yet: it declares no verify handler")
|
|
parts = [p.strip() for p in config.attention_backend.split(",")]
|
|
graph = attention_backend_info(parts[-1]).uniform_extend_graph
|
|
if graph == "host_plan" and len(parts) == 2 and attention_backend_info(parts[0]).uniform_extend_graph == "device":
|
|
# verify steps run on the decode part; a prefill part that captures on device serves them better
|
|
logger.info_rank0(f"speculative decoding: attention backend {config.attention_backend} -> {parts[0]}")
|
|
override("attention_backend", parts[0])
|
|
elif graph == "none":
|
|
raise ValueError(
|
|
f"speculative decoding needs a decode attention backend that captures multi-row graphs "
|
|
f"(fa, triton or fi), got {config.attention_backend!r}"
|
|
)
|
|
|
|
|
|
def _validate_attention_backend_choice(config, override, required: frozenset[AttnType]) -> None:
|
|
"""Config-time type x backend capability check for the resolved (or explicit)
|
|
backend string: every comma part must serve every required type and have its
|
|
packages/arch available. Replaces the per-model gates; in particular this is
|
|
where a DSV4 or MLA checkpoint rejects a generic backend before weights load,
|
|
and where a generic model rejects dsa/dsv4_sparse."""
|
|
from freetoken.attention import validate_attn_backend
|
|
from freetoken.kernel.backend import is_rocm
|
|
|
|
# Name membership first (ArgumentTypeError listing the supported names): the CLI already
|
|
# ran this, but the programmatic EngineConfig path reaches here unvalidated and would
|
|
# otherwise die on a bare KeyError from the info lookup below.
|
|
validate_attn_backend(config.attention_backend, allow_auto=False)
|
|
|
|
model_config = config.model_config
|
|
backend_parts = [p.strip() for p in config.attention_backend.split(",")]
|
|
for part in backend_parts:
|
|
info = attention_backend_info(part)
|
|
missing = required - info.supported_types
|
|
if missing:
|
|
valid = [
|
|
name
|
|
for name in (
|
|
"fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse",
|
|
"qsa_sparse",
|
|
)
|
|
if required <= attention_backend_info(name).supported_types
|
|
]
|
|
missing_names = "/".join(sorted(t.value for t in missing))
|
|
raise ValueError(
|
|
f"{getattr(model_config, 'model_type', 'model')} uses {missing_names} "
|
|
f"attention, which backend {part!r} does not support; valid backends: "
|
|
f"{', '.join(valid)} (or auto), got {config.attention_backend!r}."
|
|
)
|
|
if AttnType.SWA in required and not info.consumes_attn_spec:
|
|
# SWA models drive window/sinks/sm_scale through the per-call AttentionSpec;
|
|
# a backend that drops it would attend with the wrong window silently.
|
|
raise ValueError(
|
|
f"backend {part!r} does not consume the per-call AttentionSpec that "
|
|
f"SWA models require, got {config.attention_backend!r}."
|
|
)
|
|
|
|
# An explicitly-selected backend may require a package that isn't installed. Auto
|
|
# never resolves to one of these when its package is missing, so this only fires for
|
|
# explicit --attention-backend choices.
|
|
for part in backend_parts:
|
|
info = attention_backend_info(part)
|
|
if info.requires_flashinfer and not _flashinfer_available():
|
|
if is_rocm():
|
|
raise RuntimeError(
|
|
f"Attention backend {part!r} uses CUDA-only FlashInfer kernels "
|
|
"unavailable on ROCm; use --attention-backend auto or triton."
|
|
)
|
|
raise RuntimeError(
|
|
f"Attention backend {config.attention_backend!r} requires flashinfer, which is "
|
|
"not installed. Install it with `pip install 'freetoken[fi]'` (or "
|
|
"'freetoken[accel]'), or use --attention-backend triton."
|
|
)
|
|
if info.requires_sgl_kernel and not _sgl_flash_attn_available():
|
|
if is_rocm():
|
|
raise RuntimeError(
|
|
f"Attention backend {part!r} uses CUDA-only sgl_kernel kernels "
|
|
"unavailable on ROCm; use --attention-backend auto or triton."
|
|
)
|
|
raise RuntimeError(
|
|
f"Attention backend {config.attention_backend!r} requires sgl_kernel, which is "
|
|
"not installed. Install it with `pip install 'freetoken[sgl]'` (or "
|
|
"'freetoken[accel]'), or use --attention-backend triton."
|
|
)
|
|
if info.requires_sm100 and not is_sm100_family():
|
|
raise RuntimeError(
|
|
f"Attention backend {config.attention_backend!r} requires a compute capability "
|
|
"10.x GPU: flashinfer's trtllm-gen kernels ship sm_100a/103a cubins only. "
|
|
"Use --attention-backend fi (or triton) instead."
|
|
)
|
|
|
|
if required & {AttnType.MLA, AttnType.DSA}:
|
|
# Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64.
|
|
_kpool_ratio = max(
|
|
(s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla),
|
|
default=1,
|
|
)
|
|
want_page = 64 if _kpool_ratio > 1 else 1
|
|
if config.page_size != want_page:
|
|
logger.warning_rank0(
|
|
f"Page size {config.page_size} is auto-adjusted to {want_page} "
|
|
f"for latent-KV attention."
|
|
)
|
|
override("page_size", want_page)
|
|
|
|
for part in backend_parts:
|
|
info = attention_backend_info(part)
|
|
if info.page_sizes is not None and config.page_size not in info.page_sizes:
|
|
override("page_size", info.page_sizes[-1])
|
|
logger.warning_rank0(
|
|
f"Page size is overridden to {info.page_sizes[-1]} for the {part} backend"
|
|
)
|
|
|
|
|
|
def _host_embedding_keys(model, config: EngineConfig) -> frozenset[str]:
|
|
"""State-dict keys of the tables --embed-weights host keeps in pinned host RAM."""
|
|
if config.embed_weights != "host":
|
|
return frozenset()
|
|
if config.model_config.tie_word_embeddings:
|
|
raise ValueError(
|
|
"--embed-weights host: this model's embedding table is tied to its LM head, which reads the whole table "
|
|
"every step; serve it with --embed-weights gpu"
|
|
)
|
|
tables = {id(op.weight) for op in iter_ops(model) if type(op) is VocabParallelEmbedding}
|
|
keys = frozenset(key for key, t in model.state_dict().items() if id(t) in tables)
|
|
if not keys:
|
|
raise ValueError(
|
|
f"--embed-weights host: {type(model).__name__} has no plain embedding table to move (a DF11 table stays "
|
|
"on the GPU); serve it with --embed-weights gpu"
|
|
)
|
|
return keys
|
|
|
|
|
|
def _pinned_copy(key: str, t: torch.Tensor) -> torch.Tensor:
|
|
from freetoken.kernel.pinned import alloc_pinned_tensor
|
|
|
|
try:
|
|
host = alloc_pinned_tensor(*t.shape, dtype=t.dtype)
|
|
except RuntimeError as exc:
|
|
raise PinFailed(
|
|
f"cudaHostAlloc refused {t.nbytes / 2**20:.0f} MiB for {key} (--embed-weights host); "
|
|
"free host RAM or serve with --embed-weights gpu"
|
|
) from exc
|
|
host.copy_(t)
|
|
return host
|
|
|
|
|
|
def _make_dummy_weight_state_dict(
|
|
model_state: Dict[str, torch.Tensor],
|
|
*,
|
|
device: torch.device,
|
|
host_keys: frozenset[str] = frozenset(),
|
|
) -> Dict[str, torch.Tensor]:
|
|
state_dict: Dict[str, torch.Tensor] = {}
|
|
fp8_dtypes = (torch.float8_e4m3fn, torch.float8_e5m2)
|
|
for key, param in model_state.items():
|
|
if param.dtype in fp8_dtypes:
|
|
# torch.randn is not implemented for fp8; fill via a uint8 view with small
|
|
# codes (avoid NaN/inf fp8 encodings). Lets dummy-weight startup work for
|
|
# block-fp8 models (the dense fp8 linears are fp8 regardless of moe_strategy).
|
|
t = torch.empty(param.shape, dtype=param.dtype, device=device)
|
|
t.view(torch.uint8).random_(0, 16)
|
|
state_dict[key] = t
|
|
elif param.dtype.is_floating_point or param.dtype.is_complex:
|
|
state_dict[key] = torch.randn(param.shape, dtype=param.dtype, device=device)
|
|
elif param.dtype == torch.uint8 and key.endswith("weight_scale_inv"):
|
|
# MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse
|
|
# every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8
|
|
# buffers are packed payloads whose bytes mean something else entirely
|
|
# (GGUF blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
|
|
# keep the benign all-zeros fill below.
|
|
state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)
|
|
else:
|
|
state_dict[key] = torch.zeros(param.shape, dtype=param.dtype, device=device)
|
|
if key in host_keys:
|
|
state_dict[key] = _pinned_copy(key, state_dict[key])
|
|
return state_dict
|
|
|
|
|
|
class WeightLoadError(RuntimeError):
|
|
"""The checkpoint itself could not be read. Resource and config failures keep their own type."""
|
|
|
|
|
|
def _is_resource_failure(exc: Exception) -> bool:
|
|
if isinstance(exc, (torch.OutOfMemoryError, MemoryError, PinFailed)):
|
|
return True
|
|
# an anonymous mmap that does not fit raises ENOMEM, not MemoryError
|
|
if isinstance(exc, OSError) and exc.errno == errno.ENOMEM:
|
|
return True
|
|
# torch has no type for a failed CPU allocation
|
|
return isinstance(exc, RuntimeError) and "DefaultCPUAllocator" in str(exc)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def _weight_load_context():
|
|
"""Wrap a checkpoint read so the startup failure reason (log and /health) starts with WeightLoadError."""
|
|
try:
|
|
yield
|
|
except Exception as exc:
|
|
if _is_resource_failure(exc):
|
|
raise
|
|
raise WeightLoadError(f"{type(exc).__name__}: {exc}") from exc
|
|
|
|
|
|
def _materialize_loaded_weight_state_dict(
|
|
model_state: Dict[str, torch.Tensor],
|
|
weights: Iterable[Tuple[str, torch.Tensor]],
|
|
*,
|
|
device: torch.device,
|
|
host_keys: frozenset[str] = frozenset(),
|
|
) -> Dict[str, torch.Tensor]:
|
|
state_dict: Dict[str, torch.Tensor] = {}
|
|
for key, weight in weights:
|
|
expected = model_state.get(key)
|
|
if expected is None:
|
|
# NOTE: the quant scheme may declare no input_scale for a layer whose FTW still stores one
|
|
if key.endswith(".input_scale"):
|
|
continue
|
|
state_dict[key] = weight.to(device=device)
|
|
elif key in host_keys:
|
|
state_dict[key] = _pinned_copy(key, weight.to(dtype=expected.dtype))
|
|
else:
|
|
state_dict[key] = weight.to(device=device, dtype=expected.dtype)
|
|
return state_dict
|
|
|
|
|
|
class ForwardOutput(NamedTuple):
|
|
next_tokens_gpu: torch.Tensor
|
|
next_tokens_cpu: torch.Tensor
|
|
copy_done_event: torch.cuda.Event
|
|
|
|
|
|
class Engine:
|
|
def __init__(self, config: EngineConfig):
|
|
assert not torch.cuda.is_initialized()
|
|
set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)
|
|
set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
|
|
_ensure_expandable_segments() # before the first CUDA allocation below
|
|
|
|
from freetoken.gpu_select import bind_assigned_gpu
|
|
|
|
self.device = bind_assigned_gpu(config.tp_info.rank)
|
|
_adjust_config(config)
|
|
torch.manual_seed(42)
|
|
self.stream = torch.cuda.Stream()
|
|
torch.cuda.set_stream(self.stream)
|
|
self.dtype = config.dtype
|
|
self.config = config # retained for runtime cache rebuild (rebuild_runtime_cache)
|
|
# KV pool family fixed at construction from the model config: its classmethods own the
|
|
# page-token geometry and cost arithmetic the engine needs BEFORE the pool exists
|
|
# (num_pages sizing, --moe-cache-auto); the instance owns rebuild/validation after.
|
|
self._pool_cls = resolve_pool_class(config.model_config)
|
|
self.ctx = Context(config.page_size)
|
|
set_global_ctx(self.ctx)
|
|
|
|
self.tp_cpu_group = self._init_communication(config)
|
|
free_min, free_max = self._sync_get_memory()
|
|
init_free_memory = free_max # startup KV sizing keeps cross-rank MAX (unchanged)
|
|
self._baseline_free = free_min # rebuild baseline: cross-rank MIN, deterministic across ranks
|
|
logger.info_rank0(f"Free memory before loading model: {mem_GB(init_free_memory)}")
|
|
|
|
# ======================= Model initialization ========================
|
|
set_rope_device(self.device)
|
|
with torch.device("meta"), torch_dtype(config.dtype):
|
|
self.model = create_model(config.model_config)
|
|
self._load_weights(config)
|
|
if config.active_encoders:
|
|
from freetoken.models.blocks import SupportsMultimodal
|
|
|
|
if not isinstance(self.model, SupportsMultimodal):
|
|
raise TypeError(
|
|
f"{type(self.model).__name__} has encoders registered but lacks the SupportsMultimodal hooks; "
|
|
"run with --text-model-only"
|
|
)
|
|
# before the residency snapshot, so streamed blocks are not charged as resident weights
|
|
self.model.place_encoder_weights(config.mm.encoder_weights)
|
|
# draft weights load before any pool is sized, so every free-memory read below counts them
|
|
self.spec: SpecRuntime | None = _spec_runner()(config, self) if config.speculative_algorithm is not None else None
|
|
post_weights_free = self._sync_get_memory()[0]
|
|
self._weights_bytes = self._baseline_free - post_weights_free
|
|
# Pool-budget baseline for the desktop cache sliders: free VRAM after the weights are
|
|
# resident but before ANY runtime cache pool (MoE expert cache below, KV pages, GDN
|
|
# state) is allocated. This is the stable "if all free VRAM went to one pool" budget —
|
|
# unlike a query-time mem_get_info it doesn't drift with allocator caching, CUDA
|
|
# graphs, or other processes. Cross-rank MIN, deterministic across ranks.
|
|
self._post_weights_free = post_weights_free
|
|
self.moe_offload_cache = None
|
|
self.cpu_moe_executor = None
|
|
# Host-side auxiliary stores (--embed-weights host tables, qwen4_exp's pinned PLE table): after the weights so a
|
|
# load failure is not masked, before the MoE offload cache so the bank residency
|
|
# planning sees the pin quota the table already spent.
|
|
host_embeds = [op for op in iter_ops(self.model) if type(op) is VocabParallelEmbedding and not op.weight.is_cuda]
|
|
self._host_tables_bytes = sum(op.weight.nbytes for op in host_embeds)
|
|
if host_embeds:
|
|
logger.info_rank0(f"Embedding tables in pinned host RAM (--embed-weights host): {mem_GB(self._host_tables_bytes)}")
|
|
if hasattr(self.model, "load_host_tables"):
|
|
with _weight_load_context():
|
|
self._host_tables_bytes += int(self.model.load_host_tables(config) or 0)
|
|
if is_offload_moe_strategy(config.moe_strategy):
|
|
self._init_offload_moe_cache(config)
|
|
if hasattr(self.model, "prepare_for_runtime"):
|
|
self.model.prepare_for_runtime()
|
|
self.encoder_cache = None
|
|
self.mm_processor = None
|
|
if config.active_encoders:
|
|
from freetoken.mm.encoder_cache import EncoderCache
|
|
from freetoken.mm.processor import get_mm_processor
|
|
|
|
self.mm_processor = get_mm_processor(config.model_path, config.mm)
|
|
self.encoder_cache = EncoderCache(storage=config.mm.embed_cache_device)
|
|
logger.info_rank0(
|
|
f"Multimodal enabled: {type(self.mm_processor).__name__}, encoders "
|
|
f"{[e.kind for e in config.active_encoders]} on {config.mm.encoder_weights}, serving {sorted(config.served_modalities)}"
|
|
)
|
|
self._warmup_encoders()
|
|
elif any(getattr(config.hf_config, key, None) is not None for key in ENCODER_SECTIONS):
|
|
logger.info_rank0(
|
|
"Multimodal disabled: --text-model-only"
|
|
if config.mm.text_model_only
|
|
else "Multimodal disabled: --mm-disable"
|
|
if config.mm.disabled_encoders
|
|
else "Multimodal disabled: no encoder registered for this architecture"
|
|
)
|
|
|
|
# ======================= KV cache initialization ========================
|
|
new_free = self._sync_get_memory()[1]
|
|
# The engine measures the budget and settles the sibling GDN state pool's bytes
|
|
# off it; the KV pool family owns every geometry-specific formula behind the rest.
|
|
available_memory = _startup_kv_budget(config.memory_ratio, init_free_memory, new_free)
|
|
available_memory -= state_pool_bytes(config)
|
|
if self.spec is not None:
|
|
# the pool solves pages from its own cost; hand it the share its pages get next to the draft's
|
|
spec_fixed, draft_page = self._spec_budget(config)
|
|
page_bytes, fixed_bytes = self._pool_cls.kv_cost(config)[:2]
|
|
available_memory = fixed_bytes + (available_memory - spec_fixed - fixed_bytes) * page_bytes // (page_bytes + draft_page)
|
|
self.num_pages = self._pool_cls.solve_num_pages(config, available_memory)
|
|
num_tokens = self.num_pages * config.page_size
|
|
self.ctx.kv_cache = self.kv_cache = create_kv_pool(
|
|
config, self.num_pages, device=self.device, dtype=self.dtype
|
|
)
|
|
|
|
# ======================= Linear (GatedDeltaNet) state initialization ========================
|
|
linear_group = config.model_config.linear_attention_group()
|
|
if linear_group is not None:
|
|
from freetoken.kvcache.linear_state_pool import LinearStatePool
|
|
|
|
self.linear_state_pool = LinearStatePool(
|
|
group=linear_group,
|
|
num_slots=_linear_pool_num_slots(config),
|
|
dtype=self.dtype,
|
|
device=self.device,
|
|
tp_size=config.tp_info.size,
|
|
slot_states=config.slot_states,
|
|
)
|
|
self.ctx.linear_state_pool = self.linear_state_pool
|
|
else:
|
|
self.linear_state_pool = None
|
|
|
|
# ======================= Page table initialization ========================
|
|
# NOTE: 1. aligned to 128 bytes; 2. store raw locations instead of pages
|
|
self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
|
|
self.ctx.page_table = self.page_table = torch.zeros( # + 1 for dummy request
|
|
(config.max_running_req + 1, aligned_max_seq_len),
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
# Pools routed by the shared table but deriving reads through their own mappings (DSV4)
|
|
# re-point here (and again on any table realloc). The graph-input snapshot that reads
|
|
# through them belongs to the attention backend, built later in init_capture_graph.
|
|
self.kv_cache.attach_page_table(self.page_table)
|
|
|
|
# ======================= Attention backend initialization ========================
|
|
self.ctx.attn_backend = self.attn_backend = create_attention_backend(
|
|
config.attention_backend, config.model_config
|
|
)
|
|
self.linear_backend = LinearAttnBackend(self.device, self.linear_state_pool) if self.linear_state_pool is not None else None
|
|
|
|
# ======================= Sampler initialization ========================
|
|
self.sampler = Sampler(self.device, config.model_config.vocab_size)
|
|
|
|
# padded rows replay this request, so it carries one token per decode row
|
|
self.dummy_req = Req(
|
|
input_ids=torch.zeros(config.decode_rows_per_req, dtype=torch.int32, device="cpu"),
|
|
table_idx=config.max_running_req,
|
|
cached_len=0,
|
|
output_len=1,
|
|
uid=-1,
|
|
sampling_params=None, # type: ignore
|
|
cache_handle=None, # type: ignore
|
|
)
|
|
# padded/dummy rows index the GDN padding slot (0) so gather/scatter hits scratch.
|
|
# Naive GDN keys state by table_idx, where slot 0 is a live request and the dummy's own row is the scratch.
|
|
if self.linear_state_pool is not None and config.cache_type == "hybrid_radix":
|
|
self.dummy_req.linear_slot_idx = self.linear_state_pool.padding_slot
|
|
self.page_table[self.dummy_req.table_idx].fill_(num_tokens) # point to dummy page
|
|
|
|
# ======================= Verify state initialization ========================
|
|
if config.decode_rows_per_req > 1 and self.linear_state_pool is not None:
|
|
self.linear_state_pool.attach_verify_scratch(config.max_running_req, config.decode_rows_per_req)
|
|
if self.spec is not None:
|
|
self.spec.attach(self)
|
|
|
|
post_free_memory = self._sync_get_memory()[0]
|
|
logger.info_rank0(f"Free memory after initialization: {mem_GB(post_free_memory)}")
|
|
|
|
# ======================= Graph capture initialization ========================
|
|
self.graph_runner = self._make_graph_runner(config.cuda_graph_bs, init_free_memory, aligned_max_seq_len)
|
|
if config.attention_backend.split(",")[0] == "triton":
|
|
# Prefill runs on the first comma part; warm its autotune cache.
|
|
self._warmup_prefill()
|
|
|
|
def _init_communication(self, config: EngineConfig) -> torch.distributed.ProcessGroup:
|
|
if config.tp_info.size == 1 or config.use_pynccl:
|
|
torch.distributed.init_process_group(
|
|
backend="gloo",
|
|
rank=config.tp_info.rank,
|
|
world_size=config.tp_info.size,
|
|
timeout=timedelta(seconds=config.distributed_timeout),
|
|
init_method=config.distributed_addr,
|
|
)
|
|
tp_cpu_group = torch.distributed.group.WORLD
|
|
assert tp_cpu_group is not None
|
|
max_bytes = (
|
|
config.max_forward_len * config.model_config.hidden_size * self.dtype.itemsize
|
|
)
|
|
enable_pynccl_distributed(config.tp_info, tp_cpu_group, max_bytes)
|
|
else:
|
|
torch.distributed.init_process_group(
|
|
backend="nccl",
|
|
rank=config.tp_info.rank,
|
|
world_size=config.tp_info.size,
|
|
timeout=timedelta(seconds=config.distributed_timeout),
|
|
init_method=config.distributed_addr,
|
|
)
|
|
tp_cpu_group = torch.distributed.new_group(backend="gloo")
|
|
assert tp_cpu_group is not None
|
|
return tp_cpu_group
|
|
|
|
def _load_weights(self, config: EngineConfig) -> None:
|
|
if config.active_encoders and not config.use_dummy_weight and ftw_lacks_vision(config.model_path):
|
|
raise ValueError(
|
|
f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this "
|
|
"family served images. Reconvert it with `ft checkpoint`, add the encoder in place with "
|
|
"scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only"
|
|
)
|
|
resident = [layer for layer in iter_moe_layers(self.model) if not isinstance(layer, OffloadMoELayer)]
|
|
if resident and not config.use_dummy_weight and is_ftw_checkpoint(config.model_path) and not ftw_tensor_names(config.model_path, "experts_bank"):
|
|
raise ValueError(f"{config.model_path} keeps its routed experts as dense weights (an older `ft checkpoint --moe-backend fused`); reconvert it with `ft checkpoint`")
|
|
host_keys = _host_embedding_keys(self.model, config)
|
|
with _weight_load_context():
|
|
self.model.load_state_dict(self._load_weight_state_dict(config, host_keys))
|
|
finalize_quant(self.model)
|
|
if resident:
|
|
self._load_resident_experts(config, resident)
|
|
|
|
def _load_weight_state_dict(self, config: EngineConfig, host_keys: frozenset[str]) -> Dict[str, torch.Tensor]:
|
|
model_state = self.model.state_dict()
|
|
if config.use_dummy_weight:
|
|
return _make_dummy_weight_state_dict(model_state, device=self.device, host_keys=host_keys)
|
|
# _materialize casts each loaded tensor to its model-param dtype (model_state), so
|
|
# models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;
|
|
# MoE models exclude routed experts (they load as expert banks, not dense weights).
|
|
return _materialize_loaded_weight_state_dict(
|
|
model_state,
|
|
load_weight(
|
|
config.model_path,
|
|
self.device,
|
|
include_moe_experts=not getattr(config.model_config, "is_moe", False),
|
|
include_vision=bool(config.active_encoders),
|
|
),
|
|
device=self.device,
|
|
host_keys=host_keys,
|
|
)
|
|
|
|
def _load_resident_experts(self, config: EngineConfig, layers) -> None:
|
|
with _weight_load_context():
|
|
banks = load_expert_banks(
|
|
config.model_path,
|
|
config.model_config,
|
|
method=layers[0].quant_method,
|
|
device=self.device,
|
|
dummy=config.use_dummy_weight,
|
|
parallel={"serial": False, "parallel": True}.get(config.expert_load, None),
|
|
resident=True,
|
|
layer_methods=[layer.quant_method for layer in sorted(layers, key=lambda layer: layer.layer_id)],
|
|
)
|
|
attach_resident_banks(layers, banks)
|
|
|
|
@torch.inference_mode()
|
|
def _warmup_encoders(self) -> None:
|
|
for item in self.mm_processor.dummy_items(self.dtype, self.device):
|
|
if item.modality in self.config.served_modalities:
|
|
self.model.encode(item)
|
|
torch.cuda.synchronize(self.device)
|
|
|
|
@torch.inference_mode()
|
|
def _run_mm_encoder(self, batch: Batch) -> None:
|
|
"""Encode the chunk's cache-miss items and gather its embedding rows into batch.mm_embeds, right before the LM forward."""
|
|
cache = self.encoder_cache
|
|
jobs = batch.mm_encoder_jobs or ()
|
|
# a job whose rows are not gathered this chunk would be encoded and freed unread
|
|
planned = {row[1] for row in batch.mm_gather_plan}
|
|
orphans = [item.hash for item in jobs if item.hash not in planned]
|
|
assert not orphans, f"encoder jobs without gather rows: {orphans}"
|
|
for item in jobs:
|
|
if not cache.has(item.hash):
|
|
if item.precomputed_embeddings is not None:
|
|
emb = item.precomputed_embeddings.to(self.device, non_blocking=True)
|
|
else:
|
|
emb = self.model.encode(item)
|
|
cache.put(item.hash, emb)
|
|
# cached now; free the ~MiB feature buffer
|
|
item.feature = None
|
|
item.precomputed_embeddings = None
|
|
parts = []
|
|
for uid, item_hash, lo, hi, _, _ in batch.mm_gather_plan:
|
|
parts.append(cache.get_slice(item_hash, lo, hi, self.device))
|
|
cache.consume(item_hash, uid, hi - lo)
|
|
if parts:
|
|
batch.mm_embeds = torch.cat([p.to(self.dtype) for p in parts], dim=0)
|
|
|
|
def _resolve_auto_moe_cache_size(self, config: EngineConfig, banks, method=None, max_slots=None) -> tuple[int, int, bool]:
|
|
"""Resolve --moe-cache-auto into (moe_cache_size, num_pages, prefill_overlap).
|
|
|
|
Pure glue over the Phase-1 budget policy; isolated here so it is unit-testable
|
|
without a GPU. Reused by the Phase-2 runtime rebuild.
|
|
"""
|
|
from freetoken.engine.cache_budget import expert_bytes_per_slot, resolve_moe_cache_auto
|
|
|
|
cache_per_page, fixed_cache_size, page_tokens, min_reserve = self._pool_cls.kv_cost(config)
|
|
fixed_cache_size += state_pool_bytes(config) # sibling GDN state pool, engine-summed
|
|
spec_fixed, draft_page = self._spec_budget(config)
|
|
fixed_cache_size += spec_fixed
|
|
cache_per_page += draft_page
|
|
num_experts = config.model_config.num_experts
|
|
total_experts = config.model_config.num_moe_layers * num_experts
|
|
return resolve_moe_cache_auto(
|
|
baseline_free=self._baseline_free,
|
|
weights_bytes=self._weights_bytes,
|
|
memory_ratio=config.memory_ratio,
|
|
cache_per_page=cache_per_page,
|
|
fixed_cache_size=fixed_cache_size,
|
|
per_expert_bytes=expert_bytes_per_slot(banks.sources),
|
|
num_experts=num_experts,
|
|
total_experts=total_experts,
|
|
prefill_overlap=config.moe_prefill_overlap,
|
|
kv_reserve_tokens=max(config.kv_reserve_tokens, min_reserve),
|
|
page_size=page_tokens,
|
|
max_slots=max_slots if max_slots is not None or method is None else method.slot_limit(),
|
|
)
|
|
|
|
def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache:
|
|
from freetoken.moe.expert_banks import is_mixed
|
|
|
|
method = shared_offload_method(self.model)
|
|
num_moe_layers = config.model_config.num_moe_layers
|
|
# experts of several formats: one method per MoE layer, one slot pool per format
|
|
layer_methods = offload_layer_methods(self.model)
|
|
if not is_mixed(layer_methods):
|
|
layer_methods = None
|
|
elif config.moe_strategy == "cpu" or config.moe_cpu_layers:
|
|
raise ValueError(
|
|
"this model's experts come in several formats, which are served from per-format GPU slot pools; "
|
|
"they do not combine with --moe-strategy cpu or --moe-cpu-layers (use offload or hybrid)"
|
|
)
|
|
# Fast path: an FTW checkpoint loads its repacked banks directly.
|
|
# Slow path: load_expert_banks auto-picks parallel vs serial baseline by
|
|
# expert-tensor granularity. Both pin-after-fill.
|
|
# --expert-load: serial/parallel force the read; auto (None) lets load_expert_banks
|
|
# pick (parallel for scattered experts, with a low-RAM fallback to serial).
|
|
expert_parallel = {"serial": False, "parallel": True}.get(config.expert_load, None)
|
|
check_locked_fetch_config(config)
|
|
# --moe-cpu-layers auto without a manual pin budget: load unpinned, then pin layers until the driver refuses (settle_banks)
|
|
# a model the CPU executor cannot serve takes the estimate path, which rejects it before the load when layers must lock
|
|
probe = pin_probe_enabled(config, mixed_formats=layer_methods is not None) and _cpu_moe_executor_viable(config.model_config)
|
|
if probe:
|
|
from freetoken.moe.host_banks import HostResidency
|
|
|
|
requested_residency = [HostResidency.DEFERRED.value] * num_moe_layers
|
|
cpu_layer_ids, decode_target, split_residency = frozenset(), None, False
|
|
else:
|
|
reserved = self._host_tables_bytes + staging_reserve(config, lambda: _bank_bytes(config, method, layer_methods))
|
|
cpu_layer_ids = _resolve_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
|
|
_check_pin_budget(config, reserved=reserved, method=method, layer_methods=layer_methods)
|
|
decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=_pin_budget_bytes(reserved) is not None)
|
|
requested_residency = None
|
|
if split_residency:
|
|
from freetoken.moe.host_banks import HostResidency
|
|
|
|
requested_residency = [
|
|
HostResidency.LOCKED.value if i in cpu_layer_ids
|
|
else HostResidency.PINNED.value
|
|
for i in range(config.model_config.num_moe_layers)
|
|
]
|
|
# read before the expert load, so a bad profile fails in seconds
|
|
expert_counts = resolve_profile(
|
|
config.moe_cache_split, config.model_path,
|
|
config.model_config.num_moe_layers, config.model_config.num_experts,
|
|
)
|
|
try:
|
|
with _weight_load_context():
|
|
banks = load_expert_banks(
|
|
config.model_path,
|
|
config.model_config,
|
|
method=method,
|
|
device=self.device,
|
|
dummy=config.use_dummy_weight,
|
|
parallel=expert_parallel,
|
|
layer_residency=requested_residency,
|
|
layer_methods=layer_methods,
|
|
)
|
|
except PinFailed as exc:
|
|
raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from exc
|
|
banks, probed, staging = settle_banks(config, banks, probe=probe, has_locked=split_residency)
|
|
if probe:
|
|
cpu_layer_ids = probed
|
|
decode_target, split_residency = self._residency_split(config, cpu_layer_ids, method, capped=True)
|
|
from freetoken.engine.cache_budget import expert_bytes_per_slot, layer_slot_bytes
|
|
|
|
self._moe_layer_slot_bytes = layer_slot_bytes(banks.sources)
|
|
# hybrid: locked layers keep GPU slots of their own pools and fetch part of their misses through the pinned staging (moe/locked_fetch.py)
|
|
locked_fetch = staging is not None
|
|
layer_tags = locked_pool_tags(banks.layer_tags, cpu_layer_ids, num_moe_layers) if locked_fetch else banks.layer_tags
|
|
self._moe_pool_groups = group_layers(config.moe_cache_scope, bank_layouts(banks.sources, layer_tags), self._moe_layer_slot_bytes)
|
|
if locked_fetch:
|
|
# a locked refill costs a staged copy: place the locked pools after the pinned ones, which hold the prefill staging when they are big enough
|
|
self._moe_pool_groups.sort(key=lambda layers: layers[0] in cpu_layer_ids)
|
|
if len(self._moe_pool_groups) > 1 and cpu_layer_ids and not locked_fetch:
|
|
raise ValueError(
|
|
"several MoE slot pools (--moe-cache-scope per-layer, or experts in several quant formats) need every MoE "
|
|
"layer to decode from the GPU slot cache; they combine with --moe-cpu-layers only for locked layers under "
|
|
"--moe-strategy hybrid unless --moe-locked-fetch is 0"
|
|
)
|
|
self._moe_expert_counts = expert_counts if len(self._moe_pool_groups) > 1 else None
|
|
if len(self._moe_pool_groups) == 1 and config.moe_cache_split != "equal":
|
|
logger.warning_rank0(
|
|
f"--moe-cache-split {config.moe_cache_split} is ignored: all MoE layers share one slot pool "
|
|
f"(--moe-cache-scope per-layer gives each layer its own)"
|
|
)
|
|
self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)
|
|
# with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one request's decode step
|
|
pools = self._moe_pools(config.decode_rows_per_req if decode_target == "gpu" else 1)
|
|
size_cap = sum(p.max_num_slots for p in pools)
|
|
if config.moe_cache_auto:
|
|
size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)
|
|
object.__setattr__(config, "moe_cache_size", size)
|
|
object.__setattr__(config, "moe_prefill_overlap", overlap)
|
|
if config.num_page_override is None:
|
|
# Honor the plan's KV half too: MoE slots and KV pages were solved
|
|
# against ONE budget (ratio x baseline - weights), so both must come
|
|
# from it. Re-solving pages later from a fresh free-memory reading
|
|
# double-counts everything allocated since the weights measurement
|
|
# (this expert cache, the CPU-executor GPU buffers, allocator
|
|
# slack) and goes negative whenever the expert fill is exact --
|
|
# a greedy fill leaves no headroom for the measurement delta.
|
|
object.__setattr__(config, "num_page_override", pages)
|
|
logger.info_rank0(
|
|
f"--moe-cache-auto resolved moe_cache_size={size} "
|
|
f"num_pages={pages} (prefill_overlap={overlap})"
|
|
)
|
|
elif config.moe_cache_mem is not None:
|
|
per_slot = expert_bytes_per_slot(banks.sources)
|
|
size = min(config.moe_cache_mem // per_slot, size_cap)
|
|
# per_slot is a mean over layers, so with pools of different slot sizes the split can exceed the budget: shrink until it fits
|
|
while size > 0 and self._moe_pool_bytes(split_slots(size, pools, self._moe_expert_counts)) > config.moe_cache_mem:
|
|
size -= 1
|
|
overlap = config.moe_prefill_overlap and size >= 2 * config.model_config.num_experts
|
|
object.__setattr__(config, "moe_cache_size", size)
|
|
object.__setattr__(config, "moe_prefill_overlap", overlap)
|
|
logger.info_rank0(
|
|
f"--moe-cache-mem {mem_GB(config.moe_cache_mem)} resolved moe_cache_size={size} "
|
|
f"({per_slot / 2**20:.2f} MiB per slot, prefill_overlap={overlap})"
|
|
)
|
|
_require_offload_cache_size(config.moe_cache_size, config.model_config.num_experts)
|
|
pool_sizes = split_slots(config.moe_cache_size, pools, self._moe_expert_counts)
|
|
if len(pools) > 1:
|
|
logger.info_rank0(
|
|
f"MoE slot pools: {config.moe_cache_scope}, {len(pools)} pools of {min(pool_sizes)}-{max(pool_sizes)} "
|
|
f"slots ({config.moe_cache_split})"
|
|
)
|
|
if decode_target == "gpu":
|
|
_clamp_decode_batch_to_pools(config, pool_sizes)
|
|
layout = layer_layouts = max_slots = None
|
|
if layer_methods is not None:
|
|
_check_layer_formats(banks, layer_methods)
|
|
layer_layouts = [m.layout() for m in layer_methods]
|
|
max_slots = self._moe_slot_limit
|
|
elif method is not None:
|
|
if banks.layer_formats is not None:
|
|
raise ValueError(
|
|
f"expert banks hold experts of several formats ({sorted(set(banks.layer_formats))}) but the model "
|
|
f"binds {method.kind} / {method.kernel.name} on every MoE layer; reconvert the checkpoint"
|
|
)
|
|
if banks.kind is not None and (banks.kind, banks.kernel) != (method.kind, method.kernel.name):
|
|
raise ValueError(
|
|
f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "
|
|
f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel"
|
|
)
|
|
# GGUF layers keep their own ggml types, so the cache reads each layer's layout off its banks
|
|
layout = None if method.kind is QuantKind.GGUF else method.layout()
|
|
max_slots = method.slot_limit()
|
|
cache = OffloadMoeCache(
|
|
# Models with leading dense layers (GLM-4) only have experts on the MoE
|
|
# layers; num_moe_layers == num_layers when first_k_dense_replace == 0.
|
|
num_layers=config.model_config.num_moe_layers,
|
|
num_experts=config.model_config.num_experts,
|
|
cache_size=config.moe_cache_size,
|
|
device=self.device,
|
|
cache_policy=config.moe_cache_policy,
|
|
prefill_overlap=config.moe_prefill_overlap,
|
|
prefill_hit_d2d=config.moe_prefill_hit_d2d,
|
|
quant_format=banks.quant_format,
|
|
decode_target=decode_target,
|
|
hybrid_max_fetch=config.moe_hybrid_max_fetch,
|
|
layout=layout,
|
|
layer_layouts=layer_layouts,
|
|
max_slots=max_slots,
|
|
pool_of_layer=pool_of_layer(pools, config.model_config.num_moe_layers),
|
|
pool_sizes=pool_sizes,
|
|
)
|
|
# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer set
|
|
cache.cpu_layer_ids = cpu_layer_ids
|
|
if locked_fetch:
|
|
cache.locked_hybrid = cpu_layer_ids
|
|
cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=layer_tags, row_tail_bytes=banks.row_tail_bytes)
|
|
cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)
|
|
# Must be set before CUDA graph capture so the (device-side) accumulation ops are
|
|
# captured and re-run on every decode replay.
|
|
cache.collect_stats = config.moe_collect_stats
|
|
layers = attach_offload_moe_cache(self.model, cache)
|
|
assert len(layers) == config.model_config.num_moe_layers
|
|
if cache.decode_target in ("cpu", "hybrid"):
|
|
self._init_cpu_moe_executor(config, cache, layers)
|
|
if decode_target == "hybrid":
|
|
self._resolve_hybrid_fetch(config, cache)
|
|
if locked_fetch:
|
|
attach_locked_fetch(cache, config, staging, self._profile_layer_rows)
|
|
self.moe_offload_cache = cache
|
|
return cache
|
|
|
|
def _residency_split(self, config: EngineConfig, cpu_layer_ids, method, *, capped: bool):
|
|
"""(decode_target, split_residency) for a CPU-layer set; turns prefill overlap off when some layers stay unpinned."""
|
|
# the kernels were picked for model_config.decode_target; --moe-cpu-layers auto may still find that every bank fits the pin budget
|
|
decode_target = config.model_config.decode_target
|
|
if decode_target == "cpu" and not cpu_layer_ids:
|
|
decode_target = "gpu"
|
|
# split residency: where pinning is quota-capped (_pin_budget_bytes), pin only the GPU layers' banks and mlock the CPU layers'
|
|
# uncapped hosts keep every bank pinned (CPU decode reads them the same; overlap prefill stays on)
|
|
# not applied to plain --moe-strategy cpu; all-locked under a cap = --moe-strategy offload --moe-cpu-layers 1.0
|
|
split_residency = (
|
|
bool(cpu_layer_ids)
|
|
and config.moe_strategy in ("offload", "hybrid")
|
|
and capped
|
|
)
|
|
if config.moe_strategy == "cpu" and not split_residency:
|
|
# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead
|
|
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
|
|
|
|
budget = _pin_budget_bytes(self._host_tables_bytes)
|
|
bank_bytes = None
|
|
if budget is not None:
|
|
bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method)
|
|
if bank_bytes and bank_bytes > budget:
|
|
split_residency = True
|
|
logger.info_rank0(
|
|
f"--moe-strategy cpu: banks {bank_bytes / 2**30:.2f} GiB exceed the "
|
|
f"pin budget; OS-locking all layers instead of pinning"
|
|
)
|
|
if split_residency and config.moe_prefill_overlap:
|
|
# locked (unregistered) layers cannot feed the async pinned H2D double buffer; their prefill is a synchronous pageable copy via materialize
|
|
logger.info_rank0(
|
|
"--moe-cpu-layers split residency: disabling MoE prefill overlap "
|
|
"(locked layers prefill via synchronous pageable copies)"
|
|
)
|
|
object.__setattr__(config, "moe_prefill_overlap", False)
|
|
return decode_target, split_residency
|
|
|
|
def _moe_pool_bytes(self, pool_sizes: list[int]) -> int:
|
|
return sum(s * self._moe_layer_slot_bytes[g[0]] for g, s in zip(self._moe_pool_groups, pool_sizes))
|
|
|
|
def _moe_pools(self, batch: int) -> list:
|
|
"""The slot pools, each with room for the routed experts of ``batch`` decode tokens."""
|
|
model_config = self.config.model_config
|
|
num_experts = model_config.num_experts
|
|
min_num_slots = min(num_experts, model_config.num_experts_per_tok * batch)
|
|
return make_pools(self._moe_pool_groups, num_experts, min_num_slots, self._moe_slot_limit)
|
|
|
|
def _resolve_hybrid_fetch(self, config: EngineConfig, cache) -> None:
|
|
"""Resolve --moe-hybrid-max-fetch -1 (auto) into a bandwidth-matched fetch fraction.
|
|
|
|
Perfect fetch/compute overlap wants fetched : cpu-computed misses = pcie_bw :
|
|
(cpu_bw - pcie_bw), i.e. fetching a pcie_bw / cpu_bw fraction of each decode
|
|
step's misses -- both sides then finish together instead of one idling. The
|
|
achieved bandwidths come from the cached `ft bench bw` profile, from the row of each
|
|
MoE layer's kind (format, options, expert shape, top_k, CPU worker threads); a layer
|
|
without a row keeps the fixed cap of 1.
|
|
"""
|
|
if config.moe_hybrid_max_fetch >= 0:
|
|
return # explicit fixed cap
|
|
from freetoken.moe.bench_profile import fetch_fraction
|
|
|
|
cache.hybrid_max_fetch = 1
|
|
layer_rows = self._profile_layer_rows()
|
|
if layer_rows is None:
|
|
logger.warning_rank0("--moe-hybrid-max-fetch auto: the MoE layers bind no expert method to look up in the `ft bench bw` profile; using a fixed fetch cap of 1")
|
|
return
|
|
executor = self.cpu_moe_executor
|
|
keys, labels, rows = layer_rows
|
|
fractions = [fetch_fraction(rows.get(key)) for key in keys]
|
|
cache.hybrid_fetch_fraction = [f or 0.0 for f in fractions]
|
|
# locked layers fetch the fetch_plan share instead; their own log line lists it
|
|
groups = collections.Counter((labels[l], fractions[l]) for l in range(len(keys)) if l not in cache.locked_hybrid)
|
|
fetched = [f"{f:.1%} on {n} {label} layers" for (label, f), n in groups.items() if f is not None]
|
|
if fetched:
|
|
logger.info_rank0(
|
|
"--moe-hybrid-max-fetch auto: each decode step fetches this share of a layer's expert misses over PCIe "
|
|
f"(benched PCIe/CPU bandwidth ratio) and computes the rest on the CPU: {'; '.join(fetched)}"
|
|
)
|
|
if missing := sorted({label for label, f in groups if f is None}):
|
|
capped = sum(n for (_, f), n in groups.items() if f is None)
|
|
logger.warning_rank0(
|
|
f"--moe-hybrid-max-fetch auto: no `ft bench bw` row for {capped} MoE layers ({', '.join(missing)}) at "
|
|
f"{executor.num_threads} CPU threads; using a fixed fetch cap of 1 there. `ft bench bw --model <checkpoint>` benches each layer kind"
|
|
)
|
|
benched_isas = {rows[key].get("cpu_moe_isa") for key, f in zip(keys, fractions) if f is not None} - {None}
|
|
if other := sorted(benched_isas - set(executor.isa.split(","))):
|
|
logger.warning_rank0(
|
|
f"--moe-hybrid-max-fetch auto: some `ft bench bw` rows were benched on CPU kernels {other}, but this executor runs "
|
|
f"{executor.isa}; using them anyway"
|
|
)
|
|
|
|
def _profile_layer_rows(self):
|
|
"""(row key, label) of each MoE layer's kind in the `ft bench bw` profile, and the profile's rows; None without expert methods."""
|
|
from freetoken.moe.bench_profile import layer_kind_key, load_layer_kinds
|
|
from freetoken.moe.cpu_executor import _format_label
|
|
|
|
methods = offload_layer_methods(self.model)
|
|
if methods is None:
|
|
return None
|
|
gpu_name, gpu_uuid = _profile_gpu(self.device.index)
|
|
rows = load_layer_kinds(gpu_name=gpu_name, gpu_uuid=gpu_uuid) or {}
|
|
keys, labels = [], []
|
|
for m in methods:
|
|
fmt, opts = m.cpu_spec()
|
|
c = m.cfg
|
|
keys.append(layer_kind_key(fmt, opts, c.hidden, c.local_intermediate, c.top_k, self.cpu_moe_executor.num_threads))
|
|
labels.append(f"{_format_label(fmt, tuple(sorted(opts.items())))} {c.hidden}x{c.local_intermediate}")
|
|
return keys, labels, rows
|
|
|
|
def _init_cpu_moe_executor(self, config: EngineConfig, cache, layers) -> None:
|
|
"""Build the persistent CPU MoE executor (decode-time expert compute).
|
|
|
|
Must run before CUDA graph capture: the worker pool has to be live for the
|
|
eager warmup forward, and the pinned IO block must be stable for the captured
|
|
nodes.
|
|
"""
|
|
from freetoken.moe.cpu_executor import CpuMoeExecutor
|
|
|
|
sample = layers[0]
|
|
required = ("top_k", "activation", "apply_router_weight_on_input")
|
|
if not all(hasattr(sample, attr) for attr in required):
|
|
raise NotImplementedError(
|
|
"CPU MoE backend is not yet supported for this model architecture "
|
|
f"(MoE layer {type(sample).__name__} is missing {required})."
|
|
)
|
|
# Decode batches never exceed max_running_req, but CUDA-graph padding can
|
|
# round a batch up to the largest captured size; cover both.
|
|
max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)
|
|
max_tokens *= config.decode_rows_per_req
|
|
# Match GraphRunner's explicit-size precedence, including an empty list.
|
|
if config.cuda_graph_bs is not None:
|
|
graph_enabled = bool(config.cuda_graph_bs)
|
|
else:
|
|
graph_enabled = config.cuda_graph_max_bs is None or config.cuda_graph_max_bs > 0
|
|
by_id = sorted(layers, key=lambda layer: layer.layer_id)
|
|
layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]
|
|
executor = CpuMoeExecutor(
|
|
cache,
|
|
top_k=sample.top_k,
|
|
activation=sample.activation,
|
|
apply_router_weight_on_input=sample.apply_router_weight_on_input,
|
|
num_threads=config.moe_cpu_threads,
|
|
max_tokens=max_tokens,
|
|
device=self.device,
|
|
activation_dtype=self.dtype,
|
|
graph_enabled=graph_enabled,
|
|
swiglu_alpha=float(sample.alpha),
|
|
swiglu_limit=sample.limit,
|
|
layer_formats=layer_formats,
|
|
)
|
|
cache.set_cpu_executor(executor)
|
|
self.cpu_moe_executor = executor
|
|
if graph_enabled:
|
|
_disable_unsafe_cpu_moe_graphs(config, executor, self.dtype, self.tp_cpu_group)
|
|
|
|
def _sync_get_memory(self) -> Tuple[int, int]:
|
|
"""Get the min and max free memory across TP ranks."""
|
|
torch.cuda.synchronize(self.device)
|
|
torch.cuda.empty_cache()
|
|
torch.cuda.reset_peak_memory_stats(self.device)
|
|
free_memory = get_free_memory(self.device)
|
|
free_mem_tensor = torch.tensor([free_memory, -free_memory], device="cpu", dtype=torch.int64)
|
|
torch.distributed.all_reduce(
|
|
free_mem_tensor, op=torch.distributed.ReduceOp.MIN, group=self.tp_cpu_group
|
|
)
|
|
min_free_memory = int(free_mem_tensor[0].item())
|
|
max_free_memory = -int(free_mem_tensor[1].item())
|
|
if max_free_memory - min_free_memory > 2 * 1024 * 1024 * 1024:
|
|
logger.error(
|
|
f"Memory across TP ranks are imbalanced:"
|
|
f" min {mem_GB(min_free_memory)}, max {mem_GB(max_free_memory)}"
|
|
)
|
|
raise RuntimeError("Memory across TP ranks are imbalanced")
|
|
|
|
return min_free_memory, max_free_memory
|
|
|
|
def _target_moe_and_expert_bytes(self, moe_cache_size: int | None, pool_sizes: list[int] | None) -> tuple[int, int]:
|
|
cache = self.moe_offload_cache
|
|
if cache is None:
|
|
return moe_cache_size or 0, 0
|
|
if moe_cache_size is None:
|
|
target_moe = cache.cache_size
|
|
nbytes = sum(c.numel() * c.element_size() for _, c in cache.banks)
|
|
else:
|
|
target_moe = moe_cache_size
|
|
nbytes = self._moe_pool_bytes(pool_sizes)
|
|
# pools of several bank layouts have different slot sizes; round up so target * bytes covers them
|
|
return target_moe, -(-nbytes // target_moe)
|
|
|
|
def _resize_kv_pool(self, config, num_pages: int, num_swa_pages: int | None) -> None:
|
|
# IN-PLACE, identity-preserving: the CacheManager's swa_pool reference, ctx.kv_cache and
|
|
# the model's per-access pool property all keep pointing at THIS pool, which frees its old
|
|
# buffers before allocating the new ones. mark_for_rebind re-binds per-bind scratch on the
|
|
# next forward (graph re-capture); the prefix tree + page bookkeeping reset is the
|
|
# scheduler's generic cache_manager.rebuild.
|
|
if self.kv_cache.needs_rebind_on_rebuild:
|
|
self.model.mark_for_rebind()
|
|
self.kv_cache.rebuild_from_config(config, num_pages, num_swa_pages=num_swa_pages)
|
|
self.num_pages = num_pages
|
|
|
|
def _refresh_seq_state(self, config) -> None:
|
|
num_tokens = self.num_pages * config.page_size
|
|
self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)
|
|
if aligned_max_seq_len != self.page_table.shape[1]:
|
|
# max_seq_len changed (e.g. KV grew past the startup token budget); the page table
|
|
# columns must track it or new requests would index out of bounds. The scheduler
|
|
# re-points its managers to engine.page_table on a num_pages rebuild.
|
|
self.ctx.page_table = self.page_table = torch.zeros(
|
|
(config.max_running_req + 1, aligned_max_seq_len),
|
|
dtype=torch.int32,
|
|
device=self.device,
|
|
)
|
|
self.page_table[self.dummy_req.table_idx].fill_(num_tokens)
|
|
self.kv_cache.attach_page_table(self.page_table)
|
|
|
|
@torch.inference_mode()
|
|
def rebuild_runtime_cache(
|
|
self,
|
|
*,
|
|
moe_cache_size: int | None = None,
|
|
num_pages: int | None = None,
|
|
num_mamba_slots: int | None = None,
|
|
num_swa_pages: int | None = None,
|
|
) -> None:
|
|
"""Idle-only in-place resize of the MoE slot cache, KV page pool, GDN (mamba) state pool,
|
|
and/or the window pool (num_swa_pages: an absolute pinned window), followed by CUDA-graph
|
|
re-capture. Does NOT reload weights or host expert banks. The caller (scheduler) must
|
|
guarantee no in-flight prefill/decode.
|
|
"""
|
|
config = self.config
|
|
if (moe_cache_size is None and num_pages is None and num_mamba_slots is None
|
|
and num_swa_pages is None):
|
|
return
|
|
|
|
# 0a. Geometry prevalidation BEFORE any destructive free. An invalid target (moe
|
|
# slots on a model with no offload cache, moe below num_experts / above the
|
|
# marlin cap, non-positive pages, or too few GDN slots to run) must reject
|
|
# recoverably with the old cache intact -- NOT after teardown, which would
|
|
# leave the server unable to serve. These checks are model-agnostic.
|
|
if moe_cache_size is not None:
|
|
if self.moe_offload_cache is None:
|
|
raise CacheRebuildRejected(
|
|
"moe_cache_size requested but this model has no MoE offload cache"
|
|
)
|
|
try:
|
|
# a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead
|
|
batch = config.max_running_req * config.decode_rows_per_req if self.moe_offload_cache.decode_target == "gpu" else 1
|
|
moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)
|
|
self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)
|
|
except ValueError as e:
|
|
raise CacheRebuildRejected(str(e)) from e
|
|
if num_pages is not None and num_pages <= 0:
|
|
raise CacheRebuildRejected(f"num_pages must be positive, got {num_pages}")
|
|
if num_mamba_slots is not None:
|
|
if self.linear_state_pool is None:
|
|
raise CacheRebuildRejected(
|
|
"num_mamba_slots requested but this model has no GDN state pool"
|
|
)
|
|
# num_mamba_slots is the USABLE slot count (what the user sets and the status bar
|
|
# shows); the pool also reserves a padding sink (slot 0), so the physical pool is
|
|
# num_mamba_slots + 1. _linear_pool_min_slots is the physical floor -> usable - 1.
|
|
min_usable = _linear_pool_min_slots(config) - 1
|
|
if num_mamba_slots < min_usable:
|
|
raise CacheRebuildRejected(
|
|
f"num_mamba_slots {num_mamba_slots} is below the minimum {min_usable} "
|
|
f"(non-evictable working set for max_running_req={config.max_running_req}) "
|
|
f"needed to run; admission would deadlock"
|
|
)
|
|
if num_swa_pages is not None:
|
|
# An absolute window pin for the radix-SWA window pool (Gemma) or the DSV4 window tier;
|
|
# meaningless for dense/MHA models and the naive SWA path (concurrency x window).
|
|
if not _supports_swa_ratio(config):
|
|
raise CacheRebuildRejected(
|
|
"num_swa_pages requested but this model has no window pool "
|
|
"(needs DSV4 or a sliding-window model with --cache-type radix)"
|
|
)
|
|
if num_swa_pages <= 0:
|
|
raise CacheRebuildRejected(
|
|
f"num_swa_pages must be positive, got {num_swa_pages}"
|
|
)
|
|
|
|
# 0b. Pool-family budget fit-check BEFORE any destructive free: an unfit geometry
|
|
# must reject (recoverable) so the old caches stay intact and serving continues,
|
|
# rather than freeing and then OOMing into permanent failure. The engine supplies
|
|
# the memory account; the pool answers whether its target geometry fits.
|
|
target_moe, per_expert_bytes = self._target_moe_and_expert_bytes(
|
|
moe_cache_size, moe_pool_sizes if moe_cache_size is not None else None
|
|
)
|
|
# Price the sibling GDN state pool at ITS target (physical slots = usable + padding
|
|
# sink) and hand the bytes in -- the KV pool only budgets its own tiers.
|
|
target_mamba = (
|
|
num_mamba_slots + 1
|
|
if num_mamba_slots is not None
|
|
else (self.linear_state_pool.num_slots if self.linear_state_pool is not None else None)
|
|
)
|
|
spec_fixed, draft_page = self._spec_budget(config)
|
|
self.kv_cache.validate_rebuild(
|
|
config, num_pages=num_pages,
|
|
num_swa_pages=num_swa_pages, target_moe=target_moe,
|
|
per_expert_bytes=per_expert_bytes, baseline_free=self._baseline_free,
|
|
weights_bytes=self._weights_bytes, current_num_pages=self.num_pages,
|
|
extra_fixed_bytes=(state_pool_bytes(config, target_mamba) if target_mamba is not None else 0) + spec_fixed,
|
|
extra_page_bytes=draft_page,
|
|
extra_note=(
|
|
f", mamba={target_mamba - 1} slots" if target_mamba is not None else ""
|
|
),
|
|
)
|
|
|
|
torch.cuda.synchronize(self.device)
|
|
# Preserve the CUDA-graph batch-size set resolved at startup. The auto heuristic keys
|
|
# off free memory, which is far smaller now that the caches are resident (post-cache
|
|
# free << startup pre-load free), so re-deriving it here would silently drop large
|
|
# batch sizes after the first rebuild. Reusing the already-resolved list keeps the
|
|
# captured coverage identical (the fit-check above guarantees the graph headroom fits).
|
|
prior_graph_bs = self.graph_runner.graph_bs_list
|
|
# Point of no return for the scheduler's rollback logic: from here the live graphs and
|
|
# pools start being freed. A failure BEFORE this flag flips leaves the engine serving
|
|
# untouched (no rollback needed); after it, only a rebuild restores service.
|
|
self.rebuild_teardown_started = True
|
|
# 1. Tear down CUDA graphs + backend capture scratch (free-before-alloc).
|
|
self.attn_backend.reset_capture()
|
|
if self.linear_backend is not None:
|
|
self.linear_backend.reset_capture()
|
|
self.graph_runner.destroy_cuda_graphs()
|
|
if self.spec is not None:
|
|
# the draft KV is keyed by the target's slots, so it can stay only when the target KV pool does
|
|
self.spec.detach(keep_kv=num_pages is None and num_swa_pages is None)
|
|
# 2. Resize caches in place (each frees its old GPU tensors before allocating).
|
|
# Pin the new window first (validated above) so any KV-pool rebuild below sizes the window
|
|
# to it (_dsv4_pool_sizes / _swa_paged_num_tokens read config.swa_num_pages_override).
|
|
# frozen EngineConfig — mutate in place like the moe_cache_size path; `config.x = y` raises
|
|
# FrozenInstanceError, which here aborts the rebuild after the CUDA graphs are gone (→ 503).
|
|
if num_swa_pages is not None:
|
|
object.__setattr__(config, "swa_num_pages_override", num_swa_pages)
|
|
if moe_cache_size is not None:
|
|
assert self.moe_offload_cache is not None, "no MoE offload cache to resize"
|
|
self.moe_offload_cache.rebuild(moe_cache_size, moe_pool_sizes)
|
|
if num_pages is not None:
|
|
# sets self.num_pages (rebuilds KV + window)
|
|
self._resize_kv_pool(config, num_pages, num_swa_pages)
|
|
elif num_swa_pages is not None:
|
|
# Window-only change: no page-count change, but re-derive the window pool at the new
|
|
# pin against the CURRENT page count. This re-allocs the same-size full pool and
|
|
# the resized window, both inside the pool's own rebuild_from_config.
|
|
self._resize_kv_pool(config, self.num_pages, num_swa_pages)
|
|
if num_mamba_slots is not None:
|
|
# Reallocate the GDN state pool (frees old tensors first). Must sit between graph
|
|
# teardown and re-capture so the recaptured graphs bind the new state tensors.
|
|
# +1 for the reserved padding sink: num_mamba_slots is the usable count.
|
|
self.linear_state_pool.rebuild(num_mamba_slots + 1)
|
|
# 3. Refresh max_seq_len (+ generic page table) for the new token budget.
|
|
self._refresh_seq_state(config)
|
|
aligned_max_seq_len = self.page_table.shape[1]
|
|
# 4. Re-capture CUDA graphs against the new tensors (reset_capture above re-armed
|
|
# the backend; _sync_get_memory empties the cache so freed memory is reclaimed).
|
|
gc.collect()
|
|
free_min = self._sync_get_memory()[0]
|
|
if self.spec is not None:
|
|
self.spec.attach(self)
|
|
# reuse the startup-resolved bs set (see above)
|
|
self.graph_runner = self._make_graph_runner(prior_graph_bs, free_min, aligned_max_seq_len)
|
|
|
|
def _make_graph_runner(self, cuda_graph_bs, free_memory: int, max_seq_len: int) -> GraphRunner:
|
|
"""The target's graphs: the decode family, or with speculative decoding the verify family in its place."""
|
|
config = self.config
|
|
rows = config.decode_rows_per_req
|
|
verify = {}
|
|
if rows > 1:
|
|
# every decode step is a verify; the verify scratch has max_running_req rows
|
|
cuda_graph_bs = [
|
|
bs for bs in _determine_cuda_graph_bs(cuda_graph_bs, config.cuda_graph_max_bs, free_memory)
|
|
if bs <= config.max_running_req
|
|
]
|
|
verify = dict(rows_per_req=rows, phase="verify", batch_hook=self._prepare_verify_batch)
|
|
runner = GraphRunner(
|
|
stream=self.stream,
|
|
device=self.device,
|
|
forward=self.model.forward,
|
|
attn_backend=self.attn_backend,
|
|
cuda_graph_bs=cuda_graph_bs,
|
|
cuda_graph_max_bs=config.cuda_graph_max_bs,
|
|
free_memory=free_memory,
|
|
max_seq_len=max_seq_len,
|
|
dummy_req=self.dummy_req,
|
|
moe_offload_cache=self.moe_offload_cache,
|
|
mrope=config.model_config.model_is_mrope,
|
|
out_dtype=torch.float32,
|
|
linear_backend=self.linear_backend,
|
|
**verify,
|
|
)
|
|
if self.spec is not None:
|
|
self.spec.capture_graphs(runner.graph_bs_list, runner.graph_pool)
|
|
return runner
|
|
|
|
def _prepare_verify_batch(self, batch: Batch) -> None:
|
|
batch.features = self.spec.feature_sink(batch)
|
|
|
|
def commit_verify(self, reqs, counts, out: torch.Tensor) -> None:
|
|
"""Commit each verified request's first ``counts`` tokens (row i of ``out`` is reqs[i]) to the target's per-request state."""
|
|
if self.linear_state_pool is not None and self.linear_state_pool.verify_scratch is not None:
|
|
self.linear_state_pool.commit_verify([gdn_slot(r) for r in reqs], counts)
|
|
for req, n, tokens in zip(reqs, counts, out):
|
|
if req.output_token_counts is not None:
|
|
req.output_token_counts.scatter_add_(0, tokens[:n], req.output_token_counts.new_ones(n))
|
|
|
|
def _spec_budget(self, config) -> tuple[int, int]:
|
|
"""(fixed bytes, draft KV bytes per page) speculative decoding adds next to the KV pool."""
|
|
if getattr(self, "spec", None) is None:
|
|
return 0, 0
|
|
fixed, per_token = self.spec.budget(config)
|
|
return fixed, per_token * config.page_size
|
|
|
|
def forward_logits(self, batch: Batch) -> torch.Tensor:
|
|
"""The batch's logits from its captured graph, or eagerly when none fits."""
|
|
use_graph = self.graph_runner.can_use_cuda_graph(batch)
|
|
with self.ctx.forward_batch(batch), self.model.forward_host_ctx(batch, use_graph):
|
|
logits = self.graph_runner.replay(batch) if use_graph else self.model.forward()
|
|
if self.cpu_moe_executor is not None:
|
|
# One pinned read: surfaces a fired flag-handshake watchdog (dead coordinator
|
|
# -> stale expert outputs) as a loud error instead of silent corruption.
|
|
self.cpu_moe_executor.raise_if_unhealthy()
|
|
return logits
|
|
|
|
def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput:
|
|
assert torch.cuda.current_stream() == self.stream
|
|
if batch.mm_gather_plan:
|
|
self._run_mm_encoder(batch)
|
|
spec_prefill = self.spec is not None and batch.is_prefill
|
|
if spec_prefill:
|
|
batch.features = self.spec.feature_sink(batch)
|
|
logits = self.forward_logits(batch)
|
|
|
|
for req in batch.reqs:
|
|
req.complete_one()
|
|
|
|
batch_logits = logits[: batch.size]
|
|
next_tokens_gpu = self.sampler.sample(batch_logits, args).to(torch.int32)
|
|
next_tokens_cpu = next_tokens_gpu.to("cpu", non_blocking=True)
|
|
copy_done_event = torch.cuda.Event()
|
|
copy_done_event.record(self.stream)
|
|
if spec_prefill:
|
|
# after the D2H is queued: the drain waits for the copy only, while the draft's work overlaps it
|
|
self.spec.on_prefill(batch, next_tokens_gpu)
|
|
return ForwardOutput(next_tokens_gpu, next_tokens_cpu, copy_done_event)
|
|
|
|
@torch.inference_mode()
|
|
def _warmup_prefill(self) -> None:
|
|
"""Compile the Triton prefill path before the first real request.
|
|
|
|
Decode CUDA graph capture warms the decode path, but the first prefill
|
|
can still pay Triton/cublas setup costs. Use the dummy request row and
|
|
restore it afterwards so padded decode graph replay keeps using the
|
|
dedicated dummy KV slot.
|
|
"""
|
|
if self.max_seq_len < 2:
|
|
return
|
|
|
|
warmup_lens = [min(80, self.max_seq_len)]
|
|
if self.max_seq_len >= 128:
|
|
warmup_lens.append(128)
|
|
warmup_lens = sorted({length for length in warmup_lens if length >= 2})
|
|
if not warmup_lens:
|
|
return
|
|
|
|
dummy_row = self.page_table[self.dummy_req.table_idx]
|
|
dummy_slot = int(dummy_row[0].item())
|
|
started = torch.cuda.Event(enable_timing=True)
|
|
ended = torch.cuda.Event(enable_timing=True)
|
|
started.record(self.stream)
|
|
try:
|
|
for length in warmup_lens:
|
|
dummy_row[:length] = torch.arange(
|
|
length, dtype=torch.int32, device=self.device
|
|
)
|
|
warm_req = Req(
|
|
input_ids=torch.zeros(length, dtype=torch.int32, device="cpu"),
|
|
table_idx=self.dummy_req.table_idx,
|
|
cached_len=0,
|
|
output_len=1,
|
|
uid=-1,
|
|
sampling_params=None, # type: ignore[arg-type]
|
|
cache_handle=None, # type: ignore[arg-type]
|
|
)
|
|
batch = Batch(reqs=[warm_req], phase="prefill")
|
|
batch.padded_reqs = batch.reqs
|
|
batch.input_ids = torch.zeros(length, dtype=torch.int32, device=self.device)
|
|
batch.positions = torch.arange(length, dtype=torch.int32, device=self.device)
|
|
if self.config.model_config.model_is_mrope:
|
|
batch.mrope_positions = (
|
|
batch.positions.unsqueeze(0).expand(3, -1).contiguous()
|
|
)
|
|
batch.out_loc = dummy_row[:length]
|
|
self.attn_backend.prepare_metadata(batch)
|
|
with self.ctx.forward_batch(batch):
|
|
self.model.forward()
|
|
finally:
|
|
dummy_row.fill_(dummy_slot)
|
|
if self.moe_offload_cache is not None:
|
|
self.moe_offload_cache.reset()
|
|
ended.record(self.stream)
|
|
torch.cuda.synchronize(self.device)
|
|
logger.info_rank0(
|
|
f"Prefill warmup complete for lengths {warmup_lens} "
|
|
f"in {started.elapsed_time(ended) / 1000.0:.3f} s"
|
|
)
|
|
|
|
def shutdown(self) -> None:
|
|
self.graph_runner.destroy_cuda_graphs()
|
|
torch.distributed.destroy_process_group()
|
|
destroy_distributed()
|
|
|
|
|
|
def _profile_gpu(index: "int | None" = None) -> Tuple[str | None, str | None]:
|
|
"""(name, uuid) of visible device ``index`` (default: the current, i.e. bound, device); (None, None) without CUDA."""
|
|
if not torch.cuda.is_available():
|
|
return None, None
|
|
ident = gpu_identity(torch.cuda.current_device() if index is None else index)
|
|
return ident["name"], ident["uuid"]
|
|
|
|
|
|
def _is_unified_memory_gpu(index: "int | None" = None) -> bool:
|
|
"""True when the GPU has no separate device memory (cudaDevAttrIntegrated): host
|
|
banks and the GPU slot cache are the same DRAM, so the offload family's pinned
|
|
staging + slot gather are DRAM-to-DRAM copies with no PCIe link to hide behind.
|
|
|
|
The attribute is reliable on true-UMA parts (Jetson, GB10/DGX Spark) but not on
|
|
C2C-linked discrete-HBM parts (GH200 reports integrated=0 despite coherent CPU
|
|
memory), and it has only been verified on GB10 so far --
|
|
FREETOKEN_UNIFIED_MEMORY=0/1 overrides the probe where the attribute lies."""
|
|
env = os.environ.get("FREETOKEN_UNIFIED_MEMORY")
|
|
if env is not None:
|
|
return env.strip().lower() not in {"0", "false", "no", "off"}
|
|
if not torch.cuda.is_available():
|
|
return False
|
|
try:
|
|
dev = torch.cuda.current_device() if index is None else index
|
|
return bool(torch.cuda.get_device_properties(dev).is_integrated)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4", "gguf")
|
|
|
|
|
|
def _fused_resident_ok(model_config) -> bool:
|
|
"""Whether the resident ('fused') MoE path can hold this model's experts; DeepSeek-V4 experts stay offload."""
|
|
if getattr(model_config, "expert_quant", "none") not in _RESIDENT_EXPERT_QUANTS:
|
|
return False
|
|
return getattr(model_config, "moe_weight_format", None) in (None, "bf16", "mxfp4")
|
|
|
|
|
|
def _ensure_expandable_segments() -> None:
|
|
"""Default the CUDA allocator to expandable segments.
|
|
|
|
The motivating case is the offload prefill, which repeatedly dequantizes
|
|
variable-sized NVFP4 expert blocks to BF16 (a different size per layer as the
|
|
active-expert count varies). Under that alloc/free churn the default caching
|
|
allocator fragments badly -- reserved memory can balloon far past the actual peak
|
|
allocation (observed ~78GiB reserved for a <30GiB working set).
|
|
``expandable_segments`` lets freed regions of any size be reused, keeping
|
|
reserved ~= allocated, so it is applied to every run, not just offload ones.
|
|
|
|
Env vars are parsed once at import and ignored if set afterwards, so we apply the
|
|
setting via the runtime API instead. Must run before the first CUDA allocation (the
|
|
caller guarantees CUDA is not yet initialized). Any user-provided allocator config
|
|
is respected and left untouched.
|
|
"""
|
|
if os.environ.get("PYTORCH_ALLOC_CONF") or os.environ.get("PYTORCH_CUDA_ALLOC_CONF"):
|
|
return
|
|
try:
|
|
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
|
|
except Exception as exc: # pragma: no cover - depends on torch build
|
|
logger.info_rank0(f"Could not enable expandable_segments ({exc}); continuing")
|
|
return
|
|
logger.info_rank0("Enabled expandable_segments (override via PYTORCH_ALLOC_CONF)")
|
|
|
|
|
|
def _resolve_cache_type(has_linear_attention: bool, requested: str) -> str:
|
|
# Hybrid GDN models default to the HybridRadixCache (snapshots GDN state at chunk
|
|
# boundaries -> cross-request prefix reuse). An explicit ``--cache-type naive`` opts out
|
|
# to the old no-reuse path (debugging / parity baseline / lower GDN-state memory).
|
|
if has_linear_attention:
|
|
return "naive" if requested == "naive" else "hybrid_radix"
|
|
return requested
|
|
|
|
|
|
def _adjust_dsv41_config(config: EngineConfig, override) -> None:
|
|
"""DeepSeek-V4.1 config reconciliation, the DSV4 policy with the replay knob: sync the
|
|
runtime into ``dsv41_args``, page_size = the window page P, radix -> swa_radix, decode graph
|
|
batches <= max_running_req. Unlike DSV4 the prefill chunk keeps ``max_extend_tokens`` (whole
|
|
window pages): at a 1M ceiling the window pool alone would admit ~40K-token chunks whose
|
|
activations (64 x 512 latent queries per token) do not fit next to the expert cache."""
|
|
model_config = config.model_config
|
|
args = model_config.dsv41_args
|
|
args.max_seq_len = config.max_seq_len
|
|
args.max_batch_size = config.max_running_req + 1 # +1 dummy
|
|
args.swa_decoder_replay = config.swa_decoder_replay
|
|
P = args.window_size
|
|
override("page_size", P)
|
|
logger.info_rank0(
|
|
f"DSV41 KV pages are {P}-token window pages; page_size set to {P}; SWA decoder replay: {args.swa_decoder_replay}"
|
|
+ ("" if args.swa_decoder_replay == "exact" else f" (prefix hits stop {P} tokens before the prompt end)")
|
|
)
|
|
if getattr(config, "cache_type", "radix") != "naive":
|
|
override("cache_type", "swa_radix")
|
|
current = getattr(config, "max_extend_tokens", None) # a SchedulerConfig field
|
|
if current is not None and (extend := max(P, current // P * P)) != current:
|
|
override("max_extend_tokens", extend)
|
|
mr = config.max_running_req
|
|
if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
|
|
logger.warning_rank0(f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds max_running_req {mr}; clamping (larger decode batches never occur).")
|
|
override("cuda_graph_max_bs", mr)
|
|
if config.cuda_graph_bs is not None:
|
|
kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
|
|
if kept != list(config.cuda_graph_bs):
|
|
logger.warning_rank0(f"dropping cuda_graph_bs entries above max_running_req {mr}: {[bs for bs in config.cuda_graph_bs if bs > mr]}")
|
|
override("cuda_graph_bs", kept)
|
|
|
|
|
|
def _adjust_dsv4_config(config: EngineConfig, override) -> None:
|
|
"""DSV4 engine-config reconciliation at config-resolution time (before the pool exists).
|
|
Syncs the resolved runtime config into the opaque ``dsv4_args`` payload, sets
|
|
page_size to the window page P, forces single-chunk prefill, and clamps cuda_graph_bs/max_bs to
|
|
the DSV4 decode batch size.
|
|
"""
|
|
model_config = config.model_config
|
|
model_config.dsv4_args.max_seq_len = config.max_seq_len
|
|
model_config.dsv4_args.max_batch_size = config.max_running_req + 1 # +1 dummy
|
|
# config.swa_full_tokens_ratio is the DSV4 window/full ratio directly (default sizing);
|
|
# a runtime rebuild pins an absolute window via swa_num_pages_override instead.
|
|
# DSV4's KV page IS the P-token window page (window == radix reuse granularity == lcm of
|
|
# the compress ratios), so max_num_tokens = num_pages * page_size holds like every model.
|
|
P = model_config.dsv4_args.window_size
|
|
override("page_size", P)
|
|
logger.info_rank0(f"DSV4 KV pages are {P}-token window pages; page_size set to {P}")
|
|
# The generic CacheManager materializes DSV4 'radix' as the shared SWARadixCache (is_swa);
|
|
# 'naive' stays naive with the pool's swa currency riding swa_paged.
|
|
if getattr(config, "cache_type", "radix") != "naive":
|
|
override("cache_type", "swa_radix")
|
|
# 'radix' (SWARadixCache on the full-loc currency, carry-aware re-prefill) is the default and is
|
|
# honored, as is an explicit 'naive'. Don't let max_extend_tokens force a second chunk within
|
|
# one prompt (the pool's prefill_chunk_budget still chunks prompts larger than the window
|
|
# pool); prefill batches ragged (bs>=1), each segment resuming from its own cached_len.
|
|
if getattr(config, "max_extend_tokens", 0) < config.max_seq_len:
|
|
override("max_extend_tokens", config.max_seq_len)
|
|
|
|
# DSV4 decode batches at most max_running_req rows; its full-loc snapshot is sized to that,
|
|
# so a graph bs above it would exceed the backend's captured snapshot rows. Clamp any
|
|
# oversized explicit list / max_bs here (before GraphRunner ever sees it).
|
|
mr = config.max_running_req
|
|
if config.cuda_graph_max_bs is not None and config.cuda_graph_max_bs > mr:
|
|
logger.warning_rank0(
|
|
f"cuda_graph_max_bs {config.cuda_graph_max_bs} exceeds DSV4 max_running_req {mr}; "
|
|
"clamping to max_running_req (larger decode batches never occur)."
|
|
)
|
|
override("cuda_graph_max_bs", mr)
|
|
if config.cuda_graph_bs is not None:
|
|
kept = [bs for bs in config.cuda_graph_bs if bs <= mr]
|
|
if kept != list(config.cuda_graph_bs):
|
|
dropped = [bs for bs in config.cuda_graph_bs if bs > mr]
|
|
logger.warning_rank0(
|
|
f"dropping cuda_graph_bs entries {dropped} above DSV4 max_running_req {mr} "
|
|
"(larger decode batches never occur)."
|
|
)
|
|
override("cuda_graph_bs", kept)
|
|
|
|
|
|
def _parse_cpu_layers_spec(spec: str, num_moe_layers: int) -> frozenset[int]:
|
|
"""Parse ``--moe-cpu-layers``: an explicit MoE-layer id list (``"3,7,11"``), a count
|
|
(``"8"`` -> 8 layers evenly strided across depth), or a fraction (``"0.5"``). Ids are
|
|
indices into the MoE layers, ``[0, num_moe_layers)``."""
|
|
s = spec.strip()
|
|
if not s:
|
|
return frozenset()
|
|
if "," in s:
|
|
ids = {int(x) for x in s.split(",") if x.strip()}
|
|
for i in ids:
|
|
if not 0 <= i < num_moe_layers:
|
|
raise ValueError(
|
|
f"--moe-cpu-layers id {i} out of range [0, {num_moe_layers})"
|
|
)
|
|
return frozenset(ids)
|
|
if "." in s:
|
|
frac = float(s)
|
|
if not 0.0 <= frac <= 1.0:
|
|
raise ValueError(f"--moe-cpu-layers fraction {frac} must be in [0, 1]")
|
|
k = round(frac * num_moe_layers)
|
|
else:
|
|
k = int(s)
|
|
if not 0 <= k <= num_moe_layers:
|
|
raise ValueError(f"--moe-cpu-layers count {k} must be in [0, {num_moe_layers}]")
|
|
# k layers spread evenly across depth (frozenset dedups any rounding collisions;
|
|
# k == 0 yields an empty range, hence an empty set).
|
|
return frozenset(round(i * num_moe_layers / k) for i in range(k))
|
|
|
|
|
|
def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
|
|
"""MoE layer ids whose decode runs on the CPU executor.
|
|
|
|
``--moe-strategy cpu`` -> every layer. ``--moe-strategy offload`` + ``--moe-cpu-layers``
|
|
-> the parsed subset, or the pin-budget pick for ``auto`` (the rest stay on the GPU offload/PCIe path). Otherwise none.
|
|
"""
|
|
if config.moe_strategy == "cpu":
|
|
return frozenset(range(num_moe_layers))
|
|
spec = config.moe_cpu_layers
|
|
if not spec or not is_offload_moe_strategy(config.moe_strategy):
|
|
return frozenset()
|
|
if spec.strip() == "auto":
|
|
return _auto_cpu_layers(config, num_moe_layers, reserved=reserved, method=method)
|
|
return _parse_cpu_layers_spec(spec, num_moe_layers)
|
|
|
|
|
|
def _decode_target(config: EngineConfig) -> str:
|
|
"""Where routed experts decode, from the flags alone: hybrid co-compute, the CPU executor for some or all layers, or the GPU slot cache."""
|
|
if config.moe_strategy == "hybrid":
|
|
return "hybrid"
|
|
if config.moe_strategy == "cpu" or (config.moe_cpu_layers and is_offload_moe_strategy(config.moe_strategy)):
|
|
return "cpu"
|
|
return "gpu"
|
|
|
|
|
|
# expert activations the CPU MoE executor supports (csrc ActKind)
|
|
_CPU_MOE_ACTS = (
|
|
"silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
|
|
"swiglu_clamp",
|
|
)
|
|
|
|
|
|
def _cpu_moe_executor_viable(model_config) -> bool:
|
|
"""Whether an automatic CPU-decode decision may target the CPU MoE executor.
|
|
|
|
A default boot must degrade to GPU offload instead of crashing in CpuMoeExecutor after the whole load; explicit cpu/hybrid/--moe-cpu-layers picks still fail loudly."""
|
|
from freetoken.moe.cpu_executor import compiled_extension_supports
|
|
|
|
try:
|
|
from freetoken.kernel import _cpu_moe
|
|
except ImportError:
|
|
return False
|
|
act = getattr(model_config, "hidden_act", "silu")
|
|
moe_wfmt = getattr(model_config, "moe_weight_format", None)
|
|
if act not in _CPU_MOE_ACTS and moe_wfmt != "mxfp4":
|
|
return False
|
|
if moe_wfmt != "mxfp4" and not compiled_extension_supports(act):
|
|
return False
|
|
expert_quant = getattr(model_config, "expert_quant", "none")
|
|
fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")
|
|
if fmt == "gguf":
|
|
from freetoken.kernel import ggml_cpu
|
|
|
|
return fmt in _cpu_moe.weight_formats() and ggml_cpu.select() is not None
|
|
return fmt == "mxfp4" or fmt in _cpu_moe.weight_formats()
|
|
|
|
|
|
def _pin_budget_bytes(reserved: int = 0) -> int | None:
|
|
"""Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).
|
|
|
|
WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table)."""
|
|
if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"):
|
|
cap = int(float(env) * 2**30)
|
|
elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag
|
|
return None
|
|
else:
|
|
cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)
|
|
return max(0, cap - reserved)
|
|
|
|
|
|
def _bank_bytes(config: EngineConfig, method=None, layer_methods=None) -> int | None:
|
|
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
|
|
|
|
return ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config, method, layer_methods)
|
|
|
|
|
|
def _pin_hint(reserved: int) -> str:
|
|
if _pin_budget_bytes(reserved) is None:
|
|
return "the expert banks need more page-locked host RAM than this host has; free host RAM or serve a smaller model"
|
|
return (
|
|
"pass --moe-cpu-layers auto to lock the layers over the pin budget for CPU decode, "
|
|
"or --moe-cpu-layers <count|fraction|ids> to choose them yourself"
|
|
)
|
|
|
|
|
|
def _check_pin_budget(config: EngineConfig, *, reserved: int, method=None, layer_methods=None) -> None:
|
|
"""Stop a plain offload boot whose banks exceed a known pin budget before any bank is read."""
|
|
if config.moe_cpu_layers or config.moe_strategy not in ("offload", "hybrid"):
|
|
return
|
|
budget = _pin_budget_bytes(reserved)
|
|
bank_bytes = _bank_bytes(config, method, layer_methods) if budget is not None else None
|
|
if bank_bytes and bank_bytes > budget:
|
|
raise ValueError(
|
|
f"expert banks need {bank_bytes / 2**30:.1f} GiB of pinned host RAM but the pin budget is "
|
|
f"{budget / 2**30:.1f} GiB (WSL caps CUDA pinning; FREETOKEN_PIN_BUDGET_GB overrides); {_pin_hint(reserved)}"
|
|
)
|
|
|
|
|
|
def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, *, reserved: int = 0, method=None) -> frozenset[int]:
|
|
"""Pick CPU (locked) MoE layers for ``--moe-cpu-layers auto``: none while the banks fit the pin budget.
|
|
|
|
Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
|
|
bank_bytes = _bank_bytes(config, method)
|
|
if not bank_bytes:
|
|
return frozenset()
|
|
budget = _pin_budget_bytes(reserved)
|
|
if budget is None or bank_bytes <= budget:
|
|
return frozenset()
|
|
if not _cpu_moe_executor_viable(config.model_config):
|
|
raise ValueError(
|
|
f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB exceed the "
|
|
f"pin budget {budget / 2**30:.2f} GiB, but the CPU MoE executor cannot "
|
|
f"serve this model (see --moe-strategy cpu requirements)"
|
|
)
|
|
n = min(num_moe_layers, math.ceil(num_moe_layers * (1 - budget / bank_bytes)))
|
|
head = (n + 1) // 2
|
|
ids = frozenset(range(head)) | frozenset(range(num_moe_layers - (n - head), num_moe_layers))
|
|
logger.info_rank0(
|
|
f"--moe-cpu-layers auto: banks {bank_bytes / 2**30:.2f} GiB > pin budget "
|
|
f"{budget / 2**30:.2f} GiB; locking {n} head+tail MoE layers for CPU decode "
|
|
f"({sorted(ids)})"
|
|
)
|
|
return ids
|
|
|
|
|
|
# MoE-only knobs and the value each resolves to on a dense model. moe_strategy is handled
|
|
# separately (its dense value is 'fused', but 'auto' resolves there without a warning).
|
|
_DENSE_MOE_SETTINGS = {
|
|
"moe_cache_size": 0,
|
|
"moe_cache_rate": None,
|
|
"moe_cache_auto": False,
|
|
"moe_cache_mem": None,
|
|
"moe_cache_scope": "unified",
|
|
"moe_cache_split": "equal",
|
|
"moe_cpu_layers": None,
|
|
"moe_cpu_threads": 0,
|
|
"moe_hybrid_max_fetch": -1,
|
|
"moe_locked_fetch": -1.0,
|
|
"moe_locked_max_fetch": 8,
|
|
"moe_prefill_overlap": True,
|
|
"moe_prefill_hit_d2d": False,
|
|
"expert_load": "auto",
|
|
}
|
|
|
|
|
|
def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> QuantBackend:
|
|
"""--quant-backend with an FTW checkpoint's packed expert kernel filled in where the flag leaves that table automatic.
|
|
|
|
An entry that names another kernel is refused here, before any bank is read."""
|
|
from freetoken.checkpoint.ftw import ftw_quant_format
|
|
from freetoken.moe.legacy_format import kind_kernel_for
|
|
|
|
with _weight_load_context():
|
|
fmt = ftw_quant_format(model_path) if model_path else None
|
|
if fmt is None:
|
|
return quant_backend
|
|
try:
|
|
kind, kernel = kind_kernel_for(fmt)
|
|
except KeyError:
|
|
return quant_backend
|
|
requested = quant_backend.select(LayerKind.MOE, kind)
|
|
if requested == kernel:
|
|
return quant_backend
|
|
if requested != "auto":
|
|
raise ValueError(
|
|
f"the FTW checkpoint's expert banks were packed for {kind} / {kernel} but --quant-backend asks for "
|
|
f"{requested}; drop the entry, or reconvert with ft checkpoint --quant-backend moe.{kind}={requested}"
|
|
)
|
|
return QuantBackend(quant_backend.items + (((LayerKind.MOE, kind), kernel),))
|
|
|
|
|
|
def offload_layer_methods(model) -> list | None:
|
|
"""The expert method of each offload MoE layer of ``model``, by MoE-layer id; None for models whose MoE layers carry none (GGUF)."""
|
|
layers = [l for l in iter_offload_moe_layers(model) if getattr(l, "quant_method", None) is not None]
|
|
if not layers:
|
|
return None
|
|
layers.sort(key=lambda layer: layer.layer_id)
|
|
assert [layer.layer_id for layer in layers] == list(range(len(layers))), [layer.layer_id for layer in layers]
|
|
return [layer.quant_method for layer in layers]
|
|
|
|
|
|
def shared_offload_method(model):
|
|
"""The expert method the offload MoE layers of ``model`` use, or None for models whose MoE layers carry none (GGUF).
|
|
|
|
With experts of several formats (a checkpoint that keeps some layers unquantized) this is the format of the most
|
|
layers; the cache then takes every layer's method (``offload_layer_methods``) and keeps one slot pool per format."""
|
|
from freetoken.moe.expert_banks import method_groups
|
|
|
|
methods = offload_layer_methods(model)
|
|
if methods is None:
|
|
return None
|
|
groups = method_groups(methods)
|
|
found = ", ".join(f"{m.kind} via {m.kernel.name}" + (f" on {len(ids)} layers" if len(groups) > 1 else "") for m, ids in groups)
|
|
logger.info_rank0(f"MoE experts: {found}")
|
|
return groups[0][0]
|
|
|
|
|
|
def _slot_limit(layer_methods) -> int | None:
|
|
"""The tightest slot cap of the layers' expert kernels (a pool never mixes kernels, but the limit is checked per pool against the largest)."""
|
|
limits = [limit for limit in (m.slot_limit() for m in layer_methods) if limit is not None]
|
|
return min(limits) if limits else None
|
|
|
|
|
|
def _check_layer_formats(banks, layer_methods) -> None:
|
|
from freetoken.moe.legacy_format import legacy_format_for
|
|
|
|
want = [legacy_format_for(m.kind, m.kernel.name) for m in layer_methods]
|
|
have = banks.layer_formats or [banks.quant_format] * len(want)
|
|
bad = [layer for layer, (w, h) in enumerate(zip(want, have)) if w != h]
|
|
if bad:
|
|
raise ValueError(
|
|
f"expert banks of MoE layers {bad[:8]} were packed as {[have[l] for l in bad[:8]]} but the model binds "
|
|
f"{[want[l] for l in bad[:8]]}; reconvert the checkpoint or select those kernels"
|
|
)
|
|
|
|
|
|
def _meta_model(config: EngineConfig, decode_target: str):
|
|
from freetoken.models import create_model
|
|
from freetoken.utils.torch_utils import torch_dtype
|
|
|
|
set_quant_backend(_adjust_ftw_quant_backend(config.model_path, QuantBackend.parse(config.quant_backend)))
|
|
object.__setattr__(config.model_config, "moe_strategy", config.moe_strategy)
|
|
object.__setattr__(config.model_config, "decode_target", decode_target)
|
|
with torch.device("meta"), torch_dtype(config.dtype):
|
|
return create_model(config.model_config)
|
|
|
|
|
|
def offload_expert_methods(config: EngineConfig) -> tuple:
|
|
"""``(method, layer_methods)`` of ``config``'s model, from a meta-device build: the most common expert method and,
|
|
for experts of several formats, one per MoE layer (else None).
|
|
|
|
For tools that load expert banks without an engine (the FTW converter); they pack for GPU decode."""
|
|
from freetoken.moe.expert_banks import is_mixed
|
|
|
|
model = _meta_model(config, "gpu")
|
|
layer_methods = offload_layer_methods(model)
|
|
return shared_offload_method(model), (layer_methods if is_mixed(layer_methods) else None)
|
|
|
|
|
|
def meta_offload_layer_methods(config: EngineConfig, decode_target: str) -> list | None:
|
|
"""The expert method of each offload MoE layer of ``config``'s model, from a meta-device build for ``decode_target``; see ``offload_layer_methods``."""
|
|
return offload_layer_methods(_meta_model(config, decode_target))
|
|
|
|
|
|
def offload_expert_method(config: EngineConfig):
|
|
"""The (most common) offload expert method of ``config``'s model; see ``offload_expert_methods``."""
|
|
return offload_expert_methods(config)[0]
|
|
|
|
|
|
def _adjust_config(config: EngineConfig):
|
|
def override(attr: str, value: Any): # this is dangerous, use with caution
|
|
object.__setattr__(config, attr, value)
|
|
|
|
model_config = config.model_config
|
|
single_stream_only = getattr(model_config, "single_stream_only", False)
|
|
is_dsv4 = getattr(model_config, "dsv4_args", None) is not None
|
|
is_dsv41 = getattr(model_config, "dsv41_args", None) is not None
|
|
has_swa_attention = getattr(model_config, "has_swa_attention", False)
|
|
has_linear_attention = getattr(model_config, "has_linear_attention", False)
|
|
is_moe = getattr(model_config, "is_moe", False)
|
|
expert_quant = getattr(model_config, "expert_quant", "none")
|
|
|
|
if not is_moe:
|
|
# A dense model has no routed experts: the MoE knobs are inert, and the offload family
|
|
# is worse than inert -- engine init would build an expert cache for a model that has
|
|
# none and abort startup (weights already resident) on an unrelated expert-source
|
|
# error. Drop them at this one choke point, which the CLI and the programmatic
|
|
# LLM(...) path both pass through. 'auto'/'fused' is the silent dense resolution;
|
|
# anything else was asked for explicitly, so report what is being ignored.
|
|
dropped = [
|
|
f"{name}={getattr(config, name)!r}"
|
|
for name, dense_value in _DENSE_MOE_SETTINGS.items()
|
|
if getattr(config, name, dense_value) != dense_value
|
|
]
|
|
if config.moe_strategy not in ("auto", "fused"):
|
|
dropped.insert(0, f"moe_strategy={config.moe_strategy!r}")
|
|
override("moe_strategy", "fused")
|
|
for name, dense_value in _DENSE_MOE_SETTINGS.items():
|
|
override(name, dense_value)
|
|
if dropped:
|
|
logger.warning_rank0(
|
|
f"{getattr(model_config, 'model_type', 'model')} is a dense model (no routed "
|
|
f"experts); ignoring MoE settings: {', '.join(dropped)}"
|
|
)
|
|
|
|
if single_stream_only:
|
|
# The model runs one sequence at a time: it collapses the batch to one row and the
|
|
# decode CUDA graph is captured at bs=1. Force the runtime knobs so the KV pool, page
|
|
# table and graph capture all stay bs=1.
|
|
if config.max_running_req != 1:
|
|
override("max_running_req", 1)
|
|
if config.cuda_graph_max_bs is None or config.cuda_graph_max_bs >= 1:
|
|
override("cuda_graph_bs", [1])
|
|
override("cuda_graph_max_bs", 1)
|
|
|
|
if config.cuda_graph_max_bs is None:
|
|
override("cuda_graph_max_bs", config.max_running_req)
|
|
|
|
if is_dsv4:
|
|
_adjust_dsv4_config(config, override)
|
|
if is_dsv41:
|
|
_adjust_dsv41_config(config, override)
|
|
|
|
if has_swa_attention:
|
|
# Both SWA cache paths use the global-paged swa pool (page_size==1 only for now).
|
|
if config.page_size != 1:
|
|
raise ValueError(
|
|
f"SWA models currently support only page_size=1, got {config.page_size}."
|
|
)
|
|
# naive keeps cache_type='naive' (NaivePrefixCache, no reuse) on the paged pool (==
|
|
# sglang SWAChunkCache); radix materializes as swa_radix (SWARadixCache, cross-request
|
|
# reuse == sglang SWARadixCache). Both allocate from the same swa pool + free out-of-window.
|
|
if getattr(config, "cache_type", "radix") != "naive":
|
|
if not 0.0 < config.swa_full_tokens_ratio <= 1.0:
|
|
raise ValueError(
|
|
f"swa_full_tokens_ratio must be in (0, 1], got {config.swa_full_tokens_ratio}"
|
|
)
|
|
override("cache_type", "swa_radix")
|
|
|
|
if has_linear_attention:
|
|
override(
|
|
"cache_type",
|
|
_resolve_cache_type(True, getattr(config, "cache_type", "radix")),
|
|
)
|
|
|
|
# Type x backend capability matrix: resolve auto from the per-type priority
|
|
# lists, then validate whatever is now selected (explicit or auto) -- every
|
|
# comma part must serve every required type, with packages/arch available.
|
|
required_attn_types = _required_attn_types(model_config)
|
|
_dtype = getattr(config, "dtype", None) # duck-typed test configs omit it
|
|
if (
|
|
required_attn_types & {AttnType.BSA, AttnType.QSA}
|
|
and _dtype is not None
|
|
and _dtype.itemsize != 2
|
|
):
|
|
# Reject at config time: the BSA/QSA pool's own assert only fires after the
|
|
# model is resident (and not at all under `python -O`).
|
|
raise ValueError(
|
|
f"--dtype {config.dtype}: block-sparse attention serves 16-bit "
|
|
"compute only (the index slab budgets 2 bytes/token); use bfloat16 "
|
|
"or float16."
|
|
)
|
|
if _dtype == torch.float16 and "mxfp8" in (
|
|
getattr(model_config, "attn_quant", "none"),
|
|
getattr(model_config, "dense_quant", "none"),
|
|
):
|
|
# The MXFP8 GEMV folds the pow2-descaled fp8 weight into the activation
|
|
# dtype; fp16's narrow exponent can overflow/flush what bf16 represents
|
|
# exactly, and the combination was never numerically validated.
|
|
raise ValueError(
|
|
"--dtype float16 with MXFP8 resident weights is unsupported (the "
|
|
"W8A16 fold is only validated exact in bfloat16); use bfloat16."
|
|
)
|
|
if _dtype == torch.float16 and getattr(getattr(model_config, "quant", None), "weight_scales", False):
|
|
# GGUF NVFP4 results are stored in the activation dtype before weight_scale_2 (~1e-4) multiplies them, which overflows fp16.
|
|
raise ValueError("--dtype float16 with NVFP4 GGUF weight scales is unsupported (the unscaled results overflow float16); use bfloat16.")
|
|
if config.attention_backend == "auto":
|
|
override(
|
|
"attention_backend",
|
|
_resolve_auto_attention_backend(required_attn_types),
|
|
)
|
|
logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}")
|
|
_validate_attention_backend_choice(config, override, required_attn_types)
|
|
if getattr(config, "speculative_algorithm", None) is not None:
|
|
_adjust_spec_config(config, override, is_dsv4 or is_dsv41 or has_swa_attention)
|
|
|
|
if config.moe_cache_rate is not None:
|
|
total_experts = config.model_config.num_moe_layers * config.model_config.num_experts
|
|
override("moe_cache_size", math.ceil(total_experts * config.moe_cache_rate))
|
|
|
|
# The CPU MoE executor supports the silu/gelu family plus the clamped
|
|
# swigluoai (csrc ActKind; "gpt_oss_swiglu" rides inside the mxfp4 kernel and
|
|
# swigluoai the generic GEMV epilogue). A model with any other expert
|
|
# activation cannot decode on the CPU: reject an explicit cpu/hybrid pick at
|
|
# config time.
|
|
# hidden_act (the dense activation) stands proxy for the expert activation --
|
|
# true for every in-tree model. mxfp4 experts pass regardless: their act runs
|
|
# inside the mxfp4 kernel, not the generic epilogue.
|
|
_cpu_moe_act_ok = getattr(model_config, "hidden_act", "silu") in _CPU_MOE_ACTS or (
|
|
getattr(model_config, "moe_weight_format", None) == "mxfp4"
|
|
)
|
|
if (
|
|
is_moe
|
|
and not _cpu_moe_act_ok
|
|
and (config.moe_strategy in ("cpu", "hybrid") or config.moe_cpu_layers)
|
|
):
|
|
asked = (
|
|
f"--moe-cpu-layers={config.moe_cpu_layers!r}"
|
|
if config.moe_strategy not in ("cpu", "hybrid")
|
|
else f"--moe-strategy {config.moe_strategy!r}"
|
|
)
|
|
raise ValueError(
|
|
f"{asked}: the CPU MoE executor does not support this model's expert "
|
|
f"activation {getattr(model_config, 'hidden_act', None)!r}; drop the flag "
|
|
"and let every layer decode on the GPU offload path instead."
|
|
)
|
|
|
|
if is_moe and config.moe_strategy == "auto":
|
|
# A MoE model always defaults to the offload family: experts stream from pinned host
|
|
# banks into an auto-sized GPU slot cache, which is the only default that serves a model
|
|
# bigger than the GPU. The resident 'fused' path (every expert format but
|
|
# DeepSeek-V4's) is still reachable, but only when asked for explicitly
|
|
# -- auto never picks it, because nothing here knows whether the experts would fit in
|
|
# HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.
|
|
default_backend = "offload"
|
|
# Unified memory (GB10/DGX Spark, Jetson): there is no host/device boundary, so
|
|
# the offload family stages and gathers between two names for the same DRAM (on
|
|
# GB10 this added a measured ~130 s stall to every request, #369). Resident
|
|
# experts are the safe default here, not the risky one: the model and its banks
|
|
# page from the same pool, so the "wrong guess = weight-load OOM" rationale above
|
|
# does not apply. Only formats the resident path
|
|
# can actually hold take this branch; the rest stay on offload as before.
|
|
unified_memory = _is_unified_memory_gpu()
|
|
if unified_memory and _fused_resident_ok(model_config):
|
|
default_backend = "fused"
|
|
logger.info_rank0(
|
|
"Unified-memory GPU detected; auto-selecting 'fused' MoE strategy "
|
|
"(resident experts) instead of offload"
|
|
)
|
|
# the cache sizing flags get their own warning in the 'fused' block below
|
|
inert = [flag for flag, on in (
|
|
(f"--moe-cpu-layers={config.moe_cpu_layers}", config.moe_cpu_layers),
|
|
("--moe-prefill-hit-d2d", config.moe_prefill_hit_d2d),
|
|
("--disable-moe-prefill-overlap", not config.moe_prefill_overlap),
|
|
) if on]
|
|
if inert:
|
|
logger.warning_rank0(
|
|
f"Unified-memory GPU serves MoE experts resident; ignoring {', '.join(inert)} "
|
|
"(use --moe-strategy offload to keep the offload flags)"
|
|
)
|
|
override("moe_cpu_layers", None)
|
|
override("moe_strategy", default_backend)
|
|
logger.info_rank0(f"Auto-selected MoE strategy: {config.moe_strategy}")
|
|
|
|
if (
|
|
is_offload_moe_strategy(config.moe_strategy)
|
|
and config.moe_cache_size <= 0
|
|
and config.moe_cache_rate is None
|
|
and config.moe_cache_mem is None
|
|
and not getattr(config, "moe_cache_auto", False)
|
|
):
|
|
# args.py's "no sizing flag -> default --moe-cache-auto" only fires when the
|
|
# backend is already offload-family at *parse* time. A bare `ft serve <FTW MoE
|
|
# checkpoint>` (no --moe-strategy, no cache flags) still has moe_strategy=="auto" at
|
|
# parse time -- the auto -> offload/cpu/hybrid resolution above is the first point
|
|
# the concrete backend is known, so mirror the same default here: no sizing flag
|
|
# was given, so let the scheduler resolve the cache size from free VRAM instead of
|
|
# failing the _require_offload_cache_size guard with size=0.
|
|
override("moe_cache_auto", True)
|
|
logger.info_rank0(
|
|
"No MoE cache sizing flag given; defaulting to --moe-cache-auto for "
|
|
f"auto-selected strategy {config.moe_strategy!r}"
|
|
)
|
|
|
|
if (
|
|
is_moe
|
|
and config.moe_strategy == "offload"
|
|
and _is_unified_memory_gpu()
|
|
and _fused_resident_ok(model_config)
|
|
):
|
|
# An explicit offload pick is honored, but on unified memory the user is paying
|
|
# for copies between two names for the same DRAM; say so once at config time.
|
|
logger.warning_rank0(
|
|
"--moe-strategy offload on a unified-memory GPU: expert 'streaming' copies "
|
|
"DRAM to DRAM (there is no PCIe link to overlap it with). If the model fits, "
|
|
"--moe-strategy fused avoids the slot-cache machinery entirely."
|
|
)
|
|
|
|
if is_moe and config.moe_strategy == "fused":
|
|
# An explicit 'fused' keeps the experts resident, so there is no slot cache to size. The
|
|
# sizing flags no longer redirect the backend, so ignore them here and say so -- the
|
|
# geometry the user asked for is what runs. Report the flag actually passed: --moe-cache-
|
|
# rate was already folded into moe_cache_size above, and the sizing flags are mutually exclusive.
|
|
if config.moe_cache_rate:
|
|
inert = f"--moe-cache-rate={config.moe_cache_rate}"
|
|
elif config.moe_cache_size:
|
|
inert = f"--moe-cache-size={config.moe_cache_size}"
|
|
elif config.moe_cache_mem is not None:
|
|
inert = f"--moe-cache-mem={config.moe_cache_mem}"
|
|
elif getattr(config, "moe_cache_auto", False):
|
|
inert = "--moe-cache-auto"
|
|
else:
|
|
inert = None
|
|
if inert:
|
|
logger.warning_rank0(
|
|
f"MoE backend 'fused' keeps its experts resident; ignoring {inert} "
|
|
"(use --moe-strategy offload to serve experts from a slot cache)"
|
|
)
|
|
override("moe_cache_size", 0)
|
|
override("moe_cache_rate", None)
|
|
override("moe_cache_mem", None)
|
|
override("moe_cache_auto", False)
|
|
|
|
if is_moe and config.moe_strategy == "cpu":
|
|
# CPU-compute decode keeps experts in host RAM and computes them on the CPU;
|
|
# the GPU only holds the two-layer prefill double buffer. So the slot cache is
|
|
# fixed at exactly two expert layers (prefill overlap requires >= 2*num_experts)
|
|
# and --moe-cache-size / --moe-cache-auto / --moe-cache-rate do not apply.
|
|
num_experts = config.model_config.num_experts
|
|
if getattr(config, "moe_cache_auto", False):
|
|
override("moe_cache_auto", False)
|
|
override("moe_cache_mem", None)
|
|
override("moe_cache_size", 2 * num_experts)
|
|
override("moe_prefill_overlap", True)
|
|
logger.info_rank0(
|
|
f"MoE backend 'cpu': decode computes experts on CPU; GPU keeps a "
|
|
f"two-layer prefill buffer (moe_cache_size={2 * num_experts})"
|
|
)
|
|
|
|
if (
|
|
is_moe
|
|
and expert_quant not in _RESIDENT_EXPERT_QUANTS
|
|
and not is_offload_moe_strategy(config.moe_strategy)
|
|
):
|
|
raise ValueError(
|
|
f"{expert_quant} experts require --moe-strategy offload or cpu, "
|
|
f"got {config.moe_strategy!r}"
|
|
)
|
|
|
|
if is_moe and config.moe_cpu_layers and config.moe_strategy not in ("offload", "hybrid"):
|
|
# the layer split needs the offload host banks + slot cache; 'cpu' already runs every layer on CPU, 'fused' keeps experts resident on the GPU (no host banks)
|
|
raise ValueError(
|
|
"--moe-cpu-layers requires --moe-strategy offload or hybrid (got "
|
|
f"{config.moe_strategy!r}); use --moe-strategy cpu to run all layers on CPU"
|
|
)
|
|
|
|
if is_moe and config.moe_cpu_layers and config.moe_cpu_layers.strip() != "auto":
|
|
if not _parse_cpu_layers_spec(config.moe_cpu_layers, model_config.num_moe_layers):
|
|
override("moe_cpu_layers", None)
|
|
|
|
if is_moe:
|
|
object.__setattr__(model_config, "moe_strategy", config.moe_strategy)
|
|
object.__setattr__(model_config, "decode_target", _decode_target(config))
|
|
|
|
# Must stay LAST: page_size is only final here (_adjust_dsv4_config sets P=128, the
|
|
# TRTLLM block sets 64). Also covers the programmatic LLM(...) path that bypasses parse_args.
|
|
if config.num_token_override is not None:
|
|
if config.num_page_override is not None:
|
|
raise ValueError("--num-tokens and --num-pages are mutually exclusive")
|
|
if config.num_token_override % config.page_size != 0:
|
|
raise ValueError(
|
|
f"--num-tokens {config.num_token_override} is not a multiple of the resolved "
|
|
f"page size {config.page_size}; nearest valid values: "
|
|
f"{config.num_token_override // config.page_size * config.page_size} or "
|
|
f"{(config.num_token_override // config.page_size + 1) * config.page_size}"
|
|
)
|
|
override("num_page_override", config.num_token_override // config.page_size)
|
|
|
|
# The rope cos/sin table covers rotary_config.table_positions, and neither rope kernel
|
|
# bounds-checks the position it gathers with -- a longer ceiling reads past the table.
|
|
# DSV4 is exempt: it sizes its own table from the resolved max_seq_len (_adjust_dsv4_config).
|
|
rotary = getattr(model_config, "rotary_config", None)
|
|
seq_override = getattr(config, "max_seq_len_override", None)
|
|
if seq_override is not None and rotary is not None and not (is_dsv4 or is_dsv41):
|
|
if seq_override > rotary.table_positions:
|
|
raise ValueError(
|
|
f"--max-seq-len-override {seq_override} exceeds the model's "
|
|
f"rope table ({rotary.table_positions} positions). Serving past it would read "
|
|
"out of bounds; extend the rope with --hf-overrides (YaRN rope_parameters) instead."
|
|
)
|
|
|
|
# The startup ServerArgs dump is the *requested* config, printed in the frontend process
|
|
# before any of the resolution above ran -- so "moe_strategy='auto'" is all it can say. This
|
|
# is the one line that reports what actually runs, for every path (explicit backends never
|
|
# hit an "Auto-selected ..." log at all).
|
|
resolved = [
|
|
f"attention_backend={config.attention_backend!r}",
|
|
f"cache_type={getattr(config, 'cache_type', 'radix')!r}",
|
|
f"page_size={config.page_size}",
|
|
]
|
|
if is_moe:
|
|
resolved.insert(0, f"moe_strategy={config.moe_strategy!r}")
|
|
logger.info_rank0(f"Resolved config: {', '.join(resolved)}")
|