Files
FreeToken/python/freetoken/kvcache/cache_status.py
Xiaoze Fan c2c3e828e8 feat(spec): verify draft tokens on the target side (#836)
* feat(spec): add the speculative verify infrastructure

* feat(scheduler): run decode steps as speculative verify steps

* chore(engine): mark the speculative decoding config fields with one comment

* feat(server): add short aliases for the speculative decoding flags
2026-10-28 21:18:02 -07:00

201 lines
9.5 KiB
Python

from __future__ import annotations
from typing import Any, Dict
def window_pool_spec(config):
from . import resolve_pool_class
return resolve_pool_class(config.model_config).window_spec(config)
def _supports_swa_ratio(config) -> bool:
return window_pool_spec(config) is not None
def compute_cache_unit_bytes(engine: "Engine") -> Dict[str, int]:
"""Per-unit VRAM byte cost of each runtime cache, measured from the real allocated pool
tensors so the desktop cache panel can show a true VRAM delta per slider step:
kv_bytes_per_token -- total paged-KV bytes (across every attention layer) for one token, plus the speculative draft's KV on the same slot
moe_bytes_per_expert -- bytes of one MoE expert-cache (offload bank) slot
mamba_bytes_per_slot -- bytes of one SSM/GDN recurrent-state slot (all linear layers)
Each is 0 when the model has no such cache (dense / non-hybrid / non-offload MoE) or the pool
can't be measured. Best-effort and total: any failure degrades that unit to 0 and never raises
-- this runs on the readiness path and must not block startup.
"""
def _kv_swa() -> tuple[int, int]:
# KV (full pool) + SWA (window pool) per-token bytes come from the pool itself: the
# generic pools measure their live buffers, DSV4 reads its per-tier cost model. This
# layer never branches on the model. 0/0 for a fake engine or a model without the pool.
pool = engine.kv_cache
if pool is None:
return 0, 0
kv, swa = pool.unit_bytes()
# the draft KV pool grows with num_pages too, and the rebuild check counts it
spec = getattr(engine, "spec", None)
draft = spec.budget(engine.config)[1] if spec is not None else 0
return int(kv) + int(draft), int(swa)
def _moe() -> int:
banks = getattr(engine.moe_offload_cache, "bank_caches", None)
if not banks:
return 0
if getattr(engine.moe_offload_cache, "_raw_banks", False):
# pools of several bank layouts share byte segments: report the mean slot size
return int(sum(t.numel() for t in banks.values()) // engine.moe_offload_cache.cache_size)
# Each bank cache is (cache_size, *row_shape); one slot's bytes = row bytes summed over
# the format's banks (== cache_budget.expert_bytes_per_slot on the source rows).
return int(sum(t[0].numel() * t.element_size() for t in banks.values()))
def _mamba() -> int:
pool = engine.linear_state_pool
if pool is None:
return 0
return int(pool.bytes_per_slot())
out = {
"kv_bytes_per_token": 0,
"moe_bytes_per_expert": 0,
"mamba_bytes_per_slot": 0,
"swa_bytes_per_token": 0,
}
try:
out["kv_bytes_per_token"], out["swa_bytes_per_token"] = _kv_swa()
except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness
pass
for key, fn in (("moe_bytes_per_expert", _moe), ("mamba_bytes_per_slot", _mamba)):
try:
out[key] = fn()
except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness
out[key] = 0
return out
def _pool_budget_free_vram_bytes(engine: "Engine") -> int:
"""The pool-budget baseline captured in ``Engine.__init__``: free VRAM after weights
loaded, before any runtime cache pool was allocated (``_post_weights_free``). This is the
honest "spend it all on one pool" budget for the desktop slider upper bounds — stable for
the process lifetime, unaffected by allocator caching / CUDA graphs / later rebuilds.
0 when unavailable (fake engines in tests, capture failed). Never raises."""
try:
return max(0, int(engine._post_weights_free or 0))
except Exception: # noqa: BLE001
return 0
def compute_cache_floors(engine: "Engine") -> Dict[str, int]:
"""Per-pool minimum (floor) unit counts the runtime cache-rebuild path enforces, in the
units the desktop sliders use (KV tokens, MoE experts, usable mamba slots). Every floor is
derived live from the model config / engine validation logic -- never a baked-in constant:
kv_tokens -- rebuild_runtime_cache rejects num_pages <= 0, so the floor is one page's
worth of tokens (page_size). Owned-KV models (dsv4) have no rebuildable
generic pool -> 0.
moe_experts -- _require_offload_cache_size needs the offload cache to hold >= one MoE
layer's experts (num_experts). 0 when the model has no offload cache.
mamba_slots -- rebuild_runtime_cache rejects num_mamba_slots below
_linear_pool_min_slots(config) - 1 (the physical floor minus the reserved
padding sink -> usable-slot floor). 0 when the model has no GDN state pool.
Best-effort/total: any failure degrades that floor to 0 and never raises (readiness path)."""
config = engine.config
floors = {"kv_tokens": 0, "moe_experts": 0, "mamba_slots": 0, "swa_tokens": 0}
if config is None:
return floors
def _kv() -> int:
# The full-KV pool floor in tokens, from the pool family: generic = one page; DSV4 =
# the window working-set floor that validate_rebuild() enforces.
pool = getattr(engine, "kv_cache", None)
if pool is None:
return int(config.page_size)
return int(type(pool).min_kv_tokens(config))
def _moe() -> int:
if engine.moe_offload_cache is None:
return 0
return int(config.model_config.num_experts)
def _mamba() -> int:
from .linear_state_pool import _linear_pool_min_slots
if engine.linear_state_pool is None:
return 0
return int(_linear_pool_min_slots(config) - 1)
def _swa() -> int:
spec = window_pool_spec(config)
return spec.page_size * spec.min_pages if spec is not None else 0
for key, fn in (("kv_tokens", _kv), ("moe_experts", _moe), ("mamba_slots", _mamba),
("swa_tokens", _swa)):
try:
floors[key] = fn()
except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness
floors[key] = 0
return floors
def compute_cache_pools(engine: "Engine") -> Dict[str, int]:
"""The ACTUAL pool sizes allocated at load, in API units. The frontend otherwise only
learns them from per-generation UserReply snapshots — i.e. not until the first chat —
so this seeds /v1/cache/status geometry with truth from the moment the server is ready.
mamba is the usable slot count (num_slots minus the reserved padding sink), matching the
scheduler's reported totals. 0 for pools the model lacks; never raises."""
pools = {
"num_pages": 0, "page_size": 0, "moe_cache_size": 0, "num_mamba_slots": 0,
"swa_page_size": 0, "num_swa_pages": 0,
}
try:
config = engine.config
pools["num_pages"] = int(engine.num_pages or 0)
if config is not None:
pools["page_size"] = int(config.page_size or 0)
spec = window_pool_spec(config)
if spec is not None:
pools["swa_page_size"] = spec.page_size
pools["num_swa_pages"] = max(0, int(engine.kv_cache.window_pages))
moe = engine.moe_offload_cache
if moe is not None:
pools["moe_cache_size"] = int(moe.cache_size or 0)
lsp = engine.linear_state_pool
if lsp is not None:
pools["num_mamba_slots"] = max(0, int(lsp.num_slots or 0) - 1)
except Exception: # noqa: BLE001 -- best-effort; readiness must not depend on this
pass
return pools
def compute_cache_status_meta(engine: "Engine") -> Dict[str, Any]:
"""Full readiness ("meta", …) payload for the desktop cache panel: the per-unit VRAM costs
(compute_cache_unit_bytes) plus the post-weights pre-pool free-VRAM baseline (the sliders'
ideal-max budget), the per-pool floors, and the actual pool sizes allocated at load.
Best-effort/total -- each piece independently degrades to 0/{} and this never raises
(it runs on readiness)."""
meta: Dict[str, Any] = dict(compute_cache_unit_bytes(engine))
meta["free_vram_bytes"] = _pool_budget_free_vram_bytes(engine)
meta["floors"] = compute_cache_floors(engine)
meta["pools"] = compute_cache_pools(engine)
# Current window/full reuse ratio (the tunable knob), for DSV4 and radix-SWA; 0.0 otherwise.
cfg = engine.config
has_swa_ratio = cfg is not None and _supports_swa_ratio(cfg)
meta["swa_full_tokens_ratio"] = float(cfg.swa_full_tokens_ratio) if has_swa_ratio else 0.0
# Exact total cache VRAM budget (all pools) the engine honors: memory_ratio of the
# post-weights baseline minus weights — the same figure the rebuild fit-check uses, with
# fixed=0 so it's the whole-cache ceiling (KV + MoE + Mamba + SWA), not the KV+MoE remainder.
# Lets the desktop show the real ceiling instead of reverse-deriving it from the per-pool
# limits. 0 when the baseline wasn't captured (fake engines in tests).
from freetoken.engine.cache_budget import net_cache_budget_bytes as _net_budget
try:
_baseline = int(engine._baseline_free or 0)
_weights = int(engine._weights_bytes or 0)
_mr = float(cfg.memory_ratio) if cfg is not None else 1.0
meta["cache_budget_bytes"] = max(0, _net_budget(_mr, _baseline, _weights, 0)) if _baseline > 0 else 0
except Exception: # noqa: BLE001 -- best-effort; readiness must not depend on this
meta["cache_budget_bytes"] = 0
return meta