from __future__ import annotations from typing import Any, Dict def window_pool_spec(config): from . import resolve_pool_class return resolve_pool_class(config.model_config).window_spec(config) def _supports_swa_ratio(config) -> bool: return window_pool_spec(config) is not None def compute_cache_unit_bytes(engine: "Engine") -> Dict[str, int]: """Per-unit VRAM byte cost of each runtime cache, measured from the real allocated pool tensors so the desktop cache panel can show a true VRAM delta per slider step: kv_bytes_per_token -- total paged-KV bytes (across every attention layer) for one token, plus the speculative draft's KV on the same slot moe_bytes_per_expert -- bytes of one MoE expert-cache (offload bank) slot mamba_bytes_per_slot -- bytes of one SSM/GDN recurrent-state slot (all linear layers) Each is 0 when the model has no such cache (dense / non-hybrid / non-offload MoE) or the pool can't be measured. Best-effort and total: any failure degrades that unit to 0 and never raises -- this runs on the readiness path and must not block startup. """ def _kv_swa() -> tuple[int, int]: # KV (full pool) + SWA (window pool) per-token bytes come from the pool itself: the # generic pools measure their live buffers, DSV4 reads its per-tier cost model. This # layer never branches on the model. 0/0 for a fake engine or a model without the pool. pool = engine.kv_cache if pool is None: return 0, 0 kv, swa = pool.unit_bytes() # the draft KV pool grows with num_pages too, and the rebuild check counts it spec = getattr(engine, "spec", None) draft = spec.budget(engine.config)[1] if spec is not None else 0 return int(kv) + int(draft), int(swa) def _moe() -> int: banks = getattr(engine.moe_offload_cache, "bank_caches", None) if not banks: return 0 if getattr(engine.moe_offload_cache, "_raw_banks", False): # pools of several bank layouts share byte segments: report the mean slot size return int(sum(t.numel() for t in banks.values()) // engine.moe_offload_cache.cache_size) # Each bank cache is (cache_size, *row_shape); one slot's bytes = row bytes summed over # the format's banks (== cache_budget.expert_bytes_per_slot on the source rows). return int(sum(t[0].numel() * t.element_size() for t in banks.values())) def _mamba() -> int: pool = engine.linear_state_pool if pool is None: return 0 return int(pool.bytes_per_slot()) out = { "kv_bytes_per_token": 0, "moe_bytes_per_expert": 0, "mamba_bytes_per_slot": 0, "swa_bytes_per_token": 0, } try: out["kv_bytes_per_token"], out["swa_bytes_per_token"] = _kv_swa() except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness pass for key, fn in (("moe_bytes_per_expert", _moe), ("mamba_bytes_per_slot", _mamba)): try: out[key] = fn() except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness out[key] = 0 return out def _pool_budget_free_vram_bytes(engine: "Engine") -> int: """The pool-budget baseline captured in ``Engine.__init__``: free VRAM after weights loaded, before any runtime cache pool was allocated (``_post_weights_free``). This is the honest "spend it all on one pool" budget for the desktop slider upper bounds — stable for the process lifetime, unaffected by allocator caching / CUDA graphs / later rebuilds. 0 when unavailable (fake engines in tests, capture failed). Never raises.""" try: return max(0, int(engine._post_weights_free or 0)) except Exception: # noqa: BLE001 return 0 def compute_cache_floors(engine: "Engine") -> Dict[str, int]: """Per-pool minimum (floor) unit counts the runtime cache-rebuild path enforces, in the units the desktop sliders use (KV tokens, MoE experts, usable mamba slots). Every floor is derived live from the model config / engine validation logic -- never a baked-in constant: kv_tokens -- rebuild_runtime_cache rejects num_pages <= 0, so the floor is one page's worth of tokens (page_size). Owned-KV models (dsv4) have no rebuildable generic pool -> 0. moe_experts -- _require_offload_cache_size needs the offload cache to hold >= one MoE layer's experts (num_experts). 0 when the model has no offload cache. mamba_slots -- rebuild_runtime_cache rejects num_mamba_slots below _linear_pool_min_slots(config) - 1 (the physical floor minus the reserved padding sink -> usable-slot floor). 0 when the model has no GDN state pool. Best-effort/total: any failure degrades that floor to 0 and never raises (readiness path).""" config = engine.config floors = {"kv_tokens": 0, "moe_experts": 0, "mamba_slots": 0, "swa_tokens": 0} if config is None: return floors def _kv() -> int: # The full-KV pool floor in tokens, from the pool family: generic = one page; DSV4 = # the window working-set floor that validate_rebuild() enforces. pool = getattr(engine, "kv_cache", None) if pool is None: return int(config.page_size) return int(type(pool).min_kv_tokens(config)) def _moe() -> int: if engine.moe_offload_cache is None: return 0 return int(config.model_config.num_experts) def _mamba() -> int: from .linear_state_pool import _linear_pool_min_slots if engine.linear_state_pool is None: return 0 return int(_linear_pool_min_slots(config) - 1) def _swa() -> int: spec = window_pool_spec(config) return spec.page_size * spec.min_pages if spec is not None else 0 for key, fn in (("kv_tokens", _kv), ("moe_experts", _moe), ("mamba_slots", _mamba), ("swa_tokens", _swa)): try: floors[key] = fn() except Exception: # noqa: BLE001 -- best-effort; a bad read must never block readiness floors[key] = 0 return floors def compute_cache_pools(engine: "Engine") -> Dict[str, int]: """The ACTUAL pool sizes allocated at load, in API units. The frontend otherwise only learns them from per-generation UserReply snapshots — i.e. not until the first chat — so this seeds /v1/cache/status geometry with truth from the moment the server is ready. mamba is the usable slot count (num_slots minus the reserved padding sink), matching the scheduler's reported totals. 0 for pools the model lacks; never raises.""" pools = { "num_pages": 0, "page_size": 0, "moe_cache_size": 0, "num_mamba_slots": 0, "swa_page_size": 0, "num_swa_pages": 0, } try: config = engine.config pools["num_pages"] = int(engine.num_pages or 0) if config is not None: pools["page_size"] = int(config.page_size or 0) spec = window_pool_spec(config) if spec is not None: pools["swa_page_size"] = spec.page_size pools["num_swa_pages"] = max(0, int(engine.kv_cache.window_pages)) moe = engine.moe_offload_cache if moe is not None: pools["moe_cache_size"] = int(moe.cache_size or 0) lsp = engine.linear_state_pool if lsp is not None: pools["num_mamba_slots"] = max(0, int(lsp.num_slots or 0) - 1) except Exception: # noqa: BLE001 -- best-effort; readiness must not depend on this pass return pools def compute_cache_status_meta(engine: "Engine") -> Dict[str, Any]: """Full readiness ("meta", …) payload for the desktop cache panel: the per-unit VRAM costs (compute_cache_unit_bytes) plus the post-weights pre-pool free-VRAM baseline (the sliders' ideal-max budget), the per-pool floors, and the actual pool sizes allocated at load. Best-effort/total -- each piece independently degrades to 0/{} and this never raises (it runs on readiness).""" meta: Dict[str, Any] = dict(compute_cache_unit_bytes(engine)) meta["free_vram_bytes"] = _pool_budget_free_vram_bytes(engine) meta["floors"] = compute_cache_floors(engine) meta["pools"] = compute_cache_pools(engine) # Current window/full reuse ratio (the tunable knob), for DSV4 and radix-SWA; 0.0 otherwise. cfg = engine.config has_swa_ratio = cfg is not None and _supports_swa_ratio(cfg) meta["swa_full_tokens_ratio"] = float(cfg.swa_full_tokens_ratio) if has_swa_ratio else 0.0 # Exact total cache VRAM budget (all pools) the engine honors: memory_ratio of the # post-weights baseline minus weights — the same figure the rebuild fit-check uses, with # fixed=0 so it's the whole-cache ceiling (KV + MoE + Mamba + SWA), not the KV+MoE remainder. # Lets the desktop show the real ceiling instead of reverse-deriving it from the per-pool # limits. 0 when the baseline wasn't captured (fake engines in tests). from freetoken.engine.cache_budget import net_cache_budget_bytes as _net_budget try: _baseline = int(engine._baseline_free or 0) _weights = int(engine._weights_bytes or 0) _mr = float(cfg.memory_ratio) if cfg is not None else 1.0 meta["cache_budget_bytes"] = max(0, _net_budget(_mr, _baseline, _weights, 0)) if _baseline > 0 else 0 except Exception: # noqa: BLE001 -- best-effort; readiness must not depend on this meta["cache_budget_bytes"] = 0 return meta