"""The engine's OWN footprint. Boundary: only the serve tree's RAM/VRAM — system-wide host telemetry is not this daemon's job. RAM = summed PSS across the serve process group (shared pages counted once, the honest number). VRAM = per-process GPU memory for those pids, via ``pynvml`` if importable (optional), else parsed from ``nvidia-smi``, else 0. All best-effort and off the event loop — a missing GPU or absent NVML returns 0, never an error.""" from __future__ import annotations import subprocess import threading import time from typing import Callable from . import osproc def engine_footprint(pid: int | None) -> dict: if pid is None: return {"ramBytes": 0, "vramBytes": 0, "pids": []} pids = osproc.tree_pids(pid) ram = sum(osproc.read_pss_bytes(p) for p in pids) vram = vram_bytes_for_pids(pids) return {"ramBytes": ram, "vramBytes": vram, "pids": pids} class FootprintCache: """Single-flight + short-TTL cache over ``engine_footprint``. Clients poll metrics frequently and an NVML/nvidia-smi probe can take seconds on a busy GPU; without this, every poll pays that cost and can back up the proxy executor. Concurrent callers within the TTL collapse to one probe.""" def __init__(self, *, ttl_s: float = 2.0, now: Callable[[], float] = time.monotonic) -> None: self._ttl = ttl_s self._now = now self._lock = threading.Lock() self._cache: dict[int | None, tuple[float, dict]] = {} def get(self, pid: int | None) -> dict: with self._lock: hit = self._cache.get(pid) if hit is not None and (self._now() - hit[0]) < self._ttl: return hit[1] val = engine_footprint(pid) self._cache[pid] = (self._now(), val) return val def vram_bytes_for_pids(pids: list[int]) -> int: want = set(pids) if not want: return 0 usage = _nvml_process_vram() if usage is None: usage = _smi_process_vram() if not usage: return 0 return sum(nbytes for p, nbytes in usage.items() if p in want) # NVML is initialized ONCE and held for the daemon's life — nvmlInit()+nvmlShutdown() on every # call costs seconds on a busy GPU. None = not yet tried, True = ready, False = unavailable. _NVML = {"ready": None} _NVML_LOCK = threading.Lock() def _nvml_ready(): with _NVML_LOCK: if _NVML["ready"] is None: try: import pynvml # optional; not a hard dep pynvml.nvmlInit() _NVML["ready"] = pynvml except Exception: # noqa: BLE001 _NVML["ready"] = False return _NVML["ready"] def _nvml_process_vram() -> dict[int, int] | None: pynvml = _nvml_ready() if not pynvml: return None out: dict[int, int] = {} try: count = pynvml.nvmlDeviceGetCount() for i in range(count): handle = pynvml.nvmlDeviceGetHandleByIndex(i) for getter in ( getattr(pynvml, "nvmlDeviceGetComputeRunningProcesses_v3", None), getattr(pynvml, "nvmlDeviceGetComputeRunningProcesses", None), ): if getter is None: continue try: for proc in getter(handle): used = getattr(proc, "usedGpuMemory", None) if used: # None == "not available", per NVML out[int(proc.pid)] = out.get(int(proc.pid), 0) + int(used) break except Exception: # noqa: BLE001 continue except Exception: # noqa: BLE001 return out or None # Empty → NVML enumeration gave nothing usable (e.g. every process getter raised on a # driver/MIG mismatch); signal that with None so the nvidia-smi fallback still runs, matching # the error path above. return out or None def _smi_process_vram() -> dict[int, int]: try: out = subprocess.run( [ "nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits", ], capture_output=True, text=True, timeout=3.0, ) except (OSError, subprocess.SubprocessError): return {} if out.returncode != 0: return {} usage: dict[int, int] = {} for line in out.stdout.splitlines(): parts = [p.strip() for p in line.split(",")] if len(parts) != 2 or not parts[0].isdigit() or not parts[1].isdigit(): continue usage[int(parts[0])] = usage.get(int(parts[0]), 0) + int(parts[1]) * 1024 * 1024 # MiB return usage