fix(cache): expire idle results, apply review fixes to the disk tier

- A result now expires 75 min after its last use (RESULT_TTL_SECONDS),
  in memory and on disk, instead of 30 min after it was made and only
  when read again. Nothing tells the proxy a session ended; one idle
  past the provider's longest prompt-cache TTL has no prefix left to
  protect, so recompressing on resume costs only the compression.
  A result with a CCR marker keeps the CCR TTL from creation and is
  never written to disk: its original lives in the CCR store.
- Disk keys are namespaced by a digest of the router config and
  headroom version: two routers (per-provider Kompress overrides) or a
  restart with other settings never serve each other's output.
- A SQLite error is a miss, logged once, never a failed transform.
- connect_private_sqlite (narrows crash-left sidecars); expired rows
  deleted on open; HEADROOM_COMPRESSION_CACHE_PATH expanduser'd.
- clear() is memory-only and fences racing disk reads.
- Tests write the file under tmp_path, not ~/.headroom.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-21 20:04:04 -07:00
co-authored by Claude Opus 5.5
parent 7edb484261
commit 51287abf18
4 changed files with 252 additions and 59 deletions
+71 -29
View File
@@ -4,12 +4,18 @@ The router keeps recent results in a byte-budgeted LRU; this file holds the rest
so an entry the budget evicted comes back from disk instead of being compressed
again (which costs a Kompress pass and can give different bytes, rewriting a
cached prompt prefix). It also survives a restart and is shared by worker
processes (WAL), like the CCR store.
processes (WAL), like the CCR store; WAL needs a local filesystem.
Each router reads and writes under its own namespace (a digest of its config and
the headroom version), so routers configured differently, or a restart with
other settings, never serve each other's output. Results carrying a CCR marker
are not written here: their originals live in the CCR store, on its own clock.
The file holds compressed tool output, so it follows the CCR store's rules:
owner-only, ``workspace_dir()/compression_cache.db`` (``HEADROOM_COMPRESSION_CACHE_PATH``
to move it), never opened in stateless mode, and
``HEADROOM_COMPRESSION_CACHE_DISK=0`` keeps the cache in memory only.
to move it), expired rows deleted on open, never opened in stateless mode, and
``HEADROOM_COMPRESSION_CACHE_DISK=0`` keeps the cache in memory only. A database
error is a miss, never a failed request.
"""
from __future__ import annotations
@@ -21,7 +27,7 @@ import threading
import time
from pathlib import Path
from ..fileperms import ensure_private_file
from ..fileperms import connect_private_sqlite
logger = logging.getLogger(__name__)
@@ -35,53 +41,89 @@ CREATE TABLE IF NOT EXISTS results (
);
CREATE INDEX IF NOT EXISTS idx_results_expires ON results (expires);
"""
#: Expired rows are deleted every this many writes.
#: Expired rows are deleted every this many writes, and on open.
_PURGE_EVERY = 1000
class ResultStore:
"""Compressed results by content key; an entry expires ``ttl_seconds`` after it
was written. Thread-safe; several processes may share the file."""
"""Compressed results by key; an entry expires ``ttl_seconds`` after it was
last read or written. Thread-safe; several processes may share the file."""
def __init__(self, path: Path, ttl_seconds: float) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
ensure_private_file(path, what="compression cache")
self._path = path
self._ttl = ttl_seconds
self._lock = threading.Lock()
self._writes = 0
self._db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
self._warned = False
self._db = connect_private_sqlite(
path, what="compression cache", check_same_thread=False, isolation_level=None
)
self._db.execute("PRAGMA busy_timeout=5000")
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA synchronous=NORMAL")
self._db.executescript(_SCHEMA)
# Sessions that ended while the proxy was down go now.
self._db.execute("DELETE FROM results WHERE expires <= ?", (time.time(),))
def get(self, key: str) -> tuple[str, float, str] | None:
with self._lock:
row = self._db.execute(
"SELECT text, ratio, strategy FROM results WHERE k = ? AND expires > ?",
(key, time.time()),
).fetchone()
now = time.time()
try:
with self._lock:
row = self._db.execute(
"SELECT text, ratio, strategy, expires FROM results "
"WHERE k = ? AND expires > ?",
(key, now),
).fetchone()
# Keep what a live session still reads; refresh at most every half ttl.
if row is not None and row[3] - now < self._ttl / 2:
self._db.execute(
"UPDATE results SET expires = ? WHERE k = ?", (now + self._ttl, key)
)
except sqlite3.Error as exc:
self._failed("read", exc)
return None
return None if row is None else (row[0], row[1], row[2])
def put(self, key: str, text: str, ratio: float, strategy: str) -> None:
now = time.time()
with self._lock:
self._db.execute(
"INSERT OR REPLACE INTO results (k, text, ratio, strategy, expires) "
"VALUES (?, ?, ?, ?, ?)",
(key, text, ratio, strategy, now + self._ttl),
)
self._writes += 1
if self._writes % _PURGE_EVERY == 0:
self._db.execute("DELETE FROM results WHERE expires <= ?", (now,))
try:
with self._lock:
self._db.execute(
"INSERT OR REPLACE INTO results (k, text, ratio, strategy, expires) "
"VALUES (?, ?, ?, ?, ?)",
(key, text, ratio, strategy, now + self._ttl),
)
self._writes += 1
if self._writes % _PURGE_EVERY == 0:
self._db.execute("DELETE FROM results WHERE expires <= ?", (now,))
except sqlite3.Error as exc:
self._failed("write", exc)
def touch(self, key: str) -> None:
"""Keep ``key`` for another ttl (it is in use from memory)."""
self._run("UPDATE results SET expires = ? WHERE k = ?", (time.time() + self._ttl, key))
def delete(self, key: str) -> None:
with self._lock:
self._db.execute("DELETE FROM results WHERE k = ?", (key,))
self._run("DELETE FROM results WHERE k = ?", (key,))
def clear(self) -> None:
with self._lock:
self._db.execute("DELETE FROM results")
def _run(self, sql: str, args: tuple[object, ...]) -> None:
try:
with self._lock:
self._db.execute(sql, args)
except sqlite3.Error as exc:
self._failed("write", exc)
def _failed(self, what: str, exc: sqlite3.Error) -> None:
if not self._warned:
self._warned = True
logger.warning(
"compression cache: %s of %s failed (%s); treating it as a miss. "
"Check the disk, or set HEADROOM_COMPRESSION_CACHE_DISK=0.",
what,
self._path,
exc,
)
def open_result_store(ttl_seconds: float) -> ResultStore | None:
@@ -99,7 +141,7 @@ def open_result_store(ttl_seconds: float) -> ResultStore | None:
if not persistence_allowed("compression result cache (compression_cache.db)"):
return None
configured = os.environ.get("HEADROOM_COMPRESSION_CACHE_PATH", "").strip()
path = Path(configured) if configured else workspace_dir() / "compression_cache.db"
path = Path(configured).expanduser() if configured else workspace_dir() / "compression_cache.db"
try:
return ResultStore(path, ttl_seconds)
except (OSError, sqlite3.Error) as exc:
+106 -29
View File
@@ -51,7 +51,7 @@ from concurrent.futures import ThreadPoolExecutor
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field, replace
from enum import Enum
from typing import Any
from typing import TYPE_CHECKING, Any
from ..config import (
DEFAULT_BYTE_EXACT_EXCLUDE_TOOLS,
@@ -98,6 +98,9 @@ from .lossless_provider import (
from .mixed_content import ContentSection, mixed_content_indicators
from .relevance_split import build_relevance_query, plan_relevance_split, segment
if TYPE_CHECKING:
from ..cache.result_store import ResultStore
logger = logging.getLogger(__name__)
_extract_json_block = _mixed_content._extract_json_block
@@ -1564,6 +1567,12 @@ def _netcost_message_tokens(message: dict[str, Any], tokenizer: Tokenizer) -> in
DEFAULT_COMPRESSION_CACHE_MB = 64
#: Bytes a cache entry costs beyond its text (key, tuple, dict slot).
_CACHE_ENTRY_OVERHEAD = 256
#: How long a compression result is kept after it was last used. Results are
#: mostly one session's tool output, and nothing signals a session's end; a
#: session idle past the provider's longest prompt-cache TTL (Anthropic: 1 h)
#: has no cached prefix left to protect, so compressing it again on resume
#: costs only the compression.
RESULT_TTL_SECONDS = 75 * 60
#: Skip-tier entries kept per MB of budget (each is a key and a timestamp).
_SKIP_ENTRIES_PER_MB = 2000
@@ -1593,8 +1602,32 @@ def content_cache_key(content: str, target_ratio: float | None, gated: bool = Fa
return h.hexdigest()
def _config_namespace(config: Any) -> str:
"""A prefix for this router's keys in the shared disk tier: a digest of its
config and the headroom version. The proxy may run routers configured
differently (per-provider Kompress overrides), and a restart may change
settings; neither may serve output the other made."""
from dataclasses import asdict
from .. import __version__
def stable(value: Any) -> Any:
# Sets repr in hash-seed order, which differs per process.
if isinstance(value, (set, frozenset)):
return sorted(map(stable, value), key=repr)
if isinstance(value, dict):
return sorted((repr(k), stable(v)) for k, v in value.items())
if isinstance(value, (list, tuple)):
return [stable(v) for v in value]
return value
h = hashlib.blake2b(digest_size=8)
h.update(f"{__version__}|{stable(asdict(config))!r}".encode())
return h.hexdigest() + ":"
class CompressionCache:
"""Two-tier compression cache with TTL and a memory budget. Thread-safe.
"""Two-tier compression cache with idle expiry and a memory budget. Thread-safe.
Tier 1 (skip set): content keys that won't compress — instant skip.
@@ -1602,11 +1635,15 @@ class CompressionCache:
reuse the compressed text on subsequent requests.
Memory is bounded: results are kept in least-recently-used order within
``max_bytes`` (``HEADROOM_COMPRESSION_CACHE_MB``, default 64), the skip set
within a matching entry count, and entries expire after ``ttl_seconds``.
``max_bytes`` (``HEADROOM_COMPRESSION_CACHE_MB``, default 64; results only),
the skip set within a matching entry count. A result expires ``ttl_seconds``
after it was last used (a session idle that long has no cached prefix to
protect). A result carrying a CCR marker expires ``marker_ttl_seconds`` after
it was made instead, with the CCR original it points to.
With a ``store`` (the proxy's :class:`~headroom.cache.result_store.ResultStore`),
every result is also written to disk and an entry the budget evicted is
read back from there instead of compressed again.
marker-free results are also written to disk under ``namespace`` and an
entry the budget evicted is read back from there instead of compressed again.
Thread safety: a ``threading.Lock`` guards all read-modify-write
operations. The ``apply()`` path runs compression inside a
@@ -1617,25 +1654,37 @@ class CompressionCache:
def __init__(
self,
ttl_seconds: int = 1800,
ttl_seconds: int = RESULT_TTL_SECONDS,
max_bytes: int | None = None,
store: Any = None,
store: ResultStore | None = None,
namespace: str = "",
marker_ttl_seconds: int | None = None,
):
import threading
# Tier 2: {key: (text, ratio, strategy, created_at)}, least recently used first.
self._results: OrderedDict[str, tuple[str, float, str, float]] = OrderedDict()
from ..cache.compression_store import _get_env_default_ttl_seconds
# Tier 2: {key: (text, ratio, strategy, stamp, disk_touched, has_marker)},
# least recently used first. ``stamp`` is the last use, or the creation
# time for a result with a CCR marker.
self._results: OrderedDict[str, tuple[str, float, str, float, float, bool]] = OrderedDict()
self._bytes = 0
# Tier 1: {key: marked_at}, oldest first.
self._skip: OrderedDict[str, float] = OrderedDict()
self._max_bytes = max_bytes if max_bytes is not None else _compression_cache_budget_bytes()
self._max_skip = max(1000, self._max_bytes // (1024 * 1024) * _SKIP_ENTRIES_PER_MB)
self._store = store
self._namespace = namespace
# Bumped by clear(), so a disk read racing it is not admitted after it.
self._generation = 0
# Callbacks invoked (outside the lock) whenever ``clear()`` runs, so
# sibling state keyed by the same content keys (e.g. the router's
# frozen-verdict store) can be reset in lock-step with the cache.
self._on_clear: list[Any] = []
self._ttl_seconds = ttl_seconds
self._marker_ttl_seconds = (
marker_ttl_seconds if marker_ttl_seconds is not None else _get_env_default_ttl_seconds()
)
# Metrics
self._hits = 0
self._disk_hits = 0
@@ -1653,23 +1702,46 @@ class CompressionCache:
Use is_skipped() first to check if content is known non-compressible.
"""
t0 = time.perf_counter_ns()
touch_disk = False
with self._lock:
entry = self._results.get(key)
if entry is not None:
compressed, ratio, strategy, created_at = entry
if (time.monotonic() - created_at) < self._ttl_seconds:
compressed, ratio, strategy, stamp, disk_touched, has_marker = entry
now = time.monotonic()
ttl = self._marker_ttl_seconds if has_marker else self._ttl_seconds
if (now - stamp) < ttl:
if not has_marker:
# A hit keeps the entry for another ttl: expiry counts
# idle time. The disk copy too, at most every half ttl,
# so the budget evicting it later cannot lose it mid-session.
touch_disk = self._store is not None and (now - disk_touched >= ttl / 2)
self._results[key] = (
compressed,
ratio,
strategy,
now,
now if touch_disk else disk_touched,
False,
)
self._results.move_to_end(key)
self._hits += 1
self._record_lookup(t0)
return (compressed, ratio, strategy)
self._drop_locked(key)
self._evictions += 1
found = self._store.get(key) if self._store is not None else None
hit: tuple[str, float, str] | None = (compressed, ratio, strategy)
else:
self._drop_locked(key)
self._evictions += 1
hit = None
if hit is not None:
if touch_disk and self._store is not None:
self._store.touch(self._namespace + key)
return hit
generation = self._generation
found = self._store.get(self._namespace + key) if self._store is not None else None
with self._lock:
if found is not None:
if found is not None and generation == self._generation:
self._admit_locked(key, *found)
self._disk_hits += 1
else:
elif found is None:
self._misses += 1
self._record_lookup(t0)
return found
@@ -1689,9 +1761,9 @@ class CompressionCache:
def put(self, key: str, compressed: str, ratio: float, strategy: str) -> None:
"""Store a compressed result (Tier 2). Thread-safe."""
with self._lock:
self._admit_locked(key, compressed, ratio, strategy)
if self._store is not None:
self._store.put(key, compressed, ratio, strategy)
has_marker = self._admit_locked(key, compressed, ratio, strategy)
if self._store is not None and not has_marker:
self._store.put(self._namespace + key, compressed, ratio, strategy)
def mark_skip(self, key: str) -> None:
"""Mark content as non-compressible (Tier 1). Thread-safe."""
@@ -1705,16 +1777,21 @@ class CompressionCache:
self._drop_locked(key)
self._skip_locked(key)
if self._store is not None:
self._store.delete(key)
self._store.delete(self._namespace + key)
def _admit_locked(self, key: str, compressed: str, ratio: float, strategy: str) -> None:
def _admit_locked(self, key: str, compressed: str, ratio: float, strategy: str) -> bool:
"""Add a result, evicting least recently used ones over budget; returns
whether it carries a CCR marker."""
self._drop_locked(key)
self._results[key] = (compressed, ratio, strategy, time.monotonic())
now = time.monotonic()
has_marker = CCR_RETRIEVAL_MARKER_RE.search(compressed) is not None
self._results[key] = (compressed, ratio, strategy, now, now, has_marker)
self._bytes += sys.getsizeof(compressed) + _CACHE_ENTRY_OVERHEAD
# Least recently used first; the entry just added always stays.
while self._bytes > self._max_bytes and len(self._results) > 1:
self._drop_locked(next(iter(self._results)))
self._evictions += 1
return has_marker
def _drop_locked(self, key: str) -> None:
entry = self._results.pop(key, None)
@@ -1769,13 +1846,13 @@ class CompressionCache:
self._on_clear.append(callback)
def clear(self) -> None:
"""Clear all entries (e.g., on session end). Thread-safe."""
"""Clear this process's entries. Thread-safe. The disk tier is shared
with other workers and routers and is left alone; its rows expire."""
with self._lock:
self._results.clear()
self._bytes = 0
self._skip.clear()
if self._store is not None:
self._store.clear()
self._generation += 1
# Fire callbacks outside the lock to avoid cross-lock ordering issues.
for callback in self._on_clear:
callback()
@@ -2547,8 +2624,8 @@ class ContentRouter(Transform):
if self.config.result_cache_disk:
from ..cache.result_store import open_result_store
store = open_result_store(ttl_seconds=1800)
self._cache = CompressionCache(store=store)
store = open_result_store(ttl_seconds=RESULT_TTL_SECONDS)
self._cache = CompressionCache(store=store, namespace=_config_namespace(self.config))
# Cache-churn fix (HEADROOM_FREEZE_BLOCK_DECISION, default off):
# freeze each block's compress-vs-passthrough verdict on first
+3
View File
@@ -89,6 +89,9 @@ def _scrub_developer_headroom_env(monkeypatch, tmp_path):
# test its own store so proxy/CLI startup cannot load developer settings and
# saves cannot rewrite them. Tests of path precedence can override this.
monkeypatch.setenv("HEADROOM_SETTINGS_PATH", str(tmp_path / "headroom-settings.json"))
# The proxy's compression cache keeps results on disk; never in the
# developer's ~/.headroom, never shared between tests.
monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "compression_cache.db"))
# The scrub also deletes the HEADROOM_HARD_WATCHDOG_SECS=0 opt-out CI sets
# (#3845), so every test that enters the proxy lifespan armed the 90s
# production watchdog inside pytest; it hard-exits the process with code 1
+72 -1
View File
@@ -6,7 +6,12 @@ import subprocess
import sys
from headroom.cache.result_store import ResultStore, open_result_store
from headroom.transforms.content_router import CompressionCache, content_cache_key
from headroom.transforms.content_router import (
CompressionCache,
ContentRouter,
ContentRouterConfig,
content_cache_key,
)
def test_results_stay_within_the_memory_budget_least_recently_used_first():
@@ -54,3 +59,69 @@ def test_stateless_mode_writes_no_cache_file(tmp_path, monkeypatch):
monkeypatch.setenv("HEADROOM_STATELESS", "1")
assert open_result_store(1800) is None
assert not (tmp_path / "compression_cache.db").exists()
def test_a_result_in_use_is_kept_and_an_idle_one_is_dropped(tmp_path, monkeypatch):
# Nothing signals a session's end: a result expires once unused for the
# ttl, from memory and from disk, while a session still reading it keeps it.
import time as time_module
clock = [1000.0]
monkeypatch.setattr(time_module, "monotonic", lambda: clock[0])
monkeypatch.setattr(time_module, "time", lambda: clock[0])
store = ResultStore(tmp_path / "c.db", ttl_seconds=100)
cache = CompressionCache(ttl_seconds=100, store=store)
cache.put("live", "kept", 0.5, "kompress")
cache.put("idle", "dropped", 0.5, "kompress")
for _ in range(5): # 'live' is read every 60 s, for 300 s
clock[0] += 60
assert cache.get("live") is not None
assert cache.get("idle") is None
assert store.get("idle") is None
assert store.get("live") == ("kept", 0.5, "kompress")
def test_routers_configured_differently_never_share_output(tmp_path, monkeypatch):
# The proxy can run two routers (per-provider Kompress overrides), and a
# restart can change settings; both read the same file.
monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "c.db"))
lossy = ContentRouter(ContentRouterConfig(result_cache_disk=True))
lossless = ContentRouter(ContentRouterConfig(result_cache_disk=True, lossless=True))
lossy._cache.put("k", "lossy output", 0.3, "kompress")
assert lossless._cache.get("k") is None
same = ContentRouter(ContentRouterConfig(result_cache_disk=True))
assert same._cache.get("k") == ("lossy output", 0.3, "kompress")
def test_the_namespace_is_the_same_in_every_process():
code = (
"from headroom.transforms.content_router import ContentRouterConfig, _config_namespace;"
"print(_config_namespace(ContentRouterConfig()))"
)
seen = {
subprocess.run(
[sys.executable, "-c", code],
capture_output=True,
text=True,
check=True,
env={**__import__("os").environ, "PYTHONHASHSEED": seed},
).stdout.strip()
for seed in ("1", "2")
}
assert len(seen) == 1
def test_a_broken_database_is_a_miss_not_a_failed_request(tmp_path):
store = ResultStore(tmp_path / "c.db", 1800)
cache = CompressionCache(store=store)
store._db.close()
cache.put("k", "text", 0.5, "kompress") # no raise
cache.clear()
assert cache.get("k") is None
def test_a_result_pointing_at_a_ccr_original_is_not_kept_on_disk(tmp_path):
store = ResultStore(tmp_path / "c.db", 1800)
cache = CompressionCache(store=store)
cache.put("k", "first lines <<ccr:0123456789abcdef>>", 0.2, "x")
assert store.get("k") is None