fix(cache): bound the router's compression cache; keep the rest in SQLite

The content router's result cache had no size limit, and its TTL was
only checked when an entry was read again, so content that never came
back stayed in memory for the life of the proxy. A 300,000-result soak
grew past 550 MB.

- Results are kept least-recently-used within a byte budget
  (HEADROOM_COMPRESSION_CACHE_MB, default 64), sized with sys.getsizeof;
  the skip set is capped to match. The same soak now levels off.
- The proxy backs the cache with a SQLite file
  (headroom/cache/result_store.py, workspace_dir()/compression_cache.db,
  owner-only, WAL): an entry the budget evicted, or a restart lost,
  comes back from disk instead of being compressed again, which would
  cost a Kompress pass and could change the bytes of a cached prefix.
  Off in stateless mode and with HEADROOM_COMPRESSION_CACHE_DISK=0;
  library callers keep the cache in memory (result_cache_disk=False).
- Keys are a blake2b digest (content_cache_key) instead of Python's
  per-process hash(), so they mean the same in every process: the disk
  file is shared by workers, and a shared store for replicas can follow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-21 19:55:58 -07:00
co-authored by Claude Opus 5.5
parent e995eb9f1d
commit 7edb484261
4 changed files with 319 additions and 51 deletions
+113
View File
@@ -0,0 +1,113 @@
"""SQLite tier behind the content router's in-memory compression cache.
The router keeps recent results in a byte-budgeted LRU; this file holds the rest,
so an entry the budget evicted comes back from disk instead of being compressed
again (which costs a Kompress pass and can give different bytes, rewriting a
cached prompt prefix). It also survives a restart and is shared by worker
processes (WAL), like the CCR store.
The file holds compressed tool output, so it follows the CCR store's rules:
owner-only, ``workspace_dir()/compression_cache.db`` (``HEADROOM_COMPRESSION_CACHE_PATH``
to move it), never opened in stateless mode, and
``HEADROOM_COMPRESSION_CACHE_DISK=0`` keeps the cache in memory only.
"""
from __future__ import annotations
import logging
import os
import sqlite3
import threading
import time
from pathlib import Path
from ..fileperms import ensure_private_file
logger = logging.getLogger(__name__)
_SCHEMA = """
CREATE TABLE IF NOT EXISTS results (
k TEXT PRIMARY KEY,
text TEXT NOT NULL,
ratio REAL NOT NULL,
strategy TEXT NOT NULL,
expires REAL NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_results_expires ON results (expires);
"""
#: Expired rows are deleted every this many writes.
_PURGE_EVERY = 1000
class ResultStore:
"""Compressed results by content key; an entry expires ``ttl_seconds`` after it
was written. Thread-safe; several processes may share the file."""
def __init__(self, path: Path, ttl_seconds: float) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
ensure_private_file(path, what="compression cache")
self._ttl = ttl_seconds
self._lock = threading.Lock()
self._writes = 0
self._db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
self._db.execute("PRAGMA busy_timeout=5000")
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA synchronous=NORMAL")
self._db.executescript(_SCHEMA)
def get(self, key: str) -> tuple[str, float, str] | None:
with self._lock:
row = self._db.execute(
"SELECT text, ratio, strategy FROM results WHERE k = ? AND expires > ?",
(key, time.time()),
).fetchone()
return None if row is None else (row[0], row[1], row[2])
def put(self, key: str, text: str, ratio: float, strategy: str) -> None:
now = time.time()
with self._lock:
self._db.execute(
"INSERT OR REPLACE INTO results (k, text, ratio, strategy, expires) "
"VALUES (?, ?, ?, ?, ?)",
(key, text, ratio, strategy, now + self._ttl),
)
self._writes += 1
if self._writes % _PURGE_EVERY == 0:
self._db.execute("DELETE FROM results WHERE expires <= ?", (now,))
def delete(self, key: str) -> None:
with self._lock:
self._db.execute("DELETE FROM results WHERE k = ?", (key,))
def clear(self) -> None:
with self._lock:
self._db.execute("DELETE FROM results")
def open_result_store(ttl_seconds: float) -> ResultStore | None:
"""The disk tier, or ``None`` (memory only) in stateless mode, with
``HEADROOM_COMPRESSION_CACHE_DISK=0``, or when the file cannot be opened."""
from ..paths import persistence_allowed, workspace_dir
if os.environ.get("HEADROOM_COMPRESSION_CACHE_DISK", "1").strip().lower() in (
"0",
"false",
"no",
"off",
):
return None
if not persistence_allowed("compression result cache (compression_cache.db)"):
return None
configured = os.environ.get("HEADROOM_COMPRESSION_CACHE_PATH", "").strip()
path = Path(configured) if configured else workspace_dir() / "compression_cache.db"
try:
return ResultStore(path, ttl_seconds)
except (OSError, sqlite3.Error) as exc:
logger.warning(
"compression cache: cannot open %s (%s); keeping results in memory only. "
"Set HEADROOM_COMPRESSION_CACHE_PATH to a writable file or "
"HEADROOM_COMPRESSION_CACHE_DISK=0.",
path,
exc,
)
return None
+4
View File
@@ -1040,6 +1040,10 @@ class HeadroomProxy(
# Runs BEFORE the disable_kompress override below so that flag stays
# authoritative for turning Kompress off.
_apply_compressor_selection(router_config, config.compressors)
# The proxy is long-lived: keep compression results it has no memory
# budget for in a SQLite file instead of compressing them again
# (headroom.cache.result_store; off in stateless mode).
router_config.result_cache_disk = True
# External (non-built-in) `headroom.compressor` selections are ignored by
# `_apply_compressor_selection` (they have no enable_* flag). Thread them
# to the router here so it can route matching blocks through them; None
+146 -51
View File
@@ -45,6 +45,7 @@ import re
import sys
import threading
import time
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from contextvars import ContextVar, copy_context
@@ -1558,21 +1559,54 @@ def _netcost_message_tokens(message: dict[str, Any], tokenizer: Tokenizer) -> in
return count_content_blocks(content, tokenizer.count_text)
class CompressionCache:
"""Two-tier compression cache with TTL. Thread-safe.
#: Default memory budget of the router's compression cache, in MB
#: (``HEADROOM_COMPRESSION_CACHE_MB``).
DEFAULT_COMPRESSION_CACHE_MB = 64
#: Bytes a cache entry costs beyond its text (key, tuple, dict slot).
_CACHE_ENTRY_OVERHEAD = 256
#: Skip-tier entries kept per MB of budget (each is a key and a timestamp).
_SKIP_ENTRIES_PER_MB = 2000
Tier 1 (skip set): content hashes that won't compress — instant skip,
near-zero memory (just ints in a set).
def _compression_cache_budget_bytes() -> int:
raw = os.environ.get("HEADROOM_COMPRESSION_CACHE_MB", "").strip()
try:
mb = float(raw) if raw else DEFAULT_COMPRESSION_CACHE_MB
except ValueError:
logger.warning(
"HEADROOM_COMPRESSION_CACHE_MB=%r is not a number; using %d MB",
raw,
DEFAULT_COMPRESSION_CACHE_MB,
)
mb = DEFAULT_COMPRESSION_CACHE_MB
return max(1, int(mb * 1024 * 1024))
def content_cache_key(content: str, target_ratio: float | None, gated: bool = False) -> str:
"""The compression cache's key for ``content``: a digest, the same in every
process, so the disk tier (and replicas sharing it) can use it. The runtime
target ratio is part of it (a different ratio is a different result), and so
is the reversibility gate tool output passes (#1307)."""
h = hashlib.blake2b(digest_size=16)
h.update(f"{target_ratio!r}|{int(gated)}|".encode())
h.update(content.encode("utf-8", "surrogatepass"))
return h.hexdigest()
class CompressionCache:
"""Two-tier compression cache with TTL and a memory budget. Thread-safe.
Tier 1 (skip set): content keys that won't compress — instant skip.
Tier 2 (result cache): compressed results for content that DID compress —
reuse the compressed text on subsequent requests.
Entries expire after TTL (default 30min). No max-entries cap — TTL is the
natural bound. Memory grows proportional to compressible content × TTL,
which is bounded by session duration.
Uses in-process dict for ultra-fast lookups (~100ns). Could be backed
by memcached/Redis for multi-process deployments.
Memory is bounded: results are kept in least-recently-used order within
``max_bytes`` (``HEADROOM_COMPRESSION_CACHE_MB``, default 64), the skip set
within a matching entry count, and entries expire after ``ttl_seconds``.
With a ``store`` (the proxy's :class:`~headroom.cache.result_store.ResultStore`),
every result is also written to disk and an entry the budget evicted is
read back from there instead of compressed again.
Thread safety: a ``threading.Lock`` guards all read-modify-write
operations. The ``apply()`` path runs compression inside a
@@ -1581,20 +1615,30 @@ class CompressionCache:
wasteful) and metrics counters would drift.
"""
def __init__(self, ttl_seconds: int = 1800):
def __init__(
self,
ttl_seconds: int = 1800,
max_bytes: int | None = None,
store: Any = None,
):
import threading
# Tier 2: compressed results {hash: (text, ratio, strategy, timestamp)}
self._results: dict[int, tuple[str, float, str, float]] = {}
# Tier 1: hashes of content that won't compress {hash: timestamp}
self._skip: dict[int, float] = {}
# Tier 2: {key: (text, ratio, strategy, created_at)}, least recently used first.
self._results: OrderedDict[str, tuple[str, float, str, float]] = OrderedDict()
self._bytes = 0
# Tier 1: {key: marked_at}, oldest first.
self._skip: OrderedDict[str, float] = OrderedDict()
self._max_bytes = max_bytes if max_bytes is not None else _compression_cache_budget_bytes()
self._max_skip = max(1000, self._max_bytes // (1024 * 1024) * _SKIP_ENTRIES_PER_MB)
self._store = store
# Callbacks invoked (outside the lock) whenever ``clear()`` runs, so
# sibling state keyed by the same content hashes (e.g. the router's
# sibling state keyed by the same content keys (e.g. the router's
# frozen-verdict store) can be reset in lock-step with the cache.
self._on_clear: list[Any] = []
self._ttl_seconds = ttl_seconds
# Metrics
self._hits = 0
self._disk_hits = 0
self._misses = 0
self._skip_hits = 0
self._evictions = 0
@@ -1602,7 +1646,7 @@ class CompressionCache:
self._lookup_count = 0
self._lock = threading.Lock()
def get(self, key: int) -> tuple[str, float, str] | None:
def get(self, key: str) -> tuple[str, float, str] | None:
"""Get cached compression result. Thread-safe.
Returns (compressed_text, ratio, strategy) or None if not found/expired.
@@ -1614,19 +1658,23 @@ class CompressionCache:
if entry is not None:
compressed, ratio, strategy, created_at = entry
if (time.monotonic() - created_at) < self._ttl_seconds:
self._results.move_to_end(key)
self._hits += 1
self._total_lookup_ns += time.perf_counter_ns() - t0
self._lookup_count += 1
self._record_lookup(t0)
return (compressed, ratio, strategy)
else:
del self._results[key]
self._evictions += 1
self._misses += 1
self._total_lookup_ns += time.perf_counter_ns() - t0
self._lookup_count += 1
return None
self._drop_locked(key)
self._evictions += 1
found = self._store.get(key) if self._store is not None else None
with self._lock:
if found is not None:
self._admit_locked(key, *found)
self._disk_hits += 1
else:
self._misses += 1
self._record_lookup(t0)
return found
def is_skipped(self, key: int) -> bool:
def is_skipped(self, key: str) -> bool:
"""Check if content is known non-compressible (Tier 1). Thread-safe."""
with self._lock:
ts = self._skip.get(key)
@@ -1634,27 +1682,55 @@ class CompressionCache:
if (time.monotonic() - ts) < self._ttl_seconds:
self._skip_hits += 1
return True
else:
del self._skip[key]
self._evictions += 1
del self._skip[key]
self._evictions += 1
return False
def put(self, key: int, compressed: str, ratio: float, strategy: str) -> None:
def put(self, key: str, compressed: str, ratio: float, strategy: str) -> None:
"""Store a compressed result (Tier 2). Thread-safe."""
with self._lock:
self._results[key] = (compressed, ratio, strategy, time.monotonic())
self._admit_locked(key, compressed, ratio, strategy)
if self._store is not None:
self._store.put(key, compressed, ratio, strategy)
def mark_skip(self, key: int) -> None:
def mark_skip(self, key: str) -> None:
"""Mark content as non-compressible (Tier 1). Thread-safe."""
with self._lock:
self._skip[key] = time.monotonic()
self._skip_locked(key)
def move_to_skip(self, key: int) -> None:
def move_to_skip(self, key: str) -> None:
"""Move a result to skip set (threshold tightened, no longer qualifies).
Thread-safe."""
with self._lock:
self._results.pop(key, None)
self._skip[key] = time.monotonic()
self._drop_locked(key)
self._skip_locked(key)
if self._store is not None:
self._store.delete(key)
def _admit_locked(self, key: str, compressed: str, ratio: float, strategy: str) -> None:
self._drop_locked(key)
self._results[key] = (compressed, ratio, strategy, time.monotonic())
self._bytes += sys.getsizeof(compressed) + _CACHE_ENTRY_OVERHEAD
# Least recently used first; the entry just added always stays.
while self._bytes > self._max_bytes and len(self._results) > 1:
self._drop_locked(next(iter(self._results)))
self._evictions += 1
def _drop_locked(self, key: str) -> None:
entry = self._results.pop(key, None)
if entry is not None:
self._bytes -= sys.getsizeof(entry[0]) + _CACHE_ENTRY_OVERHEAD
def _skip_locked(self, key: str) -> None:
self._skip.pop(key, None)
self._skip[key] = time.monotonic()
while len(self._skip) > self._max_skip:
self._skip.popitem(last=False)
self._evictions += 1
def _record_lookup(self, t0: int) -> None:
self._total_lookup_ns += time.perf_counter_ns() - t0
self._lookup_count += 1
@property
def size(self) -> int:
@@ -1672,10 +1748,13 @@ class CompressionCache:
avg_ns = self._total_lookup_ns / self._lookup_count if self._lookup_count else 0
return {
"cache_hits": self._hits,
"cache_disk_hits": self._disk_hits,
"cache_skip_hits": self._skip_hits,
"cache_misses": self._misses,
"cache_evictions": self._evictions,
"cache_size": len(self._results),
"cache_bytes": self._bytes,
"cache_max_bytes": self._max_bytes,
"cache_skip_size": len(self._skip),
"cache_avg_lookup_ns": avg_ns,
}
@@ -1683,7 +1762,7 @@ class CompressionCache:
def register_on_clear(self, callback: Any) -> None:
"""Register a callback fired (outside the lock) on every ``clear()``.
Lets state keyed by the same content hashes — e.g. the router's
Lets state keyed by the same content keys — e.g. the router's
``_frozen_verdicts`` store — be reset together with the cache so it
cannot outlive the results/skip tiers it shadows.
"""
@@ -1693,7 +1772,10 @@ class CompressionCache:
"""Clear all entries (e.g., on session end). Thread-safe."""
with self._lock:
self._results.clear()
self._bytes = 0
self._skip.clear()
if self._store is not None:
self._store.clear()
# Fire callbacks outside the lock to avoid cross-lock ordering issues.
for callback in self._on_clear:
callback()
@@ -1945,6 +2027,12 @@ class ContentRouterConfig:
# emits a `<<ccr:…>>` / `Retrieve …` retrieval marker. SmartCrusher is
# additionally forced marker-free via smart_crusher_lossless_only.
lossless: bool = False
# Back the compression cache with a SQLite file (see
# headroom.cache.result_store) so results its memory budget evicts, or a
# restart loses, come back without compressing again. The proxy turns it on;
# library callers keep the cache in memory unless they ask.
result_cache_disk: bool = False
# Cross-turn (whole-conversation) verbatim de-dup. Replaces a contiguous span
# in a later tool output that already appeared verbatim in an earlier tool
# output with an in-context pointer. Prefix-monotonic (cache-safe) and
@@ -2455,7 +2543,12 @@ class ContentRouter(Transform):
# are not gated. Now backed by the #3486 ContextVar state — see
# the ``_runtime_compression_policy`` property below.
self._cache = CompressionCache()
store = None
if self.config.result_cache_disk:
from ..cache.result_store import open_result_store
store = open_result_store(ttl_seconds=1800)
self._cache = CompressionCache(store=store)
# Cache-churn fix (HEADROOM_FREEZE_BLOCK_DECISION, default off):
# freeze each block's compress-vs-passthrough verdict on first
@@ -2473,7 +2566,7 @@ class ContentRouter(Transform):
# the parallel compression pass writes verdicts from worker threads, so
# we match that posture with a dedicated lock rather than relying on
# GIL atomicity (which would not protect the read-then-evict sequence).
self._frozen_verdicts: dict[int, bool] = {}
self._frozen_verdicts: dict[str, bool] = {}
# The store is process-wide (one router per pipeline, shared by every
# session), so the cap must scale with the number of CONCURRENT
# sessions, not one user's workload: at org scale (many users behind
@@ -2636,7 +2729,7 @@ class ContentRouter(Transform):
f"preserved_chars~={preserved} (cache bust avoided)"
)
def _record_frozen_verdict(self, content_key: int, verdict: bool) -> None:
def _record_frozen_verdict(self, content_key: str, verdict: bool) -> None:
"""Record a frozen verdict (thread-safe, bounded FIFO eviction).
Caps the store at ``_frozen_verdicts_max`` entries, evicting the
@@ -2653,7 +2746,7 @@ class ContentRouter(Transform):
del self._frozen_verdicts[oldest]
self._frozen_verdicts[content_key] = verdict
def _get_frozen_verdict(self, content_key: int) -> bool | None:
def _get_frozen_verdict(self, content_key: str) -> bool | None:
"""Read a frozen verdict (thread-safe). Returns None if absent."""
with self._frozen_lock:
return self._frozen_verdicts.get(content_key)
@@ -6136,7 +6229,7 @@ class ContentRouter(Transform):
# enforce_reversibility, precomputed_detection). The detection is the one
# already computed in the routing loop below, threaded through so a
# cache-miss message is not re-detected inside compress().
_PendingTask = tuple[int, str, str, float, int, bool, "DetectionResult | None"]
_PendingTask = tuple[int, str, str, float, str, bool, "DetectionResult | None"]
pending_tasks: list[_PendingTask] = []
# #856 P2b (flag-gated, default off): net-cost frozen-floor unlock.
@@ -6476,13 +6569,13 @@ class ContentRouter(Transform):
# Tier 2 (result): known compresses → reuse compressed text.
# Key on the runtime target_ratio too: the same content compressed at
# a different ratio is a different result, so it must not alias.
content_key = hash((content, getattr(self, "_runtime_target_ratio", None)))
# Tool ground truth is gated against lossy-unrecoverable results below
# (#1307). Partition its cache namespace so a gated tool entry is never
# served from — or poisons — an ungated entry for byte-identical content.
enforce_reversibility = role == "tool"
if enforce_reversibility:
content_key = hash((content_key, True))
content_key = content_cache_key(
content, getattr(self, "_runtime_target_ratio", None), gated=enforce_reversibility
)
# Tier 1: skip set — instant rejection
if self._cache.is_skipped(content_key):
@@ -7546,7 +7639,9 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, was_compressed = self._compress_block_content(
content=tool_text,
content_key=hash((tool_text, getattr(self, "_runtime_target_ratio", None))),
content_key=content_cache_key(
tool_text, getattr(self, "_runtime_target_ratio", None)
),
context=block_context,
bias=bias,
min_ratio=min_ratio,
@@ -7616,8 +7711,8 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, _was_compressed = self._compress_block_content(
content=text_content,
content_key=hash(
(text_content, getattr(self, "_runtime_target_ratio", None))
content_key=content_cache_key(
text_content, getattr(self, "_runtime_target_ratio", None)
),
context=context,
bias=1.0,
@@ -7649,7 +7744,7 @@ class ContentRouter(Transform):
def _compress_block_content(
self,
content: str,
content_key: int,
content_key: str,
context: str,
bias: float,
min_ratio: float,
@@ -7670,7 +7765,7 @@ class ContentRouter(Transform):
Args:
content: The string content to compress.
content_key: Pre-computed ``hash(content)`` for cache lookups.
content_key: Pre-computed :func:`content_cache_key` for cache lookups.
context: User/query context for relevance-aware compression.
bias: Compression bias multiplier (tool-specific or 1.0).
min_ratio: Adaptive minimum compression ratio threshold.
+56
View File
@@ -0,0 +1,56 @@
"""The content router's compression cache stays within its memory budget."""
from __future__ import annotations
import subprocess
import sys
from headroom.cache.result_store import ResultStore, open_result_store
from headroom.transforms.content_router import CompressionCache, content_cache_key
def test_results_stay_within_the_memory_budget_least_recently_used_first():
cache = CompressionCache(max_bytes=10_000)
for i in range(100):
cache.put(f"k{i}", "x" * 1000, 0.5, "kompress")
cache.get("k0") # k0 stays recently used
assert cache.stats["cache_bytes"] <= 10_000
assert cache.get("k0") is not None
assert cache.get("k1") is None # evicted, not kept forever
def test_an_evicted_result_comes_back_from_disk_unchanged(tmp_path):
cache = CompressionCache(max_bytes=3_000, store=ResultStore(tmp_path / "c.db", 1800))
cache.put("first", "compressed once", 0.4, "kompress")
for i in range(10):
cache.put(f"k{i}", "y" * 1000, 0.5, "kompress")
# Out of memory, but read back: no second compression, same bytes.
assert cache.get("first") == ("compressed once", 0.4, "kompress")
assert cache.stats["cache_disk_hits"] == 1
def test_the_skip_set_is_bounded_too():
cache = CompressionCache(max_bytes=1024 * 1024)
for i in range(50_000):
cache.mark_skip(f"s{i}")
assert cache.skip_size <= 2000
assert cache.is_skipped("s49999")
def test_keys_are_the_same_in_every_process():
code = (
"from headroom.transforms.content_router import content_cache_key;"
"print(content_cache_key('same text', 0.5, gated=True))"
)
other = subprocess.run(
[sys.executable, "-c", code], capture_output=True, text=True, check=True
).stdout.strip()
assert other == content_cache_key("same text", 0.5, gated=True)
assert content_cache_key("same text", 0.5) != other # gated tool output kept apart
def test_stateless_mode_writes_no_cache_file(tmp_path, monkeypatch):
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
monkeypatch.setenv("HEADROOM_STATELESS", "1")
assert open_result_store(1800) is None
assert not (tmp_path / "compression_cache.db").exists()