mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cache): keep prefix-tracker state across a graceful proxy restart
Opt-in HEADROOM_PREFIX_STATE_FILE=<path>: the proxy saves the SessionTrackerStore on graceful shutdown and restores it at startup, so the first request after a restart replays the byte-identical prefix the provider cached instead of recompressing the whole history and re-writing the prompt cache from the first changed message. Only trackers still inside their cache retention are restored; the file is versioned, written atomically at mode 0600, capped at the 256 newest trackers and 64 MB, deleted once loaded, and any bad file leaves the store empty. A restored tracker seeds the session's CompressionCache on its first Anthropic token-mode request, and read-maturation state rides along, so neither the freeze clamp nor maturation rewrites cached Reads. Unset, nothing is read or written. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -401,6 +401,7 @@ headroom proxy --learn --min-evidence 3
|
||||
| `HEADROOM_THINKING_COMPACT` | Compact plain-text reasoning that models re-send every turn (Kimi/GLM/DeepSeek `reasoning_content` / inline `<think>`): Kompress it on warm turns, drop it on cold turns. No-op for Claude/Codex/OpenAI (encrypted reasoning). See [Cold-prefix hook](#cold-prefix-hook--reasoning-compaction). | `off` |
|
||||
| `HEADROOM_THINKING_COMPACT_KEEP_LAST` | Most-recent assistant turns whose reasoning is left intact (the active reasoning the model still uses). Only applies with `HEADROOM_THINKING_COMPACT`. | `1` |
|
||||
| `HEADROOM_COLD_RECOMPACT` | On a confirmed-cold turn (idle past the real cache TTL), recompact the whole prefix — cross-turn dedupe + superseded-read drop + lossless folds — instead of forwarding a byte-identical prefix to a dead cache. Warm turns unchanged. Pair with `HEADROOM_DEDUPE`. See [Cold-prefix hook](#cold-prefix-hook--reasoning-compaction). | `off` |
|
||||
| `HEADROOM_PREFIX_STATE_FILE` | Path where the proxy saves its prefix-cache trackers on a graceful shutdown and restores them on the next start (then deletes the file), so warm conversations replay the exact prefix the provider cached instead of re-writing it. The file holds conversation content: it is written `0600`, capped at the 256 newest conversations and 64 MB, and only trackers still inside their cache lifetime are restored. Not supported with multiple workers; ignored in stateless mode. | -- |
|
||||
| `HEADROOM_DEDUPE` | Whole-conversation verbatim cross-turn dedup in the router (cache-safe, information-preserving via retrieval markers). Superseded-read drop + lossless folds run without it; this adds verbatim dedup. | `off` |
|
||||
| `HEADROOM_CACHE_TTL_LEARN` | Append per-turn cache-outcome observations (provider, model, idle, hit/miss) to `cache_ttl_observations.jsonl` for the offline `headroom-cache-ttl` learner. Observation-only (no request-behavior change); respects `HEADROOM_STATELESS`; the log is size-bounded. | `off` |
|
||||
| `HEADROOM_KOMPRESS_ENDPOINT` / `HEADROOM_KOMPRESS_ENDPOINT_TOKEN` | Offload ML compression (Kompress) to a remote endpoint instead of the local ONNX model — used by reasoning compaction and the router when set. | -- |
|
||||
|
||||
Vendored
+229
@@ -26,6 +26,7 @@ import time
|
||||
from collections import OrderedDict, deque
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -71,6 +72,20 @@ _CACHE_TTL_CLEANUP_MARGIN_SECONDS = 60
|
||||
# of forks off one turn leaves that turn resumable.
|
||||
_LINEAGE_HISTORY = 3
|
||||
|
||||
# Opt-in file for SessionTrackerStore state across a graceful restart. It holds
|
||||
# conversation content, so nothing is written unless an operator sets it.
|
||||
PREFIX_STATE_FILE_ENV = "HEADROOM_PREFIX_STATE_FILE"
|
||||
_PREFIX_STATE_VERSION = 1
|
||||
# Newest trackers first; whatever does not fit starts cold, as all did before.
|
||||
_PREFIX_STATE_MAX_TRACKERS = 256
|
||||
_PREFIX_STATE_MAX_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def prefix_state_file() -> str | None:
|
||||
"""The configured tracker state file, or None when persistence is off."""
|
||||
return os.environ.get(PREFIX_STATE_FILE_ENV, "").strip() or None
|
||||
|
||||
|
||||
# (tracker, turn, resumed state) the current request resolved to. Each proxied
|
||||
# request runs in its own asyncio task, so it reads the state it resumed and
|
||||
# ``update_from_response`` records its response against its own turn, even
|
||||
@@ -1180,6 +1195,12 @@ class PrefixCacheTracker:
|
||||
# lazily by the handler when read maturation is enabled. Rides
|
||||
# here so it shares the session's affinity and TTL cleanup.
|
||||
self.read_maturation_manager: Any = None
|
||||
# Set by SessionTrackerStore.load_state: this process has no
|
||||
# CompressionCache entries behind the restored prefix yet, and the
|
||||
# handler adopts the saved maturation state (validated at load) when it
|
||||
# builds the manager.
|
||||
self.restored_from_disk = False
|
||||
self.restored_maturation_state: dict[str, Any] | None = None
|
||||
|
||||
# Stats
|
||||
self._busts_avoided: int = 0
|
||||
@@ -1609,6 +1630,25 @@ def _lineage_snapshot(obj: Any) -> Any:
|
||||
return obj
|
||||
|
||||
|
||||
def _deref(table: list[Any], refs: list[int]) -> list[Any]:
|
||||
"""Rebuild a saved message list. Restored lists share message objects,
|
||||
which is safe: tracker and lineage lists are never mutated in place."""
|
||||
if not all(type(i) is int and 0 <= i < len(table) for i in refs):
|
||||
raise ValueError("message reference out of range")
|
||||
return [table[i] for i in refs]
|
||||
|
||||
|
||||
def _decode_turn_state(table: list[Any], state: list[Any]) -> tuple[Any, ...]:
|
||||
turn, cached_tokens, cached_messages, original, forwarded = state
|
||||
return (
|
||||
int(turn),
|
||||
int(cached_tokens),
|
||||
int(cached_messages),
|
||||
_deref(table, original),
|
||||
_deref(table, forwarded),
|
||||
)
|
||||
|
||||
|
||||
class SessionTrackerStore:
|
||||
"""Manages PrefixCacheTracker instances across sessions.
|
||||
|
||||
@@ -2007,3 +2047,192 @@ class SessionTrackerStore:
|
||||
def active_sessions(self) -> int:
|
||||
"""Number of active session trackers (one per conversation lineage)."""
|
||||
return len(self._trackers)
|
||||
|
||||
def save_state(self, path: str) -> int:
|
||||
"""Write the live trackers to ``path`` for the next process to resume.
|
||||
|
||||
Called on graceful shutdown when ``HEADROOM_PREFIX_STATE_FILE`` is set.
|
||||
Without it, a restart leaves every warm conversation with no tracker:
|
||||
its next request recompresses the whole history into bytes that differ
|
||||
from the prefix the provider cached, and the cache is re-written from
|
||||
the first changed message. Newest trackers first, capped at
|
||||
``_PREFIX_STATE_MAX_TRACKERS`` and ``_PREFIX_STATE_MAX_BYTES``; a
|
||||
tracker whose messages are not JSON is skipped. Never raises; returns
|
||||
the number of trackers written.
|
||||
"""
|
||||
from headroom.fsutil import write_text
|
||||
from headroom.paths import persistence_allowed
|
||||
|
||||
if not persistence_allowed("prefix-cache tracker state"):
|
||||
return 0
|
||||
sessions = {key: sid for sid, family in self._lineages.items() for key in family}
|
||||
live = sorted(
|
||||
(item for item in self._trackers.items() if not item[1].is_expired),
|
||||
key=lambda item: item[1]._last_activity,
|
||||
reverse=True,
|
||||
)
|
||||
entries: list[str] = []
|
||||
size = 0
|
||||
for key, tracker in live[:_PREFIX_STATE_MAX_TRACKERS]:
|
||||
try:
|
||||
entry = self._encode_tracker(key, tracker, sessions.get(key))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if size + len(entry) > _PREFIX_STATE_MAX_BYTES:
|
||||
continue
|
||||
size += len(entry) + 1
|
||||
entries.append(entry)
|
||||
try:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
# write_text keeps an existing file's mode; a fresh one is 0600.
|
||||
target.unlink(missing_ok=True)
|
||||
write_text(
|
||||
target, f'{{"version":{_PREFIX_STATE_VERSION},"trackers":[{",".join(entries)}]}}'
|
||||
)
|
||||
except OSError as exc:
|
||||
logger.warning("SessionTrackerStore: prefix state not saved to %s: %s", path, exc)
|
||||
return 0
|
||||
logger.info(
|
||||
"SessionTrackerStore: saved %d of %d prefix trackers (%d bytes) to %s",
|
||||
len(entries),
|
||||
len(self._trackers),
|
||||
size,
|
||||
path,
|
||||
)
|
||||
return len(entries)
|
||||
|
||||
def _encode_tracker(self, key: str, tracker: PrefixCacheTracker, session_id: str | None) -> str:
|
||||
"""One tracker as JSON, each distinct message stored once.
|
||||
|
||||
Successive turns, the lineage snapshots and the original/forwarded
|
||||
pairs mostly repeat the same messages, so this keeps the file near one
|
||||
copy of each conversation instead of up to eleven.
|
||||
"""
|
||||
index: dict[str, int] = {}
|
||||
memo: dict[int, list[int]] = {}
|
||||
|
||||
def ref(messages: list[Any]) -> list[int]:
|
||||
if id(messages) not in memo:
|
||||
refs = []
|
||||
for message in messages:
|
||||
blob = json.dumps(message)
|
||||
refs.append(index.setdefault(blob, len(index)))
|
||||
memo[id(messages)] = refs
|
||||
return memo[id(messages)]
|
||||
|
||||
def state(s: tuple[Any, ...]) -> list[Any]:
|
||||
return [s[0], s[1], s[2], ref(s[3]), ref(s[4])]
|
||||
|
||||
maturation = tracker.read_maturation_manager
|
||||
meta = {
|
||||
"key": key,
|
||||
"session": session_id,
|
||||
"affinity": self._lineage_affinities.get(key),
|
||||
"provider": tracker.provider,
|
||||
"last_activity": tracker._last_activity,
|
||||
"cache_ttl": tracker._cache_ttl_seconds,
|
||||
"state": state(tracker._turn_state()),
|
||||
"turns": [[t, state(s)] for t, s in tracker._turn_states.items() if s is not None],
|
||||
"history": [[ref(snap), t] for snap, t in self._lineage_history.get(key, ())],
|
||||
"maturation": None if maturation is None else maturation.export_state(),
|
||||
}
|
||||
return f'{{"meta":{json.dumps(meta)},"messages":[{",".join(index)}]}}'
|
||||
|
||||
def load_state(self, path: str) -> int:
|
||||
"""Resume the trackers :meth:`save_state` wrote, before serving.
|
||||
|
||||
Trackers past their retention are dropped. The file is removed once
|
||||
loaded, so a later crash cannot bring back state older than what it
|
||||
lost. A missing file is normal; an unreadable or invalid one is logged
|
||||
and leaves the store empty. Never raises; returns trackers restored.
|
||||
"""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
restored = self._decode_state(data)
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
except Exception as exc: # any bad file must leave boot unaffected
|
||||
logger.warning(
|
||||
"SessionTrackerStore: ignoring prefix state %s (%s: %s)",
|
||||
path,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
return 0
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError as exc:
|
||||
logger.warning("SessionTrackerStore: could not remove %s: %s", path, exc)
|
||||
logger.info(
|
||||
"SessionTrackerStore: restored %d of %d prefix trackers from %s",
|
||||
restored,
|
||||
len(data["trackers"]),
|
||||
path,
|
||||
)
|
||||
return restored
|
||||
|
||||
def _decode_state(self, data: Any) -> int:
|
||||
"""Validate the whole file first, then install it; a bad file installs nothing."""
|
||||
if data.get("version") != _PREFIX_STATE_VERSION:
|
||||
raise ValueError(f"unsupported version {data.get('version')!r}")
|
||||
trackers: dict[str, PrefixCacheTracker] = {}
|
||||
families: dict[str, OrderedDict[str, list[Any]]] = {}
|
||||
affinities: dict[str, str | None] = {}
|
||||
histories: dict[str, deque[tuple[list[Any], int]]] = {}
|
||||
now = time.time()
|
||||
# Oldest first, so each family keeps its lineages in creation order.
|
||||
for entry in reversed(data["trackers"]):
|
||||
meta, table = entry["meta"], entry["messages"]
|
||||
tracker = PrefixCacheTracker(str(meta["provider"]), self._default_config)
|
||||
tracker._last_activity = float(meta["last_activity"])
|
||||
tracker.observe_cache_ttl(meta["cache_ttl"])
|
||||
# NaN or a future stamp would never expire.
|
||||
if tracker.is_expired or not tracker._last_activity <= now:
|
||||
continue
|
||||
(
|
||||
tracker._turn_number,
|
||||
tracker._cached_token_count,
|
||||
tracker._cached_message_count,
|
||||
tracker._last_original_messages,
|
||||
tracker._last_forwarded_messages,
|
||||
) = _decode_turn_state(table, meta["state"])
|
||||
tracker._turn_states = {int(t): _decode_turn_state(table, s) for t, s in meta["turns"]}
|
||||
tracker.restored_from_disk = True
|
||||
if meta["maturation"] is not None:
|
||||
from headroom.transforms.read_maturation import ReadMaturationManager
|
||||
|
||||
tracker.restored_maturation_state = ReadMaturationManager.validate_state(
|
||||
meta["maturation"]
|
||||
)
|
||||
key = str(meta["key"])
|
||||
trackers[key] = tracker
|
||||
history = deque(
|
||||
((_deref(table, snap), int(t)) for snap, t in meta["history"]),
|
||||
maxlen=_LINEAGE_HISTORY,
|
||||
)
|
||||
if not history:
|
||||
continue
|
||||
histories[key] = history
|
||||
if meta["session"] is not None:
|
||||
families.setdefault(str(meta["session"]), OrderedDict())[key] = history[0][0]
|
||||
affinity = meta["affinity"]
|
||||
affinities[key] = None if affinity is None else str(affinity)
|
||||
|
||||
# New turns and lineages must not reuse a restored id.
|
||||
turns = [t for h in histories.values() for _, t in h]
|
||||
turns += [t for tracker in trackers.values() for t in tracker._turn_states]
|
||||
lineage_ns = [
|
||||
int(n)
|
||||
for _, sep, n in (key.rpartition("\x00") for key in trackers)
|
||||
if sep and n.isdecimal()
|
||||
]
|
||||
self._turn_counter = itertools.count(max([next(self._turn_counter), *turns]) + 1)
|
||||
self._lineage_counter = itertools.count(max([next(self._lineage_counter), *lineage_ns]) + 1)
|
||||
self._trackers.update(trackers)
|
||||
self._lineage_history.update(histories)
|
||||
self._lineage_affinities.update(affinities)
|
||||
for session_id, family in families.items():
|
||||
self._lineages.setdefault(session_id, OrderedDict()).update(family)
|
||||
return len(trackers)
|
||||
|
||||
@@ -2029,6 +2029,15 @@ class AnthropicHandlerMixin:
|
||||
compression_policy = resolve_policy(getattr(request.state, "auth_mode", None))
|
||||
if is_token_mode(self.config.mode):
|
||||
comp_cache = self._get_compression_cache(session_id)
|
||||
if getattr(prefix_tracker, "restored_from_disk", False):
|
||||
# Resumed after a restart (HEADROOM_PREFIX_STATE_FILE):
|
||||
# seed the empty cache with what was forwarded, or the
|
||||
# clamp below freezes nothing and read maturation
|
||||
# rewrites Reads inside the cached prefix.
|
||||
prefix_tracker.restored_from_disk = False
|
||||
comp_cache.update_from_result(
|
||||
previous_original_messages, previous_forwarded_messages
|
||||
)
|
||||
|
||||
# Freeze + stable marking + Zone-1 swap now live in the
|
||||
# shared session engine (PROXY policy: clamp by BOTH the
|
||||
@@ -2609,6 +2618,10 @@ class AnthropicHandlerMixin:
|
||||
compression_store=get_compression_store(),
|
||||
)
|
||||
prefix_tracker.read_maturation_manager = maturation_mgr
|
||||
state = getattr(prefix_tracker, "restored_maturation_state", None)
|
||||
if state is not None:
|
||||
prefix_tracker.restored_maturation_state = None
|
||||
maturation_mgr.restore_state(state)
|
||||
_maturation_mgr = maturation_mgr
|
||||
maturation = maturation_mgr.apply(
|
||||
optimized_messages,
|
||||
|
||||
@@ -1203,6 +1203,8 @@ class HeadroomProxy(
|
||||
session_ttl_seconds=config.prefix_freeze_session_ttl,
|
||||
)
|
||||
)
|
||||
# Set by startup() when HEADROOM_PREFIX_STATE_FILE opts in.
|
||||
self._prefix_state_file: str | None = None
|
||||
|
||||
# Compression cache store for token mode (session-scoped). The dict
|
||||
# itself is mutated under `_compression_caches_lock`; the per-session
|
||||
@@ -2048,6 +2050,17 @@ class HeadroomProxy(
|
||||
async def startup(self):
|
||||
"""Initialize async resources."""
|
||||
self._get_shutdown_event().clear()
|
||||
# Opt-in: resume the prefix trackers the previous process saved on its
|
||||
# graceful shutdown, so warm conversations replay their cached prefix.
|
||||
from headroom.cache.prefix_tracker import PREFIX_STATE_FILE_ENV, prefix_state_file
|
||||
|
||||
self._prefix_state_file = prefix_state_file()
|
||||
if self._prefix_state_file and _MULTI_WORKER_CONFIG_ENV in os.environ:
|
||||
# Each worker holds its own trackers; one file cannot hold them all.
|
||||
logger.warning("%s ignored: not supported with multiple workers", PREFIX_STATE_FILE_ENV)
|
||||
self._prefix_state_file = None
|
||||
if self._prefix_state_file:
|
||||
self.session_tracker_store.load_state(self._prefix_state_file)
|
||||
self.pipeline_extensions.emit(
|
||||
PipelineStage.PRE_START,
|
||||
operation="proxy.startup",
|
||||
@@ -2395,6 +2408,9 @@ class HeadroomProxy(
|
||||
async def shutdown(self):
|
||||
"""Cleanup async resources."""
|
||||
self._get_shutdown_event().set()
|
||||
# First, before any await can hit the shutdown step timeout.
|
||||
if self._prefix_state_file:
|
||||
self.session_tracker_store.save_state(self._prefix_state_file)
|
||||
if self.http_client_h1 and self.http_client_h1 is not self.http_client:
|
||||
await self.http_client_h1.aclose()
|
||||
self.http_client_h1 = None
|
||||
|
||||
@@ -153,6 +153,31 @@ class ReadMaturationManager:
|
||||
# once per session (content and marker are stable per tool call).
|
||||
self._replay_token_deltas: dict[str, int] = {}
|
||||
|
||||
def export_state(self) -> dict[str, Any]:
|
||||
"""Session state as JSON, for ``SessionTrackerStore.save_state``."""
|
||||
return {
|
||||
"matured": {k: [m.marker, m.ccr_hash] for k, m in self._matured.items()},
|
||||
"booked": sorted(self._booked),
|
||||
"deltas": self._replay_token_deltas,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def validate_state(state: Any) -> dict[str, Any]:
|
||||
"""Normalize :meth:`export_state` output read from disk. Raises on a bad
|
||||
shape, so a bad file is rejected at load and never inside a request."""
|
||||
return {
|
||||
"matured": {str(k): [str(m), str(h)] for k, (m, h) in state["matured"].items()},
|
||||
"booked": [str(k) for k in state["booked"]],
|
||||
"deltas": {str(k): int(v) for k, v in state["deltas"].items()},
|
||||
}
|
||||
|
||||
def restore_state(self, state: dict[str, Any]) -> None:
|
||||
"""Adopt :meth:`validate_state` output from the previous process, so a
|
||||
matured Read's removal is not booked again after a restart."""
|
||||
self._matured = {k: MaturedRead(m, h) for k, (m, h) in state["matured"].items()}
|
||||
self._booked = set(state["booked"])
|
||||
self._replay_token_deltas = dict(state["deltas"])
|
||||
|
||||
# ─── Per-request entry point ────────────────────────────────────────
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -156,8 +156,13 @@ class _PromptCache:
|
||||
return read, write, hit
|
||||
|
||||
|
||||
def _drive(requests: list[list[dict[str, Any]]]) -> list[tuple[list[dict[str, Any]], int]]:
|
||||
"""Send each history through the real handler; return (forwarded, read_hit)."""
|
||||
def _drive(
|
||||
requests: list[list[dict[str, Any]]], cache: _PromptCache | None = None, **config: Any
|
||||
) -> list[tuple[list[dict[str, Any]], int]]:
|
||||
"""Send each history through the real handler; return (forwarded, read_hit).
|
||||
|
||||
One app per call: a second call with the same ``cache`` is a proxy restart
|
||||
in front of a provider whose prompt cache stayed warm."""
|
||||
app = create_app(
|
||||
ProxyConfig(
|
||||
optimize=True,
|
||||
@@ -168,9 +173,10 @@ def _drive(requests: list[list[dict[str, Any]]]) -> list[tuple[list[dict[str, An
|
||||
rate_limit_enabled=False,
|
||||
cost_tracking_enabled=False,
|
||||
log_requests=False,
|
||||
**config,
|
||||
)
|
||||
)
|
||||
cache = _PromptCache()
|
||||
cache = cache or _PromptCache()
|
||||
seen: list[tuple[list[dict[str, Any]], int]] = []
|
||||
with TestClient(app) as client:
|
||||
proxy = client.app.state.proxy
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
"""A graceful proxy restart must not cost warm conversations their prompt cache.
|
||||
|
||||
Production, 2026-10-19: five graceful restarts in one day (app relaunches for
|
||||
updates) each made the next request of every active Claude Code session read
|
||||
only the ~25k-token tools + system prefix and re-write the rest, 247k-432k
|
||||
tokens at a time, about a third of that day's cache writes. After a restart
|
||||
the proxy has no ``PrefixCacheTracker`` for the conversation, so it compresses
|
||||
the whole history afresh and the early messages come out as different bytes
|
||||
than the prefix it forwarded before the restart.
|
||||
|
||||
``HEADROOM_PREFIX_STATE_FILE`` (opt-in) saves the tracker store on graceful
|
||||
shutdown and restores it at startup, so the first request after a restart
|
||||
replays the exact prefix the provider cached.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from headroom.cache import prefix_tracker
|
||||
from headroom.cache.compression_store import reset_compression_store
|
||||
from headroom.cache.prefix_tracker import PREFIX_STATE_FILE_ENV, SessionTrackerStore
|
||||
from headroom.transforms.pipeline import TransformPipeline
|
||||
from tests.test_anthropic_fork_lineage_cache import (
|
||||
_drive,
|
||||
_main,
|
||||
_PromptCache,
|
||||
_send,
|
||||
_strip,
|
||||
_summary_fork,
|
||||
)
|
||||
|
||||
|
||||
def _wire(messages: list[dict[str, Any]]) -> str:
|
||||
"""The bytes the provider keys its prefix cache on (breakpoints excluded)."""
|
||||
return json.dumps(_strip(messages))
|
||||
|
||||
|
||||
def _with_fake_kompress(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A deterministic stand-in for the Kompress ML stage (off in this suite).
|
||||
|
||||
A cold start forwards the fast pass, which skips Kompress, while warm
|
||||
turns compress their new messages with it. So after a restart without
|
||||
state, the cold fast pass re-emits every message Kompressed after the
|
||||
first turn in a different form, which re-writes the cache from there."""
|
||||
real_apply = TransformPipeline.apply
|
||||
|
||||
def apply(self, messages, model, **kwargs): # noqa: ANN001, ANN003, ANN202
|
||||
skip = kwargs.get("skip_kompress", False)
|
||||
frozen = kwargs.get("frozen_message_count", 0) or 0
|
||||
result = real_apply(self, messages, model, **kwargs)
|
||||
if skip:
|
||||
return result
|
||||
out = []
|
||||
for i, message in enumerate(result.messages):
|
||||
content = message.get("content")
|
||||
if i >= frozen and isinstance(content, list):
|
||||
blocks = []
|
||||
for block in content:
|
||||
text = block.get("content") if isinstance(block, dict) else None
|
||||
if block.get("type") == "tool_result" and isinstance(text, str):
|
||||
if not text.endswith(" [kompressed]"):
|
||||
block = {**block, "content": text[: len(text) // 2] + " [kompressed]"}
|
||||
blocks.append(block)
|
||||
message = {**message, "content": blocks}
|
||||
out.append(message)
|
||||
result.messages = out
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(TransformPipeline, "apply", apply)
|
||||
|
||||
|
||||
def _read_turn(i: int) -> list[dict[str, Any]]:
|
||||
tid = f"toolu_r{i:02d}"
|
||||
body = "\n".join(f"{n:5d} line {n} of file {i}: some code here" for n in range(300))
|
||||
return [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": tid, "name": "Read", "input": {"file_path": f"/f{i}.py"}}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": tid, "content": body}]},
|
||||
]
|
||||
|
||||
|
||||
def _reads(turns: int) -> list[dict[str, Any]]:
|
||||
"""One Read per turn; Claude Code's breakpoint rides the newest block."""
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Review the change."}]}
|
||||
]
|
||||
for i in range(turns):
|
||||
messages += _read_turn(i)
|
||||
messages[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
|
||||
return messages
|
||||
|
||||
|
||||
# (history builder, turns before the restart, proxy config, fake Kompress)
|
||||
_SCENARIOS = {
|
||||
# Turns 1-2, restart, turn 3: Kompressed messages come back fast-passed.
|
||||
"kompress": (_main, 2, {}, True),
|
||||
# Reads held while active then cached verbatim stay verbatim; a fresh
|
||||
# process with an empty CompressionCache freezes nothing and matures them.
|
||||
"read-maturation": (
|
||||
_reads,
|
||||
5,
|
||||
{
|
||||
"read_maturation": True,
|
||||
"read_maturation_quiesce_turns": 2,
|
||||
"read_maturation_min_size_bytes": 100,
|
||||
},
|
||||
False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _restart_between_turns(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
|
||||
"""Some turns in one proxy, the next turn in a fresh one, one warm
|
||||
provider cache. Returns (last forwarded before, first forwarded after,
|
||||
its read hit)."""
|
||||
build, turns, config, kompress = _SCENARIOS[scenario]
|
||||
monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")
|
||||
# As the desktop app runs it: a cold start forwards the fast pass.
|
||||
monkeypatch.setenv("HEADROOM_BACKGROUND_COMPRESSION", "1")
|
||||
monkeypatch.setenv("HEADROOM_BACKGROUND_COMPRESSION_MIN_TOKENS", "1")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
if kompress:
|
||||
_with_fake_kompress(monkeypatch)
|
||||
cache = _PromptCache()
|
||||
reset_compression_store()
|
||||
before = _drive([build(n) for n in range(1, turns + 1)], cache, **config)
|
||||
reset_compression_store() # nothing in process memory survives a restart
|
||||
(after,) = _drive([build(turns + 1)], cache, **config)
|
||||
return before[-1][0], after[0], after[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", list(_SCENARIOS))
|
||||
def test_first_request_after_a_restart_replays_the_cached_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str
|
||||
) -> None:
|
||||
state = tmp_path / "prefix-state.json"
|
||||
monkeypatch.setenv(PREFIX_STATE_FILE_ENV, str(state))
|
||||
|
||||
previous, current, hit = _restart_between_turns(monkeypatch, tmp_path, scenario)
|
||||
|
||||
assert _wire(current[: len(previous)]) == _wire(previous)
|
||||
assert hit >= len(previous) - 1, f"read only {hit + 1}/{len(previous)} cached messages"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", list(_SCENARIOS))
|
||||
def test_without_the_state_file_a_restart_rewrites_the_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str
|
||||
) -> None:
|
||||
"""Default behavior is unchanged, and it is the bug the opt-in fixes, so
|
||||
the test above is not vacuous."""
|
||||
monkeypatch.delenv(PREFIX_STATE_FILE_ENV, raising=False)
|
||||
|
||||
previous, current, hit = _restart_between_turns(monkeypatch, tmp_path, scenario)
|
||||
|
||||
assert _wire(current[: len(previous)]) != _wire(previous)
|
||||
assert hit < len(previous) - 1
|
||||
assert list(tmp_path.iterdir()) == [], "nothing may be written without the opt-in"
|
||||
|
||||
|
||||
def _two_conversations() -> SessionTrackerStore:
|
||||
store = SessionTrackerStore()
|
||||
contextvars.Context().run(_send, store, _strip(_main(3)))
|
||||
contextvars.Context().run(_send, store, [{"role": "user", "content": "unrelated"}])
|
||||
return store
|
||||
|
||||
|
||||
def test_restored_lineages_resume_and_new_ones_get_fresh_keys(tmp_path: Path) -> None:
|
||||
"""Store level: a fork answered last still leaves the main turn resumable,
|
||||
and a new conversation never lands on a restored lineage's key."""
|
||||
store = _two_conversations()
|
||||
base = _strip(_main(3))
|
||||
contextvars.Context().run(_send, store, _strip(_summary_fork(3)))
|
||||
path = str(tmp_path / "state.json")
|
||||
assert store.save_state(path) == 2
|
||||
|
||||
restored = SessionTrackerStore()
|
||||
assert restored.load_state(path) == 2
|
||||
main = contextvars.Context().run(
|
||||
restored.resolve_tracker, "sid", "anthropic", messages=_strip(_main(4))
|
||||
)
|
||||
new = contextvars.Context().run(
|
||||
restored.resolve_tracker, "sid", "anthropic", messages=[{"role": "user", "content": "new"}]
|
||||
)
|
||||
|
||||
assert main.get_frozen_message_count() == len(base)
|
||||
assert main.get_last_original_messages() == base
|
||||
assert restored.active_sessions == 3
|
||||
assert new is not main and new.get_frozen_message_count() == 0
|
||||
|
||||
|
||||
def test_expired_trackers_are_not_restored(tmp_path: Path) -> None:
|
||||
path = tmp_path / "state.json"
|
||||
_two_conversations().save_state(str(path))
|
||||
data = json.loads(path.read_text())
|
||||
data["trackers"][0]["meta"]["last_activity"] -= 3600 # past the 600s retention
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
assert SessionTrackerStore().load_state(str(path)) == 1
|
||||
|
||||
|
||||
def _bad_ref(path: Path) -> None:
|
||||
data = json.loads(path.read_text())
|
||||
data["trackers"][0]["meta"]["state"][3] = [-1]
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
|
||||
def _bad_maturation(path: Path) -> None:
|
||||
data = json.loads(path.read_text())
|
||||
data["trackers"][-1]["meta"]["maturation"] = {"matured": [1], "booked": [], "deltas": {}}
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corrupt",
|
||||
[
|
||||
lambda p: p.write_text('{"version": 1, "trackers": [{"meta"'),
|
||||
lambda p: p.write_text('{"version": 99, "trackers": []}'),
|
||||
lambda p: p.write_text("[1, 2]"),
|
||||
_bad_ref,
|
||||
_bad_maturation,
|
||||
],
|
||||
ids=["truncated", "version", "shape", "bad-ref", "bad-maturation"],
|
||||
)
|
||||
def test_a_bad_state_file_starts_empty(tmp_path: Path, caplog, corrupt) -> None: # noqa: ANN001
|
||||
path = tmp_path / "state.json"
|
||||
_two_conversations().save_state(str(path))
|
||||
corrupt(path)
|
||||
|
||||
store = SessionTrackerStore()
|
||||
assert store.load_state(str(path)) == 0
|
||||
assert store.active_sessions == 0
|
||||
assert "ignoring prefix state" in caplog.text
|
||||
|
||||
|
||||
def test_state_file_is_private_and_removed_once_loaded(tmp_path: Path) -> None:
|
||||
path = tmp_path / "state.json"
|
||||
path.write_text("stale")
|
||||
path.chmod(0o644)
|
||||
|
||||
assert _two_conversations().save_state(str(path)) == 2
|
||||
if os.name == "posix":
|
||||
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
||||
assert SessionTrackerStore().load_state(str(path)) == 2
|
||||
assert not path.exists()
|
||||
assert SessionTrackerStore().load_state(str(path)) == 0 # missing file: cold start
|
||||
|
||||
|
||||
def test_save_keeps_the_newest_trackers_within_the_cap(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
store = _two_conversations()
|
||||
newest = max(store._trackers.values(), key=lambda t: t._last_activity)
|
||||
monkeypatch.setattr(prefix_tracker, "_PREFIX_STATE_MAX_TRACKERS", 1)
|
||||
path = str(tmp_path / "state.json")
|
||||
assert store.save_state(path) == 1
|
||||
|
||||
restored = SessionTrackerStore()
|
||||
restored.load_state(path)
|
||||
(kept,) = restored._trackers.values()
|
||||
assert kept.get_last_original_messages() == newest.get_last_original_messages()
|
||||
|
||||
monkeypatch.setattr(prefix_tracker, "_PREFIX_STATE_MAX_BYTES", 10)
|
||||
assert store.save_state(path) == 0
|
||||
|
||||
|
||||
def test_stateless_mode_writes_nothing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setenv("HEADROOM_STATELESS", "1")
|
||||
path = tmp_path / "state.json"
|
||||
assert _two_conversations().save_state(str(path)) == 0
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_read_maturation_state_survives_a_restart(tmp_path: Path) -> None:
|
||||
"""Matured Reads stay booked once: without this a restart books their
|
||||
removal again on every later request of the conversation."""
|
||||
from headroom.config import ReadMaturationConfig
|
||||
from headroom.transforms.read_maturation import MaturedRead, ReadMaturationManager
|
||||
|
||||
store = _two_conversations()
|
||||
manager = ReadMaturationManager(ReadMaturationConfig(enabled=True))
|
||||
manager._matured["toolu_r00"] = MaturedRead(marker="[Read of /f0.py ...]", ccr_hash="abc")
|
||||
manager._booked.add("toolu_r00")
|
||||
manager._replay_token_deltas["toolu_r00"] = 640
|
||||
newest = max(store._trackers.values(), key=lambda t: t._last_activity)
|
||||
newest.read_maturation_manager = manager
|
||||
path = str(tmp_path / "state.json")
|
||||
store.save_state(path)
|
||||
|
||||
restored = SessionTrackerStore()
|
||||
restored.load_state(path)
|
||||
states = [t.restored_maturation_state for t in restored._trackers.values()]
|
||||
assert all(t.restored_from_disk for t in restored._trackers.values())
|
||||
(state,) = [s for s in states if s is not None]
|
||||
adopted = ReadMaturationManager(ReadMaturationConfig(enabled=True))
|
||||
adopted.restore_state(state)
|
||||
|
||||
assert adopted._matured == manager._matured
|
||||
assert adopted._booked == manager._booked
|
||||
assert adopted._replay_token_deltas == manager._replay_token_deltas
|
||||
Reference in New Issue
Block a user