fix(cache): compress cache-mode deltas after a replaced system tail

Cache mode only splits a request into the previously forwarded prefix
plus a new delta when the history is an exact or append-only extension
of the recorded one. A request that replaced the previous trailing
role:"system" reminder diverges at that message, so it fell back to
forwarding the raw request: the history before the reminder was kept by
the cached-prefix replay, but every new message after it went out
uncompressed. On main, where the reminder also cost the tracker, the
first reminder turn after a compressed turn re-sent the compressed tool
results raw and re-wrote the provider cache.

extract_cache_stable_delta now accepts that shape when the history
before the reminder, and the model reply recorded after it, match the
current request position for position. The previously forwarded bytes
are replayed without the old reminder and everything after it is the
delta to compress. The rule reuses _extends_past_system_tail, so the
same guards apply, and HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0 disables it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-02 16:25:46 +02:00
co-authored by Claude Opus 5.5
parent cfe5ae9a08
commit 5b0c20f264
3 changed files with 171 additions and 0 deletions
+48
View File
@@ -402,6 +402,46 @@ def _extends_past_system_tail(current_messages: list[Any], previous_messages: li
)
def _replaced_system_tail_delta(
current_messages: list[dict[str, Any]],
previous_original_messages: list[dict[str, Any]],
previous_forwarded_messages: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]] | None:
"""Cache-stable split for a request that replaced the previous trailing system turn.
The handlers record the model's reply after the request it answered, so the
replaced reminder is the last recorded message or sits right before that
reply. It is left out of the replayed prefix (it is not in this request);
every other recorded message must match ``current`` position for position.
"""
if not _transient_system_lineage_enabled() or len(previous_forwarded_messages) != len(
previous_original_messages
):
return None
current = _lineage_snapshot(_canonicalize_for_prefix_compare(current_messages))
previous = _lineage_snapshot(_canonicalize_for_prefix_compare(previous_original_messages))
if len(current) != len(current_messages) or len(previous) != len(previous_original_messages):
# The projection dropped a directive-only message; raw indices would shift.
return None
for request_len in (len(previous), len(previous) - 1):
reply = previous[request_len:]
if reply and not (isinstance(reply[0], dict) and reply[0].get("role") == "assistant"):
continue
reminder = request_len - 1
if (
_extends_past_system_tail(current, previous[:request_len])
and current[reminder : reminder + len(reply)] == reply
):
return (
copy.deepcopy(
previous_forwarded_messages[:reminder]
+ previous_forwarded_messages[request_len:]
),
copy.deepcopy(current_messages[reminder + len(reply) :]),
)
return None
def classify_history_relation(
current_messages: list[dict[str, Any]],
previous_messages: list[dict[str, Any]],
@@ -462,10 +502,18 @@ def extract_cache_stable_delta(
This is a COMPARISON + slice only: the returned prefix is the previously-forwarded
bytes verbatim and the delta is the raw appended messages — never a rebuild from the
canonical projection — so the projection dropping non-semantic fields is safe.
A previous request that ended in a ``role:"system"`` reminder the client has since
replaced also qualifies: its forwarded bytes are replayed without that reminder
(``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this).
"""
if not previous_original_messages or previous_forwarded_messages is None:
return None
relation = classify_history_relation(current_messages, previous_original_messages)
if relation.kind == RELATION_DIVERGED:
return _replaced_system_tail_delta(
current_messages, previous_original_messages, previous_forwarded_messages
)
if relation.kind not in (RELATION_EXACT, RELATION_MESSAGE_APPEND):
# A same-message block append needs a block-level splice in
# ``overlay_cached_prefix``; slicing only whole messages would silently
+25
View File
@@ -13,6 +13,7 @@ from headroom.cache.prefix_tracker import (
PrefixCacheTracker,
PrefixFreezeConfig,
SessionTrackerStore,
extract_cache_stable_delta,
overlay_cached_prefix,
)
@@ -895,6 +896,30 @@ class TestConversationLineageResolution:
assert store.resolve_tracker("sid", "anthropic", current) not in (tracker, other)
assert store.resolve_tracker("sid", "anthropic", sibling) is other
def test_cache_delta_replays_prefix_past_replaced_system_tail(self, monkeypatch):
"""Cache mode's delta split leaves the replaced reminder out of the
replayed prefix, with or without the model reply the handlers record
after the request, and compresses everything after it as new."""
previous, current = self._reminder_turns()
reply = current[3]
forwarded = [*previous[:2], {**previous[2], "content": "compressed"}, previous[3], reply]
for recorded in ([*previous, reply], previous):
replay = forwarded[: len(recorded)]
prefix, delta = extract_cache_stable_delta(current, recorded, replay)
assert prefix == replay[:3] + replay[4:]
assert delta == current[len(prefix) :]
# A system message followed by a client turn is history, not a reminder.
changed = [*previous, {"role": "user", "content": "Continue."}]
dropped = [*previous[:3], changed[4], *current[3:]]
assert extract_cache_stable_delta(dropped, changed, changed) is None
# The client's copy of the reply must match the recorded one.
other = [*previous, {"role": "assistant", "content": "Something else."}]
assert extract_cache_stable_delta(current, other, other) is None
monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")
assert extract_cache_stable_delta(current, [*previous, reply], forwarded) is None
def test_shared_session_id_is_not_rotated(self, store):
"""Composition guard: lineage resolution must not leak into session-id
derivation — beta stickiness and the compression cache key on it."""
+98
View File
@@ -235,3 +235,101 @@ def test_cache_mode_prefix_mismatch_passes_through_with_tag(monkeypatch):
assert forwarded[0]["content"][0]["content"] == original_text
# ...but now visibly tagged instead of silent.
assert handler.recorded_tags["passthrough_reason"] == "cache_mode_prefix_mismatch"
def _assistant(turn: int) -> dict:
return {
"role": "assistant",
"content": [{"type": "tool_use", "id": f"toolu_{turn}", "name": "read", "input": {}}],
}
def _tool_result(turn: int) -> dict:
return {
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": f"toolu_{turn}",
"content": f"verbose tool output {turn} " * 200,
}
],
}
def _compress_unfrozen(messages, model, frozen_message_count=0, **kwargs):
"""Like ``_fake_pipeline_apply`` but leaves frozen messages alone, as the
real pipeline does."""
result = _fake_pipeline_apply(messages[frozen_message_count:], model)
return TransformResult(
messages=list(messages[:frozen_message_count]) + result.messages,
tokens_before=result.tokens_before,
tokens_after=result.tokens_after,
transforms_applied=result.transforms_applied,
)
class _ConversationHandler(_DummyAnthropicHandler):
"""Real session tracker store; the upstream answers with the next tool call."""
def __init__(self) -> None:
from headroom.cache.prefix_tracker import SessionTrackerStore
super().__init__()
self.session_tracker_store = SessionTrackerStore()
self.anthropic_pipeline = SimpleNamespace(apply=MagicMock(side_effect=_compress_unfrozen))
self.forwarded: list[list[dict]] = []
async def _retry_request(self, method, url, headers, body, **_kwargs):
self.forwarded.append(json.loads(json.dumps(body["messages"])))
reply = {
"id": f"msg_{len(self.forwarded)}",
"type": "message",
"role": "assistant",
"content": _assistant(len(self.forwarded) + 1)["content"],
"usage": {"input_tokens": 1, "output_tokens": 1},
}
return SimpleNamespace(
status_code=200, headers={}, content=json.dumps(reply).encode(), json=lambda: reply
)
def test_cache_mode_replaced_system_tail_keeps_compressed_prefix(monkeypatch):
"""Claude Code replaces a trailing system reminder every turn. Cache mode
must replay last turn's compressed prefix without the old reminder and
compress only the new messages, not fall back to raw forwarding."""
import headroom.tokenizers as _tk
monkeypatch.setattr(_tk, "get_tokenizer", lambda model: _DummyTokenizer())
handler = _ConversationHandler()
model = "claude-fable-5"
history = [{"role": "user", "content": "Read the files."}, _assistant(1), _tool_result(1)]
turns = [
history,
[*history, _assistant(2), _tool_result(2), {"role": "system", "content": "Reminder A"}],
[
*history,
_assistant(2),
_tool_result(2),
_assistant(3),
_tool_result(3),
{"role": "system", "content": "Reminder B"},
],
]
for messages in turns:
request = _build_request({"model": model, "max_tokens": 16, "messages": messages})
anyio.run(handler.handle_anthropic_messages, request)
from headroom.cache.prefix_tracker import _strip_cache_control
# The proxy re-places cache_control markers every turn; compare the bytes under them.
second, third = (_strip_cache_control(m) for m in handler.forwarded[1:])
assert [m["role"] for m in third] == [m["role"] for m in turns[2]]
# Everything before the replaced reminder is last turn's forwarded bytes...
assert third[:5] == second[:5]
assert second[2]["content"][0]["content"] == _COMPRESSED_TEXT
assert second[4]["content"][0]["content"] == _COMPRESSED_TEXT
# ...the new tool result is compressed, and the current reminder is sent as is.
assert third[6]["content"][0]["content"] == _COMPRESSED_TEXT
assert third[-1] == {"role": "system", "content": "Reminder B"}
assert "passthrough_reason" not in handler.recorded_tags