fix(cache): keep prefix lineage across a replaced system tail

Claude Code can end a /v1/messages request with a temporary role:"system"
reminder and replace it on the next turn. The recorded lineage snapshot
ends in the old reminder, so it is never a prefix of the new history and
resolve_tracker starts a fresh, empty tracker. The confirmed frozen prefix
and the previously forwarded messages are lost, the cached-prefix replay
has nothing to replay, and any recompressed history re-writes the
provider prompt cache on every reminder turn.

When no lineage matches as recorded, resolve_tracker now also accepts an
Anthropic chain that ends in a system message preceded by user/assistant
history, provided the new history keeps everything before that message
unchanged and extends past it. Only a unique longest candidate with the
same cache affinity is taken; equal-length siblings start a fresh lineage
as before, and any exact, append, or block-rewrite relation still wins.
The full current snapshot is stored as usual. Only tracker selection
changes; forwarded bytes and the replay logic are untouched.

HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0 turns the fallback off.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-02 15:40:01 +02:00
co-authored by Claude Opus 5.5
parent d5318ac235
commit 2c5af818c9
2 changed files with 170 additions and 1 deletions
+59 -1
View File
@@ -367,6 +367,39 @@ def _classify_history_canonical(
)
_TRANSIENT_SYSTEM_LINEAGE_ENV = "HEADROOM_TRANSIENT_SYSTEM_LINEAGE"
def _transient_system_lineage_enabled() -> bool:
return os.environ.get(_TRANSIENT_SYSTEM_LINEAGE_ENV, "").strip().lower() not in (
"0",
"false",
"no",
"off",
)
def _extends_past_system_tail(current_messages: list[Any], previous_messages: list[Any]) -> bool:
"""Whether ``current`` continues ``previous`` once its trailing system turn is set aside.
Claude Code ends some requests with a ``role:"system"`` reminder and
replaces it on the next turn, so the recorded history is never a prefix of
the new one. Only that last message is excluded: everything before it
must survive unchanged and include real user/assistant history, so a
changed leading or historical system instruction still diverges.
"""
if len(previous_messages) < 2 or len(current_messages) < len(previous_messages):
return False
tail = previous_messages[-1]
if not isinstance(tail, dict) or tail.get("role") != "system":
return False
stable = previous_messages[:-1]
return current_messages[: len(stable)] == stable and any(
isinstance(message, dict) and message.get("role") in ("user", "assistant")
for message in stable
)
def classify_history_relation(
current_messages: list[dict[str, Any]],
previous_messages: list[dict[str, Any]],
@@ -1433,7 +1466,10 @@ class SessionTrackerStore:
where a large leading run and two-block identity suffix survive while
the middle tail is regenerated; all other rewrites start a fresh
lineage. This keeps #2671's stable cache boundary attached without
merging unrelated parallel sub-calls.
merging unrelated parallel sub-calls. When nothing else matches, an
Anthropic chain whose trailing ``role:"system"`` reminder the client
replaced is matched on the history before that reminder
(``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this).
Byte-identical histories (templated fan-outs before they diverge)
intentionally share a tracker: their provider cache line is identical
too, so sharing is harmless.
@@ -1527,6 +1563,28 @@ class SessionTrackerStore:
len(rewrite_candidates) == 1 or rewrite_candidates[0][0] != rewrite_candidates[1][0]
):
best_key = rewrite_candidates[0][1]
elif (
not rewrite_candidates
and provider == "anthropic"
and _transient_system_lineage_enabled()
):
# Nothing matched as recorded. A chain whose only change is
# its replaced trailing system reminder is still this
# conversation; without it every such turn starts a cold
# tracker, the confirmed prefix is not replayed, and any
# recompressed history re-writes the provider cache. Only a
# unique longest chain qualifies, so equal-length siblings
# that differ in their reminder stay separate.
tail_candidates = [
(len(chain), key)
for key, chain in by_length
if self._lineage_affinities.get(key) == cache_affinity
and _extends_past_system_tail(snap, chain)
]
if tail_candidates and (
len(tail_candidates) == 1 or tail_candidates[0][0] != tail_candidates[1][0]
):
best_key = tail_candidates[0][1]
if best_key is None:
cap = self._default_config.max_lineages_per_session
+111
View File
@@ -13,6 +13,7 @@ from headroom.cache.prefix_tracker import (
PrefixCacheTracker,
PrefixFreezeConfig,
SessionTrackerStore,
overlay_cached_prefix,
)
@@ -775,6 +776,116 @@ class TestConversationLineageResolution:
fresh = store.resolve_tracker("sid", "anthropic", messages=self._history("A", 2))
assert fresh._turn_number == 0
@staticmethod
def _reminder_turns() -> tuple[list[dict], list[dict]]:
"""Claude Code shape: a trailing system reminder replaced by the next turn."""
history = [
{"role": "user", "content": "Inspect example.py."},
{"role": "assistant", "content": "Reading the file."},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "read-1",
"content": "original source line\n" * 400,
}
],
},
]
reminder = {"role": "system", "content": "Batch independent tools this turn."}
previous = [*history, reminder]
current = [
*history,
{"role": "assistant", "content": "Now run the checks."},
{"role": "user", "content": "The checks passed."},
dict(reminder),
]
return previous, current
def test_replaced_system_tail_keeps_confirmed_tracker(self, store):
"""The previous request ended in a system reminder the client swapped
for new turns. Its unchanged history must keep the tracker and its
confirmed frozen prefix instead of starting a cold lineage."""
previous, current = self._reminder_turns()
forwarded = [dict(m) for m in previous]
forwarded[2] = {**previous[2], "content": "previously cached compressed result"}
tracker = store.resolve_tracker("sid", "anthropic", previous, cache_affinity="a")
tracker.update_from_response(
cache_read_tokens=0,
cache_write_tokens=1_000_000,
messages=forwarded,
original_messages=previous,
message_token_counts=[100] * len(previous),
)
frozen = tracker.get_frozen_message_count()
assert frozen > 0
assert store.resolve_tracker("sid", "anthropic", current, cache_affinity="a") is tracker
assert tracker.get_frozen_message_count() == frozen
# The existing replay restores the unchanged prefix and stops at the
# replaced reminder; this turn's tail is forwarded as processed.
processed = [dict(m) for m in current]
processed[2] = {**current[2], "content": "a different compressed result"}
replayed = overlay_cached_prefix(
processed, current, previous, forwarded, confirmed_frozen_count=frozen
)
assert replayed[:3] == forwarded[:3]
assert replayed[3:] == processed[3:]
def test_moving_system_tail_records_full_snapshots(self, store):
"""The reminder moves and changes every turn; each turn stays on one
tracker and the store records the full current history."""
previous, current = self._reminder_turns()
tracker = store.resolve_tracker("sid", "anthropic", previous)
for i in range(4):
current[-1] = {"role": "system", "content": f"reminder {i}"}
assert store.resolve_tracker("sid", "anthropic", current) is tracker
assert store._lineages["sid"]["sid"][-1]["content"][0]["text"] == f"reminder {i}"
assert len(store._lineages["sid"]["sid"]) == len(current)
current = [
*current[:-1],
{"role": "assistant", "content": f"reply {i}"},
{"role": "user", "content": f"next {i}"},
current[-1],
]
def test_system_tail_fallback_respects_affinity_and_provider(self):
previous, current = self._reminder_turns()
for provider, affinity in (("anthropic", "other-tools"), ("openai", "a")):
store = SessionTrackerStore()
tracker = store.resolve_tracker("sid", provider, previous, cache_affinity="a")
resolved = store.resolve_tracker("sid", provider, current, cache_affinity=affinity)
assert resolved is not tracker
@pytest.mark.parametrize(
("index", "role"),
[(0, "user"), (0, "system"), (2, "system")],
ids=["changed-history", "changed-leading-system", "changed-historical-system"],
)
def test_system_tail_fallback_requires_unchanged_history(self, store, index, role):
"""Only the trailing reminder is set aside: any other change, including
a changed leading or historical system instruction, still diverges."""
previous, current = self._reminder_turns()
previous.insert(index, {"role": role, "content": "original"})
current.insert(index, {"role": role, "content": "changed"})
tracker = store.resolve_tracker("sid", "anthropic", previous)
assert store.resolve_tracker("sid", "anthropic", current) is not tracker
def test_ambiguous_system_tail_siblings_are_not_merged(self, store, monkeypatch):
"""Two lineages that differ only in their reminder (recorded here with
the fallback disabled) are an ambiguous match: start a fresh lineage
instead of guessing, and an exact match still wins."""
previous, current = self._reminder_turns()
sibling = [*previous[:-1], {"role": "system", "content": "other reminder"}]
monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")
tracker = store.resolve_tracker("sid", "anthropic", previous)
other = store.resolve_tracker("sid", "anthropic", sibling)
assert other is not tracker
monkeypatch.delenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE")
assert store.resolve_tracker("sid", "anthropic", current) not in (tracker, other)
assert store.resolve_tracker("sid", "anthropic", sibling) is other
def test_shared_session_id_is_not_rotated(self, store):
"""Composition guard: lineage resolution must not leak into session-id
derivation — beta stickiness and the compression cache key on it."""