fix(ccr): decide marker freshness from what was forwarded, not position

Scanning only the messages after the last assistant turn missed a marker
that token mode creates in an earlier tool result on this request. The
model sees that marker for the first time, so a headroom_retrieve call for
it has to be handled server-side, but the turn streamed and the call would
reach the client.

The gate now scans the whole outgoing body and buffers when an owned
marker is absent from the messages this session forwarded last turn,
reusing has_new_ccr_markers. Markers already forwarded keep streaming,
as before.
This commit is contained in:
sudhindradesai
2026-11-01 22:43:30 -06:00
parent 183440cea3
commit 64164fc29b
2 changed files with 103 additions and 75 deletions
+27 -24
View File
@@ -625,20 +625,28 @@ class AnthropicHandlerMixin:
return False
@staticmethod
def _outgoing_body_has_redeemable_marker(body: Any) -> bool:
"""Does this turn's new content carry a marker retrieval could expand?
def _outgoing_body_has_redeemable_marker(
body: Any,
previous_forwarded_messages: list[dict[str, Any]] | None = None,
) -> bool:
"""Does the outgoing body show the model a marker it has not seen yet?
``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so
a request carrying none cannot benefit from the buffered path (#3071).
Only messages after the last assistant turn are scanned. A marker stays
in the history for the rest of the session, so scanning the whole body
buffered every later turn once one tool result had been compressed:
the buffered share climbed toward 90% of turns with zero retrievals,
and time-to-first-byte became the whole generation (#3587). The model
already had the marker in front of it on the turn that introduced it,
and that turn was buffered. A later streamed turn that does call
``headroom_retrieve`` behaves like any turn with no marker.
Only markers new to the model count. A marker stays in the history for
the rest of the session, so counting every marker in the body buffered
every later turn once one tool result had been compressed: the buffered
share climbed toward 90% of turns with zero retrievals, and
time-to-first-byte became the whole generation (#3587). A marker that
was already forwarded on an earlier turn was in front of the model on
that turn, and that turn was buffered.
"New" is decided against what this session actually forwarded last
turn, not by message position: token mode can compress an earlier tool
result on this request, so a marker first seen now may sit anywhere in
the history. With no forwarded history to compare against, every owned
marker counts as new.
Ownership is verified rather than shape-matched: the marker shape is not
unique to Headroom, and adopting another context tool's hash would send
@@ -654,30 +662,24 @@ class AnthropicHandlerMixin:
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
return False
last_assistant = max(
(
i
for i, message in enumerate(messages)
if isinstance(message, dict) and message.get("role") == "assistant"
),
default=-1,
)
new_messages = messages[last_assistant + 1 :]
if not new_messages:
return False
try:
from headroom.ccr.tool_injection import CCRToolInjector
from headroom.proxy.helpers import has_new_ccr_markers
probe = CCRToolInjector(
provider="anthropic",
inject_tool=False,
inject_system_instructions=False,
)
probe.scan_for_markers(new_messages)
probe.scan_for_markers(messages)
if not probe.detected_hashes:
return False
probe.verify_ownership()
return bool(probe.detected_hashes)
return has_new_ccr_markers(
current_detected_hashes=list(probe.detected_hashes),
previous_forwarded_messages=previous_forwarded_messages,
provider="anthropic",
)
except Exception: # pragma: no cover - defensive
logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)
return True
@@ -4164,7 +4166,8 @@ class AnthropicHandlerMixin:
and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
)
buffered_retrieval_can_help = (
retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)
retrieve_tool_is_offered
and self._outgoing_body_has_redeemable_marker(body, previous_forwarded_messages)
)
wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)
if retrieve_tool_is_offered and not buffered_retrieval_can_help:
@@ -289,59 +289,52 @@ def test_buffering_is_gated_on_a_redeemable_marker(
assert calls["stream_body"]["stream"] is True
def _agent_turn_body(*, marker: str, marker_in_latest_turn: bool) -> dict:
"""A two-turn agent loop whose only owned marker is old or new this turn."""
old_output = f"earlier output at <<ccr:{marker}>>" if not marker_in_latest_turn else "short"
new_output = f"new output at <<ccr:{marker}>>" if marker_in_latest_turn else "short"
def _owned_marker(content: str) -> str:
"""Store one more original in the (fixture-reset) store; return its hash."""
from headroom.cache.compression_store import get_compression_store
return get_compression_store().store(
content,
"<<ccr:placeholder>>",
original_tokens=100,
compressed_tokens=5,
tool_name="Read",
)
def _tool_turn(tool_use_id: str, output: str) -> list[dict]:
return [
{
"role": "assistant",
"content": [{"type": "tool_use", "id": tool_use_id, "name": "Read", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": output}],
},
]
def _agent_body(messages: list[dict]) -> dict:
return {
"model": "claude-sonnet-4-20250514",
"max_tokens": 64,
"stream": True,
"tools": [RETRIEVE_TOOL],
"messages": [
{"role": "user", "content": "read the logs"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "tu_1", "name": "Read", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": old_output}],
},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "tu_2", "name": "Read", "input": {}}],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "tu_2", "content": new_output}],
},
],
"messages": messages,
}
@pytest.mark.parametrize(
("marker_in_latest_turn", "expect_buffered"),
[
(True, True),
# The marker was already in front of the model on the turn that
# introduced it. Buffering every later turn for it is #3587: the
# buffered share climbed toward 90% of turns with zero retrievals.
(False, False),
],
)
def test_buffering_is_gated_on_a_marker_new_this_turn(
marker_in_latest_turn: bool, expect_buffered: bool, ccr_marker: str
) -> None:
"""A marker carried forward in history does not make every later turn buffer."""
calls: dict[str, object] = {}
def _replay_session(turns: list[list[dict]]) -> list[str]:
"""Send each message list as one turn of the same session; report the path taken."""
paths: list[str] = []
async def fake_stream_response(url, headers, body, *args, **kwargs): # noqa: ANN001
calls["stream_body"] = body
paths.append("streamed")
return fastapi.responses.StreamingResponse(iter([SSE_BODY]), media_type="text/event-stream")
async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
calls["buffered_body"] = json.loads(json.dumps(req_body))
paths.append("buffered")
return httpx.Response(
200,
json={
@@ -360,18 +353,50 @@ def test_buffering_is_gated_on_a_marker_new_this_turn(
with TestClient(app) as client:
client.app.state.proxy._stream_response = fake_stream_response
client.app.state.proxy._retry_request = fake_retry
resp = client.post(
"/v1/messages",
json=_agent_turn_body(marker=ccr_marker, marker_in_latest_turn=marker_in_latest_turn),
headers=_headers(),
)
for messages in turns:
resp = client.post("/v1/messages", json=_agent_body(messages), headers=_headers())
assert resp.status_code == 200, resp.text
return paths
assert resp.status_code == 200, resp.text
if expect_buffered:
assert "buffered_body" in calls, "a marker new this turn must still buffer"
else:
assert "stream_body" in calls, "an old marker alone must keep the client streaming"
assert "buffered_body" not in calls
def test_a_marker_already_forwarded_does_not_buffer_later_turns(ccr_marker: str) -> None:
"""A marker carried forward in history does not make every later turn buffer.
The marker was in front of the model on the turn that introduced it, and
that turn buffered. Buffering every later turn for it is #3587: the
buffered share climbed toward 90% of turns with zero retrievals.
"""
first = [
{"role": "user", "content": "read the logs"},
*_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),
]
second = [*first, *_tool_turn("tu_2", "short")]
assert _replay_session([first, second]) == ["buffered", "streamed"]
def test_a_marker_first_shown_in_an_earlier_message_still_buffers(ccr_marker: str) -> None:
"""New to the model is what counts, not where the marker sits.
Token mode can compress an earlier tool result on a later request, so the
model first sees that marker in the middle of the history. Retrieval has
to be handled server-side there too, or the call reaches the client.
"""
newly_compressed = _owned_marker("a different, uncompressed tool output")
first = [
{"role": "user", "content": "read the logs"},
*_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),
]
second = [
{"role": "user", "content": "read the logs"},
*_tool_turn(
"tu_1",
f"earlier output at <<ccr:{ccr_marker}>> and <<ccr:{newly_compressed}>>",
),
*_tool_turn("tu_2", "short"),
]
assert _replay_session([first, second]) == ["buffered", "buffered"]
def test_cache_hit_never_replays_a_foreign_content_type() -> None: