fix(ccr): serve headroom_retrieve on streamed turns instead of leaking it

A turn whose redeemable markers were all forwarded earlier streams rather
than buffering (#3587), but the model can still ask for one of those
originals. The client never declared headroom_retrieve, so a call that
reached it failed with "No such tool available".

Those turns now stream through the same filter the proxy uses for its
memory tools: headroom_retrieve is withheld, served from the compression
store, and the continuation round streams into the same client message.
Client tool calls in such a round are held until it ends, so a round that
retrieved alongside a client tool is continued without the unrun client
call, as the buffered path already does. Text before the first tool call
still streams live. Turns that show the model a new marker still buffer.
This commit is contained in:
sudhindradesai
2026-11-02 20:45:51 -06:00
parent 64164fc29b
commit d10a74461a
5 changed files with 586 additions and 66 deletions
+17
View File
@@ -118,6 +118,23 @@ class CCRResponseHandler:
self._retrieval_count = 0
self._retrieval_count_lock = __import__("threading").Lock()
def serve_withheld_retrievals(self, tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Run Anthropic ``headroom_retrieve`` calls withheld from a stream.
Returns one ``tool_result`` block per call that names a hash, in call
order, for the continuation round the streaming path sends upstream.
A call without a parsable hash gets no result, so the caller can tell
the round cannot be replayed.
"""
ccr_calls, _ = self._parse_ccr_tool_calls({"content": tool_calls}, "anthropic")
if not ccr_calls:
return []
with self._retrieval_count_lock:
self._retrieval_count += len(ccr_calls)
results = [self._execute_retrieval(call) for call in ccr_calls]
message = self._create_tool_result_message(results, "anthropic")
return list(message.get("content", []))
def has_ccr_tool_calls(
self,
response: dict[str, Any],
+52 -31
View File
@@ -625,43 +625,38 @@ class AnthropicHandlerMixin:
return False
@staticmethod
def _outgoing_body_has_redeemable_marker(
def _outgoing_redeemable_markers(
body: Any,
previous_forwarded_messages: list[dict[str, Any]] | None = None,
) -> bool:
"""Does the outgoing body show the model a marker it has not seen yet?
) -> tuple[bool, bool]:
"""Which owned markers does the outgoing body carry?
``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so
a request carrying none cannot benefit from the buffered path (#3071).
Returns ``(any_owned, new_owned)``. ``headroom_retrieve`` exists only to
expand a ``<<ccr:...>>`` marker, so a request carrying none cannot use
it (#3071).
Only markers new to the model count. A marker stays in the history for
the rest of the session, so counting every marker in the body buffered
every later turn once one tool result had been compressed: the buffered
share climbed toward 90% of turns with zero retrievals, and
time-to-first-byte became the whole generation (#3587). A marker that
was already forwarded on an earlier turn was in front of the model on
that turn, and that turn was buffered.
"New" is decided against what this session actually forwarded last
turn, not by message position: token mode can compress an earlier tool
result on this request, so a marker first seen now may sit anywhere in
the history. With no forwarded history to compare against, every owned
marker counts as new.
``new_owned`` is a marker the model has not seen yet: one absent from
what this session forwarded last turn. With no forwarded history to
compare against, every owned marker counts as new. "New" is decided
against the forwarded history, not by message position: token mode can
compress an earlier tool result on this request, so a marker first
seen now may sit anywhere in the history.
Ownership is verified rather than shape-matched: the marker shape is not
unique to Headroom, and adopting another context tool's hash would send
the model to an endpoint that is guaranteed to miss (#2836). A hash that
survives ``verify_ownership`` is redeemable right now.
Errors are swallowed deliberately and answered ``True``. This gates a
wire-format decision, and the safe direction on an unexpected message
shape is the long-standing buffered behavior, not a silent change.
Errors are swallowed deliberately and answered ``(True, True)``. This
gates a wire-format decision, and the safe direction on an unexpected
message shape is the long-standing buffered behavior, not a silent
change.
"""
if not isinstance(body, dict):
return True
return True, True
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
return False
return False, False
try:
from headroom.ccr.tool_injection import CCRToolInjector
from headroom.proxy.helpers import has_new_ccr_markers
@@ -673,16 +668,27 @@ class AnthropicHandlerMixin:
)
probe.scan_for_markers(messages)
if not probe.detected_hashes:
return False
return False, False
probe.verify_ownership()
return has_new_ccr_markers(
if not probe.detected_hashes:
return False, False
return True, has_new_ccr_markers(
current_detected_hashes=list(probe.detected_hashes),
previous_forwarded_messages=previous_forwarded_messages,
provider="anthropic",
)
except Exception: # pragma: no cover - defensive
logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)
return True
return True, True
@classmethod
def _outgoing_body_has_redeemable_marker(
cls,
body: Any,
previous_forwarded_messages: list[dict[str, Any]] | None = None,
) -> bool:
"""Does the outgoing body show the model an owned marker it has not seen?"""
return cls._outgoing_redeemable_markers(body, previous_forwarded_messages)[1]
@staticmethod
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
@@ -4165,20 +4171,34 @@ class AnthropicHandlerMixin:
and ccr_response_handler_enabled
and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
)
buffered_retrieval_can_help = (
retrieve_tool_is_offered
and self._outgoing_body_has_redeemable_marker(body, previous_forwarded_messages)
#
# A marker the model already saw on an earlier turn can still be
# retrieved, so it does not justify buffering every later turn
# (#3587), but it does mean a retrieval call may come. Those
# turns stream with ``headroom_retrieve`` withheld from the
# client and served by the proxy in a continuation round.
owned_marker, new_owned_marker = (
self._outgoing_redeemable_markers(body, previous_forwarded_messages)
if retrieve_tool_is_offered
else (False, False)
)
wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)
if retrieve_tool_is_offered and not buffered_retrieval_can_help:
wants_buffered_stream_ccr = bool(new_owned_marker)
if retrieve_tool_is_offered and not owned_marker:
logger.info(
f"[{request_id}] CCR: headroom_retrieve is resident but this "
"request carries no redeemable marker, so server-side "
"retrieval cannot fire; keeping the streaming path (#3071)"
)
elif retrieve_tool_is_offered and not new_owned_marker:
logger.info(
f"[{request_id}] CCR: every redeemable marker was already "
"forwarded; streaming with headroom_retrieve served by the "
"proxy (#3587)"
)
buffered_stream_ccr = (
wants_buffered_stream_ccr and not outbound_locked_to_client_bytes
)
serve_ccr_retrieval_in_stream = bool(owned_marker) and not buffered_stream_ccr
if wants_buffered_stream_ccr and outbound_locked_to_client_bytes:
logger.info(
f"[{request_id}] CCR: signed thinking blocks force byte-faithful "
@@ -4352,6 +4372,7 @@ class AnthropicHandlerMixin:
session_key=session_key,
server_memory_tool_names=server_memory_tool_names,
client_beta=client_anthropic_beta or "",
serve_ccr_retrieval=serve_ccr_retrieval_in_stream,
)
else:
# Whatever set it — the client's own ``stream: false`` or
+118 -28
View File
@@ -520,17 +520,26 @@ class StreamingMixin:
server_memory_tool_names: frozenset[str],
stream_state: dict[str, Any],
request_id: str,
serve_retrieval: bool = False,
) -> AsyncIterator[bytes]:
"""Execute withheld memory tool calls and stream continuation rounds.
"""Execute withheld proxy tool calls and stream continuation rounds.
``memory_filter`` has forwarded one upstream round to the client minus
the proxy-owned memory tool calls and the message tail. When the round
stopped on memory tool calls alone, run them, send the results upstream
the proxy-owned tool calls and the message tail. When the round
stopped on proxy tool calls alone, run them, send the results upstream
as the next turn, and stream that round's blocks into the same client
message. Otherwise end the client message. A round that also called a
client tool still has its memory calls executed, but the turn goes back
to the client, which cannot carry the memory results.
With ``serve_retrieval``, ``headroom_retrieve`` is a proxy tool too: the
client never declared it, so a call that reached it would fail (#3587).
The filter then holds client tool calls until the round ends, and a
round that retrieved alongside a client tool is continued without the
unrun client calls, for the model to re-issue with the content in
hand. That is what the buffered path does with the same round.
"""
from headroom.ccr.tool_injection import CCR_TOOL_NAME
from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE
try:
@@ -541,18 +550,47 @@ class StreamingMixin:
if not isinstance(messages, list):
messages = None
headers = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}
proxy_tool_names = server_memory_tool_names | (
frozenset({CCR_TOOL_NAME}) if serve_retrieval else frozenset()
)
def is_memory_call(block: Any) -> bool:
def is_proxy_call(block: Any) -> bool:
return (
isinstance(block, dict)
and block.get("type") == "tool_use"
and block.get("name") in server_memory_tool_names
and block.get("name") in proxy_tool_names
)
def is_client_call(block: Any) -> bool:
return (
isinstance(block, dict)
and block.get("type") == "tool_use"
and not is_proxy_call(block)
)
rounds = 0
# What the client received across rounds; the prefix tracker must see
# this, not round one, because it is what the client sends back.
visible_content: list[Any] = []
def record_visible(
round_response: dict[str, Any] | None,
round_content: list[Any],
*,
dropped_client_calls: bool,
) -> None:
if round_response is None:
return
visible_content.extend(
block
for block in round_content
if not is_proxy_call(block) and not (dropped_client_calls and is_client_call(block))
)
stream_state["memory_client_message"] = {
**round_response,
"content": list(visible_content),
}
# Usage of rounds whose message_delta the client never receives.
prior_usage: dict[str, int] = {}
while True:
@@ -560,13 +598,9 @@ class StreamingMixin:
if response is not None:
round_content = response.get("content")
content = round_content if isinstance(round_content, list) else []
visible_content.extend(block for block in content if not is_memory_call(block))
stream_state["memory_client_message"] = {
**response,
"content": list(visible_content),
}
if not memory_filter.hid_tool_calls:
record_visible(response, content, dropped_client_calls=False)
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
yield frame
return
@@ -574,7 +608,11 @@ class StreamingMixin:
# Run the calls the filter recorded, so they are not lost when the
# round could not be reconstructed. Only the proxy's own calls: a
# memory-named tool the client declared is the client's to run.
memory_calls = memory_filter.hidden_calls()
hidden_calls = memory_filter.hidden_calls()
memory_calls = [c for c in hidden_calls if c.get("name") in server_memory_tool_names]
retrieval_calls = [
c for c in hidden_calls if serve_retrieval and c.get("name") == CCR_TOOL_NAME
]
tool_results: list[dict[str, Any]] = []
if memory_calls and memory_user_id is not None and self.memory_handler is not None:
tool_results = await self.memory_handler.handle_memory_tool_calls(
@@ -583,24 +621,38 @@ class StreamingMixin:
"anthropic",
request_context=memory_request_ctx,
)
logger.info(
f"[{request_id}] Memory: Executed {len(tool_results)}/"
f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "
"withheld from the SSE stream"
)
logger.info(
f"[{request_id}] Memory: Executed {len(tool_results)}/"
f"{len(memory_calls)} proxy-handled tool call(s) "
"withheld from the SSE stream"
)
ccr_handler = getattr(self, "ccr_response_handler", None)
if retrieval_calls and ccr_handler is not None:
retrieval_results = ccr_handler.serve_withheld_retrievals(retrieval_calls)
tool_results = [*tool_results, *retrieval_results]
logger.info(
f"[{request_id}] CCR: Served {len(retrieval_results)}/"
f"{len(retrieval_calls)} headroom_retrieve call(s) withheld from "
"the SSE stream"
)
replayable = response is not None and len(tool_results) == sum(
1 for block in content if is_memory_call(block)
1 for block in content if is_proxy_call(block)
)
# Unrun client calls can be dropped only while the client has not
# seen them, and only for a retrieval: memory results alone do not
# give the model a reason to decide again.
drop_client_calls = bool(retrieval_calls) and memory_filter.holds_client_tool_use
can_continue = (
memory_filter.stop_reason == "tool_use"
and not memory_filter.visible_tool_use
and (drop_client_calls or not memory_filter.holds_client_tool_use)
and messages is not None
and replayable
and rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS
)
if not can_continue:
if memory_filter.visible_tool_use:
if memory_filter.visible_tool_use or memory_filter.holds_client_tool_use:
logger.info(
f"[{request_id}] Memory: Round also called a client tool; "
"returning the turn to the client"
@@ -608,18 +660,31 @@ class StreamingMixin:
elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:
logger.warning(
f"[{request_id}] Memory: Stopped after {rounds} continuation "
"round(s) that only called memory tools"
"round(s) that only called proxy tools"
)
elif not replayable:
logger.warning(
f"[{request_id}] Memory: Could not rebuild the round for a "
"continuation; ending the turn after running its memory calls"
"continuation; ending the turn after running its proxy calls"
)
record_visible(response, content, dropped_client_calls=False)
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
yield frame
return
assert response is not None and messages is not None and isinstance(base_body, dict)
record_visible(response, content, dropped_client_calls=drop_client_calls)
# Blocks held after the first tool call (rarely any but the calls
# themselves) still belong to the client's message.
for frame in memory_filter.release_held(drop_tool_use=drop_client_calls):
yield frame
if drop_client_calls:
logger.info(
f"[{request_id}] CCR: Round called headroom_retrieve alongside a "
"client tool; serving the retrieval and dropping the unrun client "
"call(s) for the model to re-issue"
)
content = [block for block in content if not is_client_call(block)]
round_usage = response.get("usage")
if isinstance(round_usage, dict):
for key in _ROUND_USAGE_KEYS:
@@ -647,9 +712,10 @@ class StreamingMixin:
yield _upstream_error_frame(error_bytes, request_id)
return
memory_filter = MemoryToolStreamFilter(
server_memory_tool_names,
proxy_tool_names,
index_offset=memory_filter.next_index,
forward_message_start=False,
hold_tool_use=serve_retrieval,
)
# Same cap as the first round's buffer; past it the round is
# still streamed, just not rebuilt for a further continuation.
@@ -1079,6 +1145,7 @@ class StreamingMixin:
savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
serve_ccr_retrieval: bool = False,
) -> Response | StreamingResponse:
"""Stream response with metrics tracking and memory tool handling.
@@ -1127,6 +1194,7 @@ class StreamingMixin:
savings_measure=savings_measure,
server_memory_tool_names=server_memory_tool_names,
client_beta=client_beta,
serve_ccr_retrieval=serve_ccr_retrieval,
)
except (Exception, asyncio.CancelledError):
self._cleanup_mid_turn_stream(session_key)
@@ -1162,6 +1230,7 @@ class StreamingMixin:
savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
serve_ccr_retrieval: bool = False,
) -> Response | StreamingResponse:
"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""
from fastapi.responses import Response, StreamingResponse
@@ -1336,6 +1405,9 @@ class StreamingMixin:
and self.memory_handler is not None
and provider == "anthropic"
)
# Withholding headroom_retrieve needs the Anthropic SSE shape the
# filter and the continuation round are written for.
serve_ccr_retrieval = serve_ccr_retrieval and provider == "anthropic"
# Open connection before generator to capture upstream response headers
# (needed to forward ratelimit headers to the client via StreamingResponse)
@@ -1663,11 +1735,23 @@ class StreamingMixin:
pending_messages: list[dict] = []
# Proxy-injected memory tools are invisible to the client, so their
# tool_use blocks must never reach it (GH #2195).
from headroom.ccr.tool_injection import CCR_TOOL_NAME
# Memory tools are withheld only when this request can run them;
# the continuation must classify calls exactly as the filter did.
filter_memory_tool_names = (
server_memory_tool_names or frozenset() if memory_enabled else frozenset()
)
proxy_tool_names = filter_memory_tool_names | (
frozenset({CCR_TOOL_NAME}) if serve_ccr_retrieval else frozenset()
)
memory_filter = (
MemoryToolStreamFilter(server_memory_tool_names)
if memory_enabled and server_memory_tool_names
MemoryToolStreamFilter(proxy_tool_names, hold_tool_use=serve_ccr_retrieval)
if proxy_tool_names
else None
)
# A withheld retrieval needs the round rebuilt for its continuation.
retrieval_tracking = serve_ccr_retrieval
def _client_frame(frame: bytes) -> bytes:
"""``_guard_client_bytes``, noting when the terminal event goes out."""
@@ -1734,6 +1818,7 @@ class StreamingMixin:
_track_sse = (
_codex_wire_debug
or memory_enabled
or retrieval_tracking
or (prefix_tracker is not None and provider == "anthropic")
)
if _track_sse:
@@ -1746,6 +1831,7 @@ class StreamingMixin:
"disabling memory detection for this request"
)
memory_enabled = False
retrieval_tracking = False
# Parse complete SSE events from buffer. A failure (e.g.
# invalid UTF-8) is raised once the chunk is forwarded:
@@ -1806,7 +1892,7 @@ class StreamingMixin:
# be an upstream protocol violation — surface loudly.
full_sse_data: str = full_sse_bytes.decode("utf-8") if full_sse_bytes else ""
if memory_enabled and full_sse_data:
if (memory_enabled or retrieval_tracking) and full_sse_data:
# Check for Claude Code credential error
if "only authorized for use with Claude Code" in full_sse_data:
logger.warning(
@@ -1836,9 +1922,10 @@ class StreamingMixin:
outbound_bytes=outbound_bytes,
memory_user_id=memory_user_id,
memory_request_ctx=memory_request_ctx,
server_memory_tool_names=server_memory_tool_names or frozenset(),
server_memory_tool_names=filter_memory_tool_names,
stream_state=stream_state,
request_id=request_id,
serve_retrieval=serve_ccr_retrieval,
):
guarded_frame = _client_frame(frame)
if guarded_frame:
@@ -1847,8 +1934,10 @@ class StreamingMixin:
parsed_response = (
stream_state.pop("memory_client_message", None) or parsed_response
)
elif parsed_response and self.memory_handler.has_memory_tool_calls(
parsed_response, provider
elif (
memory_enabled
and parsed_response
and self.memory_handler.has_memory_tool_calls(parsed_response, provider)
):
logger.info(
f"[{request_id}] Memory: Detected tool calls in streaming response"
@@ -1881,9 +1970,10 @@ class StreamingMixin:
outbound_bytes=outbound_bytes,
memory_user_id=memory_user_id,
memory_request_ctx=memory_request_ctx,
server_memory_tool_names=server_memory_tool_names or frozenset(),
server_memory_tool_names=filter_memory_tool_names,
stream_state=stream_state,
request_id=request_id,
serve_retrieval=serve_ccr_retrieval,
):
guarded_frame = _client_frame(frame)
if guarded_frame:
+79 -7
View File
@@ -90,6 +90,12 @@ class MemoryToolStreamFilter:
extend the message the client already opened.
max_retained_bytes: Limit on the bytes held back at any time;
``DEFAULT_MAX_RETAINED_BYTES`` when None.
hold_tool_use: Hold every block from the round's first ``tool_use``
on, until the caller decides with ``release_held``. A withheld
``headroom_retrieve`` alongside a client tool is served by
dropping the client call for the model to re-issue, which is
only possible while that call has not reached the client.
Text before the first tool call still streams live.
"""
def __init__(
@@ -99,8 +105,17 @@ class MemoryToolStreamFilter:
index_offset: int = 0,
forward_message_start: bool = True,
max_retained_bytes: int | None = None,
hold_tool_use: bool = False,
) -> None:
self._tool_names = frozenset(tool_names)
self._hold_tool_use = hold_tool_use
self._holding = False
# Frames held from the first tool_use on, with their upstream index;
# client indices are assigned when they are released.
self._held: list[tuple[str, dict[str, Any], bytes, int]] = []
self._held_bytes = 0
# Upstream indices of held client tool_use blocks.
self._held_tool_use: set[int] = set()
self._max_retained_bytes = (
DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes
)
@@ -130,6 +145,39 @@ class MemoryToolStreamFilter:
def hidden_tool_names(self) -> list[str]:
return [block["name"] for block, _ in self._hidden.values()]
@property
def holds_client_tool_use(self) -> bool:
"""A client ``tool_use`` block is held and has not reached the client."""
return bool(self._held_tool_use)
def release_held(self, *, drop_tool_use: bool = False) -> list[bytes]:
"""Return the held frames, re-indexed, for the client.
``drop_tool_use`` leaves out the held client ``tool_use`` blocks: the
round goes back to the model with the proxy's results instead, and
the model re-issues whatever it still needs.
"""
frames: list[bytes] = []
dropped = self._held_tool_use if drop_tool_use else set()
for event_name, payload, frame, upstream_index in self._held:
if upstream_index in dropped:
continue
if event_name == "content_block_start":
block = payload.get("content_block")
if isinstance(block, dict) and block.get("type") == "tool_use":
self.visible_tool_use = True
self._index_map[upstream_index] = self.next_index
self.next_index += 1
client_index = self._index_map.get(upstream_index, upstream_index)
if client_index == upstream_index:
frames.append(frame)
else:
frames.append(_render_frame(event_name, {**payload, "index": client_index}))
self._held.clear()
self._held_bytes = 0
self._held_tool_use.clear()
return frames
def hidden_calls(self) -> list[dict[str, Any]]:
"""The withheld ``tool_use`` blocks that finished with a complete input.
@@ -181,7 +229,12 @@ class MemoryToolStreamFilter:
forwarded = self._route(frame)
if forwarded is not None:
out.append(forwarded)
retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())
retained = (
len(self._buffer)
+ self._tail_bytes
+ self._held_bytes
+ sum(self._hidden_bytes.values())
)
if retained > self._max_retained_bytes:
self._buffer.clear()
self._hidden.clear()
@@ -189,6 +242,9 @@ class MemoryToolStreamFilter:
self._hidden_stopped.clear()
self._tail.clear()
self._tail_bytes = 0
self._held.clear()
self._held_bytes = 0
self._held_tool_use.clear()
raise MemoryToolStreamOverflowError(
f"memory tool stream filter would retain {retained} bytes "
f"(limit {self._max_retained_bytes})"
@@ -232,11 +288,18 @@ class MemoryToolStreamFilter:
self._hidden[upstream_index] = (block, [])
self._hidden_bytes[upstream_index] = len(frame)
self._hidden_stopped.discard(upstream_index)
self._holding = self._hold_tool_use
return None
if isinstance(block, dict) and block.get("type") == "tool_use":
self.visible_tool_use = True
self._index_map[upstream_index] = self.next_index
self.next_index += 1
is_tool_use = isinstance(block, dict) and block.get("type") == "tool_use"
if self._hold_tool_use and (self._holding or is_tool_use):
self._holding = True
if is_tool_use:
self._held_tool_use.add(upstream_index)
else:
if is_tool_use:
self.visible_tool_use = True
self._index_map[upstream_index] = self.next_index
self.next_index += 1
if upstream_index in self._hidden:
delta = payload.get("delta")
@@ -252,17 +315,26 @@ class MemoryToolStreamFilter:
elif event_name == "content_block_stop":
self._hidden_stopped.add(upstream_index)
return None
if self._holding and upstream_index not in self._index_map:
self._held.append((event_name, payload, frame, upstream_index))
self._held_bytes += len(frame)
return None
client_index = self._index_map.get(upstream_index, upstream_index)
if client_index == upstream_index:
return frame
return _render_frame(event_name, {**payload, "index": client_index})
def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:
"""End the client message after this round without a continuation."""
"""End the client message after this round without a continuation.
Held frames are released first, client tool calls included: with no
continuation, the round is the model's answer as it stands.
"""
held = self.release_held()
ends_on_hidden_call = (
self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use"
)
return self.tail_frames(
return held + self.tail_frames(
stop_reason="end_turn" if ends_on_hidden_call else None,
prior_usage=prior_usage,
)
+320
View File
@@ -0,0 +1,320 @@
"""``headroom_retrieve`` on a streamed turn is served by the proxy (#3587).
A turn whose redeemable markers were all forwarded on earlier turns streams
instead of buffering, so the client gets the first token right away. The model
can still ask for one of those originals. The client never declared
``headroom_retrieve`` (a coding agent answers "No such tool available"), so
the call must not reach it: the stream withholds it, the proxy serves it from
the compression store, and the continuation round streams into the same client
message.
"""
from __future__ import annotations
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
fastapi = pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
from headroom.proxy.memory_tool_stream import MemoryToolStreamFilter # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
RETRIEVE_ONLY = frozenset({"headroom_retrieve"})
ORIGINAL = "line 1 of the original build log\nline 2: the error the model needs"
def _frame(payload: dict[str, Any]) -> bytes:
return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
def _sse(blocks: list[dict[str, Any]], stop_reason: str) -> bytes:
"""An Anthropic SSE response carrying ``blocks``."""
frames = [
_frame(
{
"type": "message_start",
"message": {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "sonnet-test",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 10, "output_tokens": 1},
},
}
)
]
for index, block in enumerate(blocks):
if block["type"] == "text":
start = {"type": "text", "text": ""}
delta = {"type": "text_delta", "text": block["text"]}
else:
start = {**block, "input": {}}
delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
frames.append(
_frame({"type": "content_block_start", "index": index, "content_block": start})
)
frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))
frames.append(_frame({"type": "content_block_stop", "index": index}))
frames.append(
_frame(
{
"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
"usage": {"output_tokens": 5},
}
)
)
frames.append(_frame({"type": "message_stop"}))
return b"".join(frames)
def _events(raw: bytes) -> list[dict[str, Any]]:
return [
json.loads(line[len("data: ") :])
for line in raw.decode().splitlines()
if line.startswith("data: ")
]
def _tool_names(events: list[dict[str, Any]]) -> list[str]:
return [
e["content_block"]["name"]
for e in events
if e["type"] == "content_block_start" and e["content_block"].get("type") == "tool_use"
]
TEXT = {"type": "text", "text": "Let me look at the earlier output."}
BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
def _retrieve(hash_key: str) -> dict[str, Any]:
return {
"type": "tool_use",
"id": "toolu_ccr",
"name": "headroom_retrieve",
"input": {"hash": hash_key},
}
class TestHoldingFilter:
def test_text_before_the_first_tool_call_streams_live(self) -> None:
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
forwarded = _events(b"".join(flt.feed(_sse([TEXT, BASH], "tool_use"))))
assert [e["type"] for e in forwarded] == [
"message_start",
"content_block_start",
"content_block_delta",
"content_block_stop",
]
assert flt.holds_client_tool_use
def test_without_a_retrieval_the_round_is_byte_identical(self) -> None:
raw = _sse([TEXT, BASH], "tool_use")
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())
assert out == raw
def test_dropping_held_client_calls(self) -> None:
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
flt.feed(_sse([BASH, _retrieve("abc")], "tool_use"))
assert flt.hidden_tool_names == ["headroom_retrieve"]
assert flt.release_held(drop_tool_use=True) == []
assert flt.next_index == 0
assert not flt.visible_tool_use
def _sse_response(raw: bytes) -> MagicMock:
response = MagicMock()
response.headers = httpx.Headers({"content-type": "text/event-stream"})
response.status_code = 200
async def aiter_bytes(raw: bytes = raw):
# Split mid-frame so the filter has to reassemble events.
yield raw[:41]
yield raw[41:]
response.aiter_bytes = aiter_bytes
response.aclose = AsyncMock()
return response
@pytest.fixture
def owned_marker():
"""A marker the proxy owns, with its original in the compression store."""
from headroom.cache.backends import InMemoryBackend
from headroom.cache.compression_store import get_compression_store, reset_compression_store
reset_compression_store()
store = get_compression_store(backend=InMemoryBackend())
hash_key = store.store(
ORIGINAL,
"<<ccr:placeholder>>",
original_tokens=100,
compressed_tokens=5,
tool_name="Bash",
)
try:
yield hash_key
finally:
reset_compression_store()
def _two_turn_session(
marker: str, second_turn_rounds: list[bytes]
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
"""Turn 1 introduces ``marker``; turn 2 only carries it in history.
Returns the client's SSE events for turn 2, the bodies the proxy sent
upstream on turn 2's stream (first round, then continuations), and how
many buffered upstream requests turn 2 made.
"""
first_turn = [
{"role": "user", "content": "build it"},
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "tu_1",
"content": f"build log compressed at <<ccr:{marker}>>",
}
],
},
]
second_turn = [
*first_turn,
{"role": "assistant", "content": [{"type": "text", "text": "The build failed."}]},
{"role": "user", "content": "what was the exact error?"},
]
buffered_calls: list[int] = []
async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
buffered_calls.append(1)
return httpx.Response(
200,
json={
"id": "msg_t1",
"type": "message",
"role": "assistant",
"model": "sonnet-test",
"content": [{"type": "text", "text": "The build failed."}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 5},
},
headers={"content-type": "application/json"},
)
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
memory_enabled=False,
)
headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
tools = [
create_ccr_tool_definition("anthropic"),
{"name": "Bash", "description": "run", "input_schema": {"type": "object"}},
]
def body(messages: list[dict[str, Any]]) -> dict[str, Any]:
return {
"model": "sonnet-test",
"max_tokens": 64,
"stream": True,
"tools": tools,
"messages": messages,
}
app = create_app(config)
with TestClient(app) as client:
proxy = client.app.state.proxy
proxy._retry_request = fake_retry
first = client.post("/v1/messages", json=body(first_turn), headers=headers)
assert first.status_code == 200, first.text
assert len(buffered_calls) == 1, "the turn that introduces the marker buffers"
upstream = MagicMock(spec=httpx.AsyncClient)
upstream.build_request = MagicMock(return_value=MagicMock())
upstream.send = AsyncMock(side_effect=[_sse_response(r) for r in second_turn_rounds])
proxy.http_client = upstream
second = client.post("/v1/messages", json=body(second_turn), headers=headers)
assert second.status_code == 200, second.text
sent = [json.loads(c.kwargs["content"]) for c in upstream.build_request.call_args_list]
return _events(second.content), sent, len(buffered_calls) - 1
def test_later_turn_retrieval_is_served_by_the_proxy(owned_marker: str) -> None:
events, sent, buffered = _two_turn_session(
owned_marker,
[
_sse([TEXT, _retrieve(owned_marker)], "tool_use"),
_sse([{"type": "text", "text": "The error is on line 2."}], "end_turn"),
],
)
assert buffered == 0, "a turn with only already-forwarded markers keeps streaming"
assert "headroom_retrieve" not in _tool_names(events)
texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
assert texts == ["Let me look at the earlier output.", "The error is on line 2."]
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
assert stops == ["end_turn"]
assert [e["type"] for e in events].count("message_start") == 1
assert len(sent) == 2
continuation = sent[1]["messages"]
assert continuation[-2]["content"][-1]["name"] == "headroom_retrieve"
served = continuation[-1]["content"][0]
assert served["type"] == "tool_result" and served["tool_use_id"] == "toolu_ccr"
assert json.loads(served["content"])["original_content"] == ORIGINAL
@pytest.mark.parametrize("client_call_first", [True, False])
def test_retrieval_mixed_with_a_client_tool(owned_marker: str, client_call_first: bool) -> None:
calls = [BASH, _retrieve(owned_marker)]
if not client_call_first:
calls.reverse()
reissued = {**BASH, "id": "toolu_bash_2"}
events, sent, buffered = _two_turn_session(
owned_marker,
[
_sse(calls, "tool_use"),
_sse([{"type": "text", "text": "Now rerun it."}, reissued], "tool_use"),
],
)
assert buffered == 0
# Only the model's decision made with the content in hand reaches the client.
assert _tool_names(events) == ["Bash"]
started = [e["content_block"] for e in events if e["type"] == "content_block_start"]
assert [b.get("id") for b in started if b.get("type") == "tool_use"] == ["toolu_bash_2"]
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
assert stops == ["tool_use"]
assert [e["index"] for e in events if e["type"] == "content_block_start"] == [0, 1]
continuation = sent[1]["messages"]
assert [b["name"] for b in continuation[-2]["content"]] == ["headroom_retrieve"]
assert json.loads(continuation[-1]["content"][0]["content"])["original_content"] == ORIGINAL
def test_old_marker_turn_without_retrieval_streams_unchanged(owned_marker: str) -> None:
raw = _sse([{"type": "text", "text": "It failed on line 2."}, BASH], "tool_use")
events, sent, buffered = _two_turn_session(owned_marker, [raw])
assert buffered == 0
assert len(sent) == 1, "no continuation round without a retrieval"
assert events == _events(raw)