mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(ccr): serve headroom_retrieve on streamed turns instead of leaking it
A turn whose redeemable markers were all forwarded earlier streams rather than buffering (#3587), but the model can still ask for one of those originals. The client never declared headroom_retrieve, so a call that reached it failed with "No such tool available". Those turns now stream through the same filter the proxy uses for its memory tools: headroom_retrieve is withheld, served from the compression store, and the continuation round streams into the same client message. Client tool calls in such a round are held until it ends, so a round that retrieved alongside a client tool is continued without the unrun client call, as the buffered path already does. Text before the first tool call still streams live. Turns that show the model a new marker still buffer.
This commit is contained in:
@@ -118,6 +118,23 @@ class CCRResponseHandler:
|
||||
self._retrieval_count = 0
|
||||
self._retrieval_count_lock = __import__("threading").Lock()
|
||||
|
||||
def serve_withheld_retrievals(self, tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Run Anthropic ``headroom_retrieve`` calls withheld from a stream.
|
||||
|
||||
Returns one ``tool_result`` block per call that names a hash, in call
|
||||
order, for the continuation round the streaming path sends upstream.
|
||||
A call without a parsable hash gets no result, so the caller can tell
|
||||
the round cannot be replayed.
|
||||
"""
|
||||
ccr_calls, _ = self._parse_ccr_tool_calls({"content": tool_calls}, "anthropic")
|
||||
if not ccr_calls:
|
||||
return []
|
||||
with self._retrieval_count_lock:
|
||||
self._retrieval_count += len(ccr_calls)
|
||||
results = [self._execute_retrieval(call) for call in ccr_calls]
|
||||
message = self._create_tool_result_message(results, "anthropic")
|
||||
return list(message.get("content", []))
|
||||
|
||||
def has_ccr_tool_calls(
|
||||
self,
|
||||
response: dict[str, Any],
|
||||
|
||||
@@ -625,43 +625,38 @@ class AnthropicHandlerMixin:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _outgoing_body_has_redeemable_marker(
|
||||
def _outgoing_redeemable_markers(
|
||||
body: Any,
|
||||
previous_forwarded_messages: list[dict[str, Any]] | None = None,
|
||||
) -> bool:
|
||||
"""Does the outgoing body show the model a marker it has not seen yet?
|
||||
) -> tuple[bool, bool]:
|
||||
"""Which owned markers does the outgoing body carry?
|
||||
|
||||
``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so
|
||||
a request carrying none cannot benefit from the buffered path (#3071).
|
||||
Returns ``(any_owned, new_owned)``. ``headroom_retrieve`` exists only to
|
||||
expand a ``<<ccr:...>>`` marker, so a request carrying none cannot use
|
||||
it (#3071).
|
||||
|
||||
Only markers new to the model count. A marker stays in the history for
|
||||
the rest of the session, so counting every marker in the body buffered
|
||||
every later turn once one tool result had been compressed: the buffered
|
||||
share climbed toward 90% of turns with zero retrievals, and
|
||||
time-to-first-byte became the whole generation (#3587). A marker that
|
||||
was already forwarded on an earlier turn was in front of the model on
|
||||
that turn, and that turn was buffered.
|
||||
|
||||
"New" is decided against what this session actually forwarded last
|
||||
turn, not by message position: token mode can compress an earlier tool
|
||||
result on this request, so a marker first seen now may sit anywhere in
|
||||
the history. With no forwarded history to compare against, every owned
|
||||
marker counts as new.
|
||||
``new_owned`` is a marker the model has not seen yet: one absent from
|
||||
what this session forwarded last turn. With no forwarded history to
|
||||
compare against, every owned marker counts as new. "New" is decided
|
||||
against the forwarded history, not by message position: token mode can
|
||||
compress an earlier tool result on this request, so a marker first
|
||||
seen now may sit anywhere in the history.
|
||||
|
||||
Ownership is verified rather than shape-matched: the marker shape is not
|
||||
unique to Headroom, and adopting another context tool's hash would send
|
||||
the model to an endpoint that is guaranteed to miss (#2836). A hash that
|
||||
survives ``verify_ownership`` is redeemable right now.
|
||||
|
||||
Errors are swallowed deliberately and answered ``True``. This gates a
|
||||
wire-format decision, and the safe direction on an unexpected message
|
||||
shape is the long-standing buffered behavior, not a silent change.
|
||||
Errors are swallowed deliberately and answered ``(True, True)``. This
|
||||
gates a wire-format decision, and the safe direction on an unexpected
|
||||
message shape is the long-standing buffered behavior, not a silent
|
||||
change.
|
||||
"""
|
||||
if not isinstance(body, dict):
|
||||
return True
|
||||
return True, True
|
||||
messages = body.get("messages")
|
||||
if not isinstance(messages, list) or not messages:
|
||||
return False
|
||||
return False, False
|
||||
try:
|
||||
from headroom.ccr.tool_injection import CCRToolInjector
|
||||
from headroom.proxy.helpers import has_new_ccr_markers
|
||||
@@ -673,16 +668,27 @@ class AnthropicHandlerMixin:
|
||||
)
|
||||
probe.scan_for_markers(messages)
|
||||
if not probe.detected_hashes:
|
||||
return False
|
||||
return False, False
|
||||
probe.verify_ownership()
|
||||
return has_new_ccr_markers(
|
||||
if not probe.detected_hashes:
|
||||
return False, False
|
||||
return True, has_new_ccr_markers(
|
||||
current_detected_hashes=list(probe.detected_hashes),
|
||||
previous_forwarded_messages=previous_forwarded_messages,
|
||||
provider="anthropic",
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)
|
||||
return True
|
||||
return True, True
|
||||
|
||||
@classmethod
|
||||
def _outgoing_body_has_redeemable_marker(
|
||||
cls,
|
||||
body: Any,
|
||||
previous_forwarded_messages: list[dict[str, Any]] | None = None,
|
||||
) -> bool:
|
||||
"""Does the outgoing body show the model an owned marker it has not seen?"""
|
||||
return cls._outgoing_redeemable_markers(body, previous_forwarded_messages)[1]
|
||||
|
||||
@staticmethod
|
||||
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
|
||||
@@ -4165,20 +4171,34 @@ class AnthropicHandlerMixin:
|
||||
and ccr_response_handler_enabled
|
||||
and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
|
||||
)
|
||||
buffered_retrieval_can_help = (
|
||||
retrieve_tool_is_offered
|
||||
and self._outgoing_body_has_redeemable_marker(body, previous_forwarded_messages)
|
||||
#
|
||||
# A marker the model already saw on an earlier turn can still be
|
||||
# retrieved, so it does not justify buffering every later turn
|
||||
# (#3587), but it does mean a retrieval call may come. Those
|
||||
# turns stream with ``headroom_retrieve`` withheld from the
|
||||
# client and served by the proxy in a continuation round.
|
||||
owned_marker, new_owned_marker = (
|
||||
self._outgoing_redeemable_markers(body, previous_forwarded_messages)
|
||||
if retrieve_tool_is_offered
|
||||
else (False, False)
|
||||
)
|
||||
wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)
|
||||
if retrieve_tool_is_offered and not buffered_retrieval_can_help:
|
||||
wants_buffered_stream_ccr = bool(new_owned_marker)
|
||||
if retrieve_tool_is_offered and not owned_marker:
|
||||
logger.info(
|
||||
f"[{request_id}] CCR: headroom_retrieve is resident but this "
|
||||
"request carries no redeemable marker, so server-side "
|
||||
"retrieval cannot fire; keeping the streaming path (#3071)"
|
||||
)
|
||||
elif retrieve_tool_is_offered and not new_owned_marker:
|
||||
logger.info(
|
||||
f"[{request_id}] CCR: every redeemable marker was already "
|
||||
"forwarded; streaming with headroom_retrieve served by the "
|
||||
"proxy (#3587)"
|
||||
)
|
||||
buffered_stream_ccr = (
|
||||
wants_buffered_stream_ccr and not outbound_locked_to_client_bytes
|
||||
)
|
||||
serve_ccr_retrieval_in_stream = bool(owned_marker) and not buffered_stream_ccr
|
||||
if wants_buffered_stream_ccr and outbound_locked_to_client_bytes:
|
||||
logger.info(
|
||||
f"[{request_id}] CCR: signed thinking blocks force byte-faithful "
|
||||
@@ -4352,6 +4372,7 @@ class AnthropicHandlerMixin:
|
||||
session_key=session_key,
|
||||
server_memory_tool_names=server_memory_tool_names,
|
||||
client_beta=client_anthropic_beta or "",
|
||||
serve_ccr_retrieval=serve_ccr_retrieval_in_stream,
|
||||
)
|
||||
else:
|
||||
# Whatever set it — the client's own ``stream: false`` or
|
||||
|
||||
@@ -520,17 +520,26 @@ class StreamingMixin:
|
||||
server_memory_tool_names: frozenset[str],
|
||||
stream_state: dict[str, Any],
|
||||
request_id: str,
|
||||
serve_retrieval: bool = False,
|
||||
) -> AsyncIterator[bytes]:
|
||||
"""Execute withheld memory tool calls and stream continuation rounds.
|
||||
"""Execute withheld proxy tool calls and stream continuation rounds.
|
||||
|
||||
``memory_filter`` has forwarded one upstream round to the client minus
|
||||
the proxy-owned memory tool calls and the message tail. When the round
|
||||
stopped on memory tool calls alone, run them, send the results upstream
|
||||
the proxy-owned tool calls and the message tail. When the round
|
||||
stopped on proxy tool calls alone, run them, send the results upstream
|
||||
as the next turn, and stream that round's blocks into the same client
|
||||
message. Otherwise end the client message. A round that also called a
|
||||
client tool still has its memory calls executed, but the turn goes back
|
||||
to the client, which cannot carry the memory results.
|
||||
|
||||
With ``serve_retrieval``, ``headroom_retrieve`` is a proxy tool too: the
|
||||
client never declared it, so a call that reached it would fail (#3587).
|
||||
The filter then holds client tool calls until the round ends, and a
|
||||
round that retrieved alongside a client tool is continued without the
|
||||
unrun client calls, for the model to re-issue with the content in
|
||||
hand. That is what the buffered path does with the same round.
|
||||
"""
|
||||
from headroom.ccr.tool_injection import CCR_TOOL_NAME
|
||||
from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE
|
||||
|
||||
try:
|
||||
@@ -541,18 +550,47 @@ class StreamingMixin:
|
||||
if not isinstance(messages, list):
|
||||
messages = None
|
||||
headers = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}
|
||||
proxy_tool_names = server_memory_tool_names | (
|
||||
frozenset({CCR_TOOL_NAME}) if serve_retrieval else frozenset()
|
||||
)
|
||||
|
||||
def is_memory_call(block: Any) -> bool:
|
||||
def is_proxy_call(block: Any) -> bool:
|
||||
return (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "tool_use"
|
||||
and block.get("name") in server_memory_tool_names
|
||||
and block.get("name") in proxy_tool_names
|
||||
)
|
||||
|
||||
def is_client_call(block: Any) -> bool:
|
||||
return (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "tool_use"
|
||||
and not is_proxy_call(block)
|
||||
)
|
||||
|
||||
rounds = 0
|
||||
# What the client received across rounds; the prefix tracker must see
|
||||
# this, not round one, because it is what the client sends back.
|
||||
visible_content: list[Any] = []
|
||||
|
||||
def record_visible(
|
||||
round_response: dict[str, Any] | None,
|
||||
round_content: list[Any],
|
||||
*,
|
||||
dropped_client_calls: bool,
|
||||
) -> None:
|
||||
if round_response is None:
|
||||
return
|
||||
visible_content.extend(
|
||||
block
|
||||
for block in round_content
|
||||
if not is_proxy_call(block) and not (dropped_client_calls and is_client_call(block))
|
||||
)
|
||||
stream_state["memory_client_message"] = {
|
||||
**round_response,
|
||||
"content": list(visible_content),
|
||||
}
|
||||
|
||||
# Usage of rounds whose message_delta the client never receives.
|
||||
prior_usage: dict[str, int] = {}
|
||||
while True:
|
||||
@@ -560,13 +598,9 @@ class StreamingMixin:
|
||||
if response is not None:
|
||||
round_content = response.get("content")
|
||||
content = round_content if isinstance(round_content, list) else []
|
||||
visible_content.extend(block for block in content if not is_memory_call(block))
|
||||
stream_state["memory_client_message"] = {
|
||||
**response,
|
||||
"content": list(visible_content),
|
||||
}
|
||||
|
||||
if not memory_filter.hid_tool_calls:
|
||||
record_visible(response, content, dropped_client_calls=False)
|
||||
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
|
||||
yield frame
|
||||
return
|
||||
@@ -574,7 +608,11 @@ class StreamingMixin:
|
||||
# Run the calls the filter recorded, so they are not lost when the
|
||||
# round could not be reconstructed. Only the proxy's own calls: a
|
||||
# memory-named tool the client declared is the client's to run.
|
||||
memory_calls = memory_filter.hidden_calls()
|
||||
hidden_calls = memory_filter.hidden_calls()
|
||||
memory_calls = [c for c in hidden_calls if c.get("name") in server_memory_tool_names]
|
||||
retrieval_calls = [
|
||||
c for c in hidden_calls if serve_retrieval and c.get("name") == CCR_TOOL_NAME
|
||||
]
|
||||
tool_results: list[dict[str, Any]] = []
|
||||
if memory_calls and memory_user_id is not None and self.memory_handler is not None:
|
||||
tool_results = await self.memory_handler.handle_memory_tool_calls(
|
||||
@@ -583,24 +621,38 @@ class StreamingMixin:
|
||||
"anthropic",
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
logger.info(
|
||||
f"[{request_id}] Memory: Executed {len(tool_results)}/"
|
||||
f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "
|
||||
"withheld from the SSE stream"
|
||||
)
|
||||
logger.info(
|
||||
f"[{request_id}] Memory: Executed {len(tool_results)}/"
|
||||
f"{len(memory_calls)} proxy-handled tool call(s) "
|
||||
"withheld from the SSE stream"
|
||||
)
|
||||
ccr_handler = getattr(self, "ccr_response_handler", None)
|
||||
if retrieval_calls and ccr_handler is not None:
|
||||
retrieval_results = ccr_handler.serve_withheld_retrievals(retrieval_calls)
|
||||
tool_results = [*tool_results, *retrieval_results]
|
||||
logger.info(
|
||||
f"[{request_id}] CCR: Served {len(retrieval_results)}/"
|
||||
f"{len(retrieval_calls)} headroom_retrieve call(s) withheld from "
|
||||
"the SSE stream"
|
||||
)
|
||||
|
||||
replayable = response is not None and len(tool_results) == sum(
|
||||
1 for block in content if is_memory_call(block)
|
||||
1 for block in content if is_proxy_call(block)
|
||||
)
|
||||
# Unrun client calls can be dropped only while the client has not
|
||||
# seen them, and only for a retrieval: memory results alone do not
|
||||
# give the model a reason to decide again.
|
||||
drop_client_calls = bool(retrieval_calls) and memory_filter.holds_client_tool_use
|
||||
can_continue = (
|
||||
memory_filter.stop_reason == "tool_use"
|
||||
and not memory_filter.visible_tool_use
|
||||
and (drop_client_calls or not memory_filter.holds_client_tool_use)
|
||||
and messages is not None
|
||||
and replayable
|
||||
and rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS
|
||||
)
|
||||
if not can_continue:
|
||||
if memory_filter.visible_tool_use:
|
||||
if memory_filter.visible_tool_use or memory_filter.holds_client_tool_use:
|
||||
logger.info(
|
||||
f"[{request_id}] Memory: Round also called a client tool; "
|
||||
"returning the turn to the client"
|
||||
@@ -608,18 +660,31 @@ class StreamingMixin:
|
||||
elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:
|
||||
logger.warning(
|
||||
f"[{request_id}] Memory: Stopped after {rounds} continuation "
|
||||
"round(s) that only called memory tools"
|
||||
"round(s) that only called proxy tools"
|
||||
)
|
||||
elif not replayable:
|
||||
logger.warning(
|
||||
f"[{request_id}] Memory: Could not rebuild the round for a "
|
||||
"continuation; ending the turn after running its memory calls"
|
||||
"continuation; ending the turn after running its proxy calls"
|
||||
)
|
||||
record_visible(response, content, dropped_client_calls=False)
|
||||
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
|
||||
yield frame
|
||||
return
|
||||
|
||||
assert response is not None and messages is not None and isinstance(base_body, dict)
|
||||
record_visible(response, content, dropped_client_calls=drop_client_calls)
|
||||
# Blocks held after the first tool call (rarely any but the calls
|
||||
# themselves) still belong to the client's message.
|
||||
for frame in memory_filter.release_held(drop_tool_use=drop_client_calls):
|
||||
yield frame
|
||||
if drop_client_calls:
|
||||
logger.info(
|
||||
f"[{request_id}] CCR: Round called headroom_retrieve alongside a "
|
||||
"client tool; serving the retrieval and dropping the unrun client "
|
||||
"call(s) for the model to re-issue"
|
||||
)
|
||||
content = [block for block in content if not is_client_call(block)]
|
||||
round_usage = response.get("usage")
|
||||
if isinstance(round_usage, dict):
|
||||
for key in _ROUND_USAGE_KEYS:
|
||||
@@ -647,9 +712,10 @@ class StreamingMixin:
|
||||
yield _upstream_error_frame(error_bytes, request_id)
|
||||
return
|
||||
memory_filter = MemoryToolStreamFilter(
|
||||
server_memory_tool_names,
|
||||
proxy_tool_names,
|
||||
index_offset=memory_filter.next_index,
|
||||
forward_message_start=False,
|
||||
hold_tool_use=serve_retrieval,
|
||||
)
|
||||
# Same cap as the first round's buffer; past it the round is
|
||||
# still streamed, just not rebuilt for a further continuation.
|
||||
@@ -1079,6 +1145,7 @@ class StreamingMixin:
|
||||
savings_measure: ItemMeasure | None = None,
|
||||
server_memory_tool_names: frozenset[str] | None = None,
|
||||
client_beta: str | None = None,
|
||||
serve_ccr_retrieval: bool = False,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Stream response with metrics tracking and memory tool handling.
|
||||
|
||||
@@ -1127,6 +1194,7 @@ class StreamingMixin:
|
||||
savings_measure=savings_measure,
|
||||
server_memory_tool_names=server_memory_tool_names,
|
||||
client_beta=client_beta,
|
||||
serve_ccr_retrieval=serve_ccr_retrieval,
|
||||
)
|
||||
except (Exception, asyncio.CancelledError):
|
||||
self._cleanup_mid_turn_stream(session_key)
|
||||
@@ -1162,6 +1230,7 @@ class StreamingMixin:
|
||||
savings_measure: ItemMeasure | None = None,
|
||||
server_memory_tool_names: frozenset[str] | None = None,
|
||||
client_beta: str | None = None,
|
||||
serve_ccr_retrieval: bool = False,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
@@ -1336,6 +1405,9 @@ class StreamingMixin:
|
||||
and self.memory_handler is not None
|
||||
and provider == "anthropic"
|
||||
)
|
||||
# Withholding headroom_retrieve needs the Anthropic SSE shape the
|
||||
# filter and the continuation round are written for.
|
||||
serve_ccr_retrieval = serve_ccr_retrieval and provider == "anthropic"
|
||||
|
||||
# Open connection before generator to capture upstream response headers
|
||||
# (needed to forward ratelimit headers to the client via StreamingResponse)
|
||||
@@ -1663,11 +1735,23 @@ class StreamingMixin:
|
||||
pending_messages: list[dict] = []
|
||||
# Proxy-injected memory tools are invisible to the client, so their
|
||||
# tool_use blocks must never reach it (GH #2195).
|
||||
from headroom.ccr.tool_injection import CCR_TOOL_NAME
|
||||
|
||||
# Memory tools are withheld only when this request can run them;
|
||||
# the continuation must classify calls exactly as the filter did.
|
||||
filter_memory_tool_names = (
|
||||
server_memory_tool_names or frozenset() if memory_enabled else frozenset()
|
||||
)
|
||||
proxy_tool_names = filter_memory_tool_names | (
|
||||
frozenset({CCR_TOOL_NAME}) if serve_ccr_retrieval else frozenset()
|
||||
)
|
||||
memory_filter = (
|
||||
MemoryToolStreamFilter(server_memory_tool_names)
|
||||
if memory_enabled and server_memory_tool_names
|
||||
MemoryToolStreamFilter(proxy_tool_names, hold_tool_use=serve_ccr_retrieval)
|
||||
if proxy_tool_names
|
||||
else None
|
||||
)
|
||||
# A withheld retrieval needs the round rebuilt for its continuation.
|
||||
retrieval_tracking = serve_ccr_retrieval
|
||||
|
||||
def _client_frame(frame: bytes) -> bytes:
|
||||
"""``_guard_client_bytes``, noting when the terminal event goes out."""
|
||||
@@ -1734,6 +1818,7 @@ class StreamingMixin:
|
||||
_track_sse = (
|
||||
_codex_wire_debug
|
||||
or memory_enabled
|
||||
or retrieval_tracking
|
||||
or (prefix_tracker is not None and provider == "anthropic")
|
||||
)
|
||||
if _track_sse:
|
||||
@@ -1746,6 +1831,7 @@ class StreamingMixin:
|
||||
"disabling memory detection for this request"
|
||||
)
|
||||
memory_enabled = False
|
||||
retrieval_tracking = False
|
||||
|
||||
# Parse complete SSE events from buffer. A failure (e.g.
|
||||
# invalid UTF-8) is raised once the chunk is forwarded:
|
||||
@@ -1806,7 +1892,7 @@ class StreamingMixin:
|
||||
# be an upstream protocol violation — surface loudly.
|
||||
full_sse_data: str = full_sse_bytes.decode("utf-8") if full_sse_bytes else ""
|
||||
|
||||
if memory_enabled and full_sse_data:
|
||||
if (memory_enabled or retrieval_tracking) and full_sse_data:
|
||||
# Check for Claude Code credential error
|
||||
if "only authorized for use with Claude Code" in full_sse_data:
|
||||
logger.warning(
|
||||
@@ -1836,9 +1922,10 @@ class StreamingMixin:
|
||||
outbound_bytes=outbound_bytes,
|
||||
memory_user_id=memory_user_id,
|
||||
memory_request_ctx=memory_request_ctx,
|
||||
server_memory_tool_names=server_memory_tool_names or frozenset(),
|
||||
server_memory_tool_names=filter_memory_tool_names,
|
||||
stream_state=stream_state,
|
||||
request_id=request_id,
|
||||
serve_retrieval=serve_ccr_retrieval,
|
||||
):
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
@@ -1847,8 +1934,10 @@ class StreamingMixin:
|
||||
parsed_response = (
|
||||
stream_state.pop("memory_client_message", None) or parsed_response
|
||||
)
|
||||
elif parsed_response and self.memory_handler.has_memory_tool_calls(
|
||||
parsed_response, provider
|
||||
elif (
|
||||
memory_enabled
|
||||
and parsed_response
|
||||
and self.memory_handler.has_memory_tool_calls(parsed_response, provider)
|
||||
):
|
||||
logger.info(
|
||||
f"[{request_id}] Memory: Detected tool calls in streaming response"
|
||||
@@ -1881,9 +1970,10 @@ class StreamingMixin:
|
||||
outbound_bytes=outbound_bytes,
|
||||
memory_user_id=memory_user_id,
|
||||
memory_request_ctx=memory_request_ctx,
|
||||
server_memory_tool_names=server_memory_tool_names or frozenset(),
|
||||
server_memory_tool_names=filter_memory_tool_names,
|
||||
stream_state=stream_state,
|
||||
request_id=request_id,
|
||||
serve_retrieval=serve_ccr_retrieval,
|
||||
):
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
|
||||
@@ -90,6 +90,12 @@ class MemoryToolStreamFilter:
|
||||
extend the message the client already opened.
|
||||
max_retained_bytes: Limit on the bytes held back at any time;
|
||||
``DEFAULT_MAX_RETAINED_BYTES`` when None.
|
||||
hold_tool_use: Hold every block from the round's first ``tool_use``
|
||||
on, until the caller decides with ``release_held``. A withheld
|
||||
``headroom_retrieve`` alongside a client tool is served by
|
||||
dropping the client call for the model to re-issue, which is
|
||||
only possible while that call has not reached the client.
|
||||
Text before the first tool call still streams live.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -99,8 +105,17 @@ class MemoryToolStreamFilter:
|
||||
index_offset: int = 0,
|
||||
forward_message_start: bool = True,
|
||||
max_retained_bytes: int | None = None,
|
||||
hold_tool_use: bool = False,
|
||||
) -> None:
|
||||
self._tool_names = frozenset(tool_names)
|
||||
self._hold_tool_use = hold_tool_use
|
||||
self._holding = False
|
||||
# Frames held from the first tool_use on, with their upstream index;
|
||||
# client indices are assigned when they are released.
|
||||
self._held: list[tuple[str, dict[str, Any], bytes, int]] = []
|
||||
self._held_bytes = 0
|
||||
# Upstream indices of held client tool_use blocks.
|
||||
self._held_tool_use: set[int] = set()
|
||||
self._max_retained_bytes = (
|
||||
DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes
|
||||
)
|
||||
@@ -130,6 +145,39 @@ class MemoryToolStreamFilter:
|
||||
def hidden_tool_names(self) -> list[str]:
|
||||
return [block["name"] for block, _ in self._hidden.values()]
|
||||
|
||||
@property
|
||||
def holds_client_tool_use(self) -> bool:
|
||||
"""A client ``tool_use`` block is held and has not reached the client."""
|
||||
return bool(self._held_tool_use)
|
||||
|
||||
def release_held(self, *, drop_tool_use: bool = False) -> list[bytes]:
|
||||
"""Return the held frames, re-indexed, for the client.
|
||||
|
||||
``drop_tool_use`` leaves out the held client ``tool_use`` blocks: the
|
||||
round goes back to the model with the proxy's results instead, and
|
||||
the model re-issues whatever it still needs.
|
||||
"""
|
||||
frames: list[bytes] = []
|
||||
dropped = self._held_tool_use if drop_tool_use else set()
|
||||
for event_name, payload, frame, upstream_index in self._held:
|
||||
if upstream_index in dropped:
|
||||
continue
|
||||
if event_name == "content_block_start":
|
||||
block = payload.get("content_block")
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
self.visible_tool_use = True
|
||||
self._index_map[upstream_index] = self.next_index
|
||||
self.next_index += 1
|
||||
client_index = self._index_map.get(upstream_index, upstream_index)
|
||||
if client_index == upstream_index:
|
||||
frames.append(frame)
|
||||
else:
|
||||
frames.append(_render_frame(event_name, {**payload, "index": client_index}))
|
||||
self._held.clear()
|
||||
self._held_bytes = 0
|
||||
self._held_tool_use.clear()
|
||||
return frames
|
||||
|
||||
def hidden_calls(self) -> list[dict[str, Any]]:
|
||||
"""The withheld ``tool_use`` blocks that finished with a complete input.
|
||||
|
||||
@@ -181,7 +229,12 @@ class MemoryToolStreamFilter:
|
||||
forwarded = self._route(frame)
|
||||
if forwarded is not None:
|
||||
out.append(forwarded)
|
||||
retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())
|
||||
retained = (
|
||||
len(self._buffer)
|
||||
+ self._tail_bytes
|
||||
+ self._held_bytes
|
||||
+ sum(self._hidden_bytes.values())
|
||||
)
|
||||
if retained > self._max_retained_bytes:
|
||||
self._buffer.clear()
|
||||
self._hidden.clear()
|
||||
@@ -189,6 +242,9 @@ class MemoryToolStreamFilter:
|
||||
self._hidden_stopped.clear()
|
||||
self._tail.clear()
|
||||
self._tail_bytes = 0
|
||||
self._held.clear()
|
||||
self._held_bytes = 0
|
||||
self._held_tool_use.clear()
|
||||
raise MemoryToolStreamOverflowError(
|
||||
f"memory tool stream filter would retain {retained} bytes "
|
||||
f"(limit {self._max_retained_bytes})"
|
||||
@@ -232,11 +288,18 @@ class MemoryToolStreamFilter:
|
||||
self._hidden[upstream_index] = (block, [])
|
||||
self._hidden_bytes[upstream_index] = len(frame)
|
||||
self._hidden_stopped.discard(upstream_index)
|
||||
self._holding = self._hold_tool_use
|
||||
return None
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
self.visible_tool_use = True
|
||||
self._index_map[upstream_index] = self.next_index
|
||||
self.next_index += 1
|
||||
is_tool_use = isinstance(block, dict) and block.get("type") == "tool_use"
|
||||
if self._hold_tool_use and (self._holding or is_tool_use):
|
||||
self._holding = True
|
||||
if is_tool_use:
|
||||
self._held_tool_use.add(upstream_index)
|
||||
else:
|
||||
if is_tool_use:
|
||||
self.visible_tool_use = True
|
||||
self._index_map[upstream_index] = self.next_index
|
||||
self.next_index += 1
|
||||
|
||||
if upstream_index in self._hidden:
|
||||
delta = payload.get("delta")
|
||||
@@ -252,17 +315,26 @@ class MemoryToolStreamFilter:
|
||||
elif event_name == "content_block_stop":
|
||||
self._hidden_stopped.add(upstream_index)
|
||||
return None
|
||||
if self._holding and upstream_index not in self._index_map:
|
||||
self._held.append((event_name, payload, frame, upstream_index))
|
||||
self._held_bytes += len(frame)
|
||||
return None
|
||||
client_index = self._index_map.get(upstream_index, upstream_index)
|
||||
if client_index == upstream_index:
|
||||
return frame
|
||||
return _render_frame(event_name, {**payload, "index": client_index})
|
||||
|
||||
def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:
|
||||
"""End the client message after this round without a continuation."""
|
||||
"""End the client message after this round without a continuation.
|
||||
|
||||
Held frames are released first, client tool calls included: with no
|
||||
continuation, the round is the model's answer as it stands.
|
||||
"""
|
||||
held = self.release_held()
|
||||
ends_on_hidden_call = (
|
||||
self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use"
|
||||
)
|
||||
return self.tail_frames(
|
||||
return held + self.tail_frames(
|
||||
stop_reason="end_turn" if ends_on_hidden_call else None,
|
||||
prior_usage=prior_usage,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
"""``headroom_retrieve`` on a streamed turn is served by the proxy (#3587).
|
||||
|
||||
A turn whose redeemable markers were all forwarded on earlier turns streams
|
||||
instead of buffering, so the client gets the first token right away. The model
|
||||
can still ask for one of those originals. The client never declared
|
||||
``headroom_retrieve`` (a coding agent answers "No such tool available"), so
|
||||
the call must not reach it: the stream withholds it, the proxy serves it from
|
||||
the compression store, and the continuation round streams into the same client
|
||||
message.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
fastapi = pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
|
||||
from headroom.proxy.memory_tool_stream import MemoryToolStreamFilter # noqa: E402
|
||||
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
||||
|
||||
RETRIEVE_ONLY = frozenset({"headroom_retrieve"})
|
||||
ORIGINAL = "line 1 of the original build log\nline 2: the error the model needs"
|
||||
|
||||
|
||||
def _frame(payload: dict[str, Any]) -> bytes:
|
||||
return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
def _sse(blocks: list[dict[str, Any]], stop_reason: str) -> bytes:
|
||||
"""An Anthropic SSE response carrying ``blocks``."""
|
||||
frames = [
|
||||
_frame(
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "sonnet-test",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
}
|
||||
)
|
||||
]
|
||||
for index, block in enumerate(blocks):
|
||||
if block["type"] == "text":
|
||||
start = {"type": "text", "text": ""}
|
||||
delta = {"type": "text_delta", "text": block["text"]}
|
||||
else:
|
||||
start = {**block, "input": {}}
|
||||
delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
|
||||
frames.append(
|
||||
_frame({"type": "content_block_start", "index": index, "content_block": start})
|
||||
)
|
||||
frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))
|
||||
frames.append(_frame({"type": "content_block_stop", "index": index}))
|
||||
frames.append(
|
||||
_frame(
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
|
||||
"usage": {"output_tokens": 5},
|
||||
}
|
||||
)
|
||||
)
|
||||
frames.append(_frame({"type": "message_stop"}))
|
||||
return b"".join(frames)
|
||||
|
||||
|
||||
def _events(raw: bytes) -> list[dict[str, Any]]:
|
||||
return [
|
||||
json.loads(line[len("data: ") :])
|
||||
for line in raw.decode().splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
|
||||
|
||||
def _tool_names(events: list[dict[str, Any]]) -> list[str]:
|
||||
return [
|
||||
e["content_block"]["name"]
|
||||
for e in events
|
||||
if e["type"] == "content_block_start" and e["content_block"].get("type") == "tool_use"
|
||||
]
|
||||
|
||||
|
||||
TEXT = {"type": "text", "text": "Let me look at the earlier output."}
|
||||
BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
|
||||
|
||||
|
||||
def _retrieve(hash_key: str) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "tool_use",
|
||||
"id": "toolu_ccr",
|
||||
"name": "headroom_retrieve",
|
||||
"input": {"hash": hash_key},
|
||||
}
|
||||
|
||||
|
||||
class TestHoldingFilter:
|
||||
def test_text_before_the_first_tool_call_streams_live(self) -> None:
|
||||
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
|
||||
forwarded = _events(b"".join(flt.feed(_sse([TEXT, BASH], "tool_use"))))
|
||||
assert [e["type"] for e in forwarded] == [
|
||||
"message_start",
|
||||
"content_block_start",
|
||||
"content_block_delta",
|
||||
"content_block_stop",
|
||||
]
|
||||
assert flt.holds_client_tool_use
|
||||
|
||||
def test_without_a_retrieval_the_round_is_byte_identical(self) -> None:
|
||||
raw = _sse([TEXT, BASH], "tool_use")
|
||||
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
|
||||
out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())
|
||||
assert out == raw
|
||||
|
||||
def test_dropping_held_client_calls(self) -> None:
|
||||
flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
|
||||
flt.feed(_sse([BASH, _retrieve("abc")], "tool_use"))
|
||||
assert flt.hidden_tool_names == ["headroom_retrieve"]
|
||||
assert flt.release_held(drop_tool_use=True) == []
|
||||
assert flt.next_index == 0
|
||||
assert not flt.visible_tool_use
|
||||
|
||||
|
||||
def _sse_response(raw: bytes) -> MagicMock:
|
||||
response = MagicMock()
|
||||
response.headers = httpx.Headers({"content-type": "text/event-stream"})
|
||||
response.status_code = 200
|
||||
|
||||
async def aiter_bytes(raw: bytes = raw):
|
||||
# Split mid-frame so the filter has to reassemble events.
|
||||
yield raw[:41]
|
||||
yield raw[41:]
|
||||
|
||||
response.aiter_bytes = aiter_bytes
|
||||
response.aclose = AsyncMock()
|
||||
return response
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owned_marker():
|
||||
"""A marker the proxy owns, with its original in the compression store."""
|
||||
from headroom.cache.backends import InMemoryBackend
|
||||
from headroom.cache.compression_store import get_compression_store, reset_compression_store
|
||||
|
||||
reset_compression_store()
|
||||
store = get_compression_store(backend=InMemoryBackend())
|
||||
hash_key = store.store(
|
||||
ORIGINAL,
|
||||
"<<ccr:placeholder>>",
|
||||
original_tokens=100,
|
||||
compressed_tokens=5,
|
||||
tool_name="Bash",
|
||||
)
|
||||
try:
|
||||
yield hash_key
|
||||
finally:
|
||||
reset_compression_store()
|
||||
|
||||
|
||||
def _two_turn_session(
|
||||
marker: str, second_turn_rounds: list[bytes]
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
|
||||
"""Turn 1 introduces ``marker``; turn 2 only carries it in history.
|
||||
|
||||
Returns the client's SSE events for turn 2, the bodies the proxy sent
|
||||
upstream on turn 2's stream (first round, then continuations), and how
|
||||
many buffered upstream requests turn 2 made.
|
||||
"""
|
||||
first_turn = [
|
||||
{"role": "user", "content": "build it"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "tu_1",
|
||||
"content": f"build log compressed at <<ccr:{marker}>>",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
second_turn = [
|
||||
*first_turn,
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "The build failed."}]},
|
||||
{"role": "user", "content": "what was the exact error?"},
|
||||
]
|
||||
buffered_calls: list[int] = []
|
||||
|
||||
async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
|
||||
buffered_calls.append(1)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_t1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "sonnet-test",
|
||||
"content": [{"type": "text", "text": "The build failed."}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
},
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
|
||||
config = ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=False,
|
||||
rate_limit_enabled=False,
|
||||
memory_enabled=False,
|
||||
)
|
||||
headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
|
||||
tools = [
|
||||
create_ccr_tool_definition("anthropic"),
|
||||
{"name": "Bash", "description": "run", "input_schema": {"type": "object"}},
|
||||
]
|
||||
|
||||
def body(messages: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {
|
||||
"model": "sonnet-test",
|
||||
"max_tokens": 64,
|
||||
"stream": True,
|
||||
"tools": tools,
|
||||
"messages": messages,
|
||||
}
|
||||
|
||||
app = create_app(config)
|
||||
with TestClient(app) as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy._retry_request = fake_retry
|
||||
first = client.post("/v1/messages", json=body(first_turn), headers=headers)
|
||||
assert first.status_code == 200, first.text
|
||||
assert len(buffered_calls) == 1, "the turn that introduces the marker buffers"
|
||||
|
||||
upstream = MagicMock(spec=httpx.AsyncClient)
|
||||
upstream.build_request = MagicMock(return_value=MagicMock())
|
||||
upstream.send = AsyncMock(side_effect=[_sse_response(r) for r in second_turn_rounds])
|
||||
proxy.http_client = upstream
|
||||
second = client.post("/v1/messages", json=body(second_turn), headers=headers)
|
||||
assert second.status_code == 200, second.text
|
||||
|
||||
sent = [json.loads(c.kwargs["content"]) for c in upstream.build_request.call_args_list]
|
||||
return _events(second.content), sent, len(buffered_calls) - 1
|
||||
|
||||
|
||||
def test_later_turn_retrieval_is_served_by_the_proxy(owned_marker: str) -> None:
|
||||
events, sent, buffered = _two_turn_session(
|
||||
owned_marker,
|
||||
[
|
||||
_sse([TEXT, _retrieve(owned_marker)], "tool_use"),
|
||||
_sse([{"type": "text", "text": "The error is on line 2."}], "end_turn"),
|
||||
],
|
||||
)
|
||||
|
||||
assert buffered == 0, "a turn with only already-forwarded markers keeps streaming"
|
||||
assert "headroom_retrieve" not in _tool_names(events)
|
||||
texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
|
||||
assert texts == ["Let me look at the earlier output.", "The error is on line 2."]
|
||||
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
|
||||
assert stops == ["end_turn"]
|
||||
assert [e["type"] for e in events].count("message_start") == 1
|
||||
|
||||
assert len(sent) == 2
|
||||
continuation = sent[1]["messages"]
|
||||
assert continuation[-2]["content"][-1]["name"] == "headroom_retrieve"
|
||||
served = continuation[-1]["content"][0]
|
||||
assert served["type"] == "tool_result" and served["tool_use_id"] == "toolu_ccr"
|
||||
assert json.loads(served["content"])["original_content"] == ORIGINAL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_call_first", [True, False])
|
||||
def test_retrieval_mixed_with_a_client_tool(owned_marker: str, client_call_first: bool) -> None:
|
||||
calls = [BASH, _retrieve(owned_marker)]
|
||||
if not client_call_first:
|
||||
calls.reverse()
|
||||
reissued = {**BASH, "id": "toolu_bash_2"}
|
||||
events, sent, buffered = _two_turn_session(
|
||||
owned_marker,
|
||||
[
|
||||
_sse(calls, "tool_use"),
|
||||
_sse([{"type": "text", "text": "Now rerun it."}, reissued], "tool_use"),
|
||||
],
|
||||
)
|
||||
|
||||
assert buffered == 0
|
||||
# Only the model's decision made with the content in hand reaches the client.
|
||||
assert _tool_names(events) == ["Bash"]
|
||||
started = [e["content_block"] for e in events if e["type"] == "content_block_start"]
|
||||
assert [b.get("id") for b in started if b.get("type") == "tool_use"] == ["toolu_bash_2"]
|
||||
stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
|
||||
assert stops == ["tool_use"]
|
||||
assert [e["index"] for e in events if e["type"] == "content_block_start"] == [0, 1]
|
||||
|
||||
continuation = sent[1]["messages"]
|
||||
assert [b["name"] for b in continuation[-2]["content"]] == ["headroom_retrieve"]
|
||||
assert json.loads(continuation[-1]["content"][0]["content"])["original_content"] == ORIGINAL
|
||||
|
||||
|
||||
def test_old_marker_turn_without_retrieval_streams_unchanged(owned_marker: str) -> None:
|
||||
raw = _sse([{"type": "text", "text": "It failed on line 2."}, BASH], "tool_use")
|
||||
events, sent, buffered = _two_turn_session(owned_marker, [raw])
|
||||
|
||||
assert buffered == 0
|
||||
assert len(sent) == 1, "no continuation round without a retrieval"
|
||||
assert events == _events(raw)
|
||||
Reference in New Issue
Block a user