mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): count a hang-up as completed only after the client got the terminal event
The hang-up handler read stream_state["stream_terminal"], which the usage parser sets from the whole upstream chunk before any of it is forwarded. With the memory tool filter on, one chunk holding a whole message sets it before the client gets its first frame, and the filter holds message_delta and message_stop until after the stream. A client that closed at the first frame was recorded as a 200. Track the terminal event at the client-facing yield instead. Unfiltered, each yielded chunk carries every event parsed so far, so the parser's flag is reused. Filtered frames are whole events, so they are classified as they go out, including the closing and continuation frames. Moving the parse ahead of the yield also meant a chunk with a valid event followed by invalid UTF-8 was dropped and replaced by the error event. A parse failure is now raised after the chunk is forwarded, as before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from headroom.proxy.helpers import (
|
||||
RETRYABLE_OVERLOAD_STATUSES,
|
||||
jitter_delay_ms,
|
||||
overload_retry_is_futile,
|
||||
parse_sse_events_from_byte_buffer,
|
||||
retry_after_ms,
|
||||
)
|
||||
from headroom.proxy.provider_usage import (
|
||||
@@ -1656,6 +1657,9 @@ class StreamingMixin:
|
||||
# the outcome only, so pending messages are not drained into a
|
||||
# stream nobody reads.
|
||||
client_left_after_terminal = False
|
||||
# A terminal event the client was handed, not just one parsed from
|
||||
# upstream: the memory filter holds message_stop until the end.
|
||||
client_got_terminal = False
|
||||
pending_messages: list[dict] = []
|
||||
# Proxy-injected memory tools are invisible to the client, so their
|
||||
# tool_use blocks must never reach it (GH #2195).
|
||||
@@ -1665,6 +1669,23 @@ class StreamingMixin:
|
||||
else None
|
||||
)
|
||||
|
||||
def _client_frame(frame: bytes) -> bytes:
|
||||
"""``_guard_client_bytes``, noting when the terminal event goes out."""
|
||||
nonlocal client_got_terminal
|
||||
guarded = _guard_client_bytes(frame)
|
||||
if guarded and not client_got_terminal:
|
||||
if memory_filter is None:
|
||||
# Unfiltered, the client gets each whole chunk once it
|
||||
# is parsed, so every event the parser has seen.
|
||||
client_got_terminal = stream_state["stream_terminal"]
|
||||
else:
|
||||
# Filtered frames are complete events the filter decoded.
|
||||
client_got_terminal = any(
|
||||
_sse_event_outcome(name, data, stream_state["sse_dialect"])[1]
|
||||
for name, data in parse_sse_events_from_byte_buffer(bytearray(frame))
|
||||
)
|
||||
return guarded
|
||||
|
||||
try:
|
||||
async with contextlib.aclosing(upstream_response) as response:
|
||||
sse_chunk_index = 0
|
||||
@@ -1726,8 +1747,14 @@ class StreamingMixin:
|
||||
)
|
||||
memory_enabled = False
|
||||
|
||||
# Parse complete SSE events from buffer
|
||||
usage = self._parse_sse_usage_from_buffer(stream_state, provider)
|
||||
# Parse complete SSE events from buffer. A failure (e.g.
|
||||
# invalid UTF-8) is raised once the chunk is forwarded:
|
||||
# malformed upstream content still reaches the client.
|
||||
parse_error: Exception | None = None
|
||||
try:
|
||||
usage = self._parse_sse_usage_from_buffer(stream_state, provider)
|
||||
except Exception as e:
|
||||
usage, parse_error = None, e
|
||||
if usage:
|
||||
if "input_tokens" in usage:
|
||||
stream_state["input_tokens"] = usage["input_tokens"]
|
||||
@@ -1762,9 +1789,11 @@ class StreamingMixin:
|
||||
[chunk] if memory_filter is None else memory_filter.feed(chunk)
|
||||
)
|
||||
for frame in client_frames:
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
if parse_error is not None:
|
||||
raise parse_error
|
||||
|
||||
# Memory tool handling after stream completes
|
||||
# Chunks were already yielded in real-time above, so we only
|
||||
@@ -1787,7 +1816,7 @@ class StreamingMixin:
|
||||
)
|
||||
if memory_filter is not None:
|
||||
for frame in memory_filter.closing_frames():
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
guarded_tail = _drain_context_guard()
|
||||
@@ -1811,7 +1840,7 @@ class StreamingMixin:
|
||||
stream_state=stream_state,
|
||||
request_id=request_id,
|
||||
):
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
memory_filter = None
|
||||
@@ -1856,7 +1885,7 @@ class StreamingMixin:
|
||||
stream_state=stream_state,
|
||||
request_id=request_id,
|
||||
):
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
guarded_frame = _client_frame(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
|
||||
@@ -1911,7 +1940,7 @@ class StreamingMixin:
|
||||
# Codex closes the connection as soon as it reads
|
||||
# response.completed, before the upstream EOF ends the loop.
|
||||
client_left_after_terminal = (
|
||||
stream_state["stream_terminal"] and not stream_state["stream_failed"]
|
||||
client_got_terminal and not stream_state["stream_failed"]
|
||||
)
|
||||
raise
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
|
||||
|
||||
@@ -4,6 +4,8 @@ Codex CLI closes the connection as soon as it reads ``response.completed``,
|
||||
before the upstream EOF ends the proxy's read loop. The outcome was decided
|
||||
only after that loop, so every such turn recorded as a 502 failure with no
|
||||
usage and no cache metrics, although the client got a complete 200 response.
|
||||
Success needs the terminal event the client was handed, not one only parsed
|
||||
from upstream: the memory filter holds ``message_stop`` until after the stream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -40,8 +42,60 @@ COMPLETED = _event(
|
||||
},
|
||||
)
|
||||
|
||||
# A whole Anthropic message in one upstream chunk: the parser sees message_stop
|
||||
# before the client gets the first frame the memory filter splits it into.
|
||||
MESSAGE = b"".join(
|
||||
[
|
||||
_event(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "m1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-test",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 1_000, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
),
|
||||
_event(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text"}},
|
||||
),
|
||||
_event(
|
||||
"content_block_delta",
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": "ok"},
|
||||
},
|
||||
),
|
||||
_event("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
_event(
|
||||
"message_delta",
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 5},
|
||||
},
|
||||
),
|
||||
_event("message_stop", {"type": "message_stop"}),
|
||||
]
|
||||
)
|
||||
MEMORY_STREAM = {
|
||||
"url": "https://api.anthropic.com/v1/messages",
|
||||
"headers": {"x-api-key": "sk-test"},
|
||||
"body": {"model": "claude-test", "max_tokens": 100, "stream": True, "messages": []},
|
||||
"provider": "anthropic",
|
||||
"model": "claude-test",
|
||||
"memory_user_id": "user-1",
|
||||
"server_memory_tool_names": frozenset({"memory_save"}),
|
||||
}
|
||||
|
||||
def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
|
||||
|
||||
def _proxy(*chunks: bytes, upstream_eof: asyncio.Event | None = None) -> HeadroomProxy:
|
||||
proxy = object.__new__(HeadroomProxy)
|
||||
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
|
||||
proxy._config = MagicMock()
|
||||
@@ -55,9 +109,10 @@ def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
|
||||
response.status_code = 200
|
||||
|
||||
async def aiter_bytes():
|
||||
yield START
|
||||
yield COMPLETED
|
||||
await upstream_eof.wait() # the upstream EOF the client never waits for
|
||||
for chunk in chunks or (START, COMPLETED):
|
||||
yield chunk
|
||||
if upstream_eof is not None:
|
||||
await upstream_eof.wait() # the upstream EOF the client never waits for
|
||||
|
||||
response.aiter_bytes = aiter_bytes
|
||||
response.aclose = AsyncMock()
|
||||
@@ -66,27 +121,30 @@ def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
|
||||
return proxy
|
||||
|
||||
|
||||
async def _stream(proxy: HeadroomProxy):
|
||||
response = await proxy._stream_response(
|
||||
url="https://chatgpt.com/backend-api/codex/responses",
|
||||
headers={"authorization": "Bearer test"},
|
||||
body={"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
|
||||
provider="openai",
|
||||
model="gpt-5.4",
|
||||
request_id="test-hangup",
|
||||
original_tokens=10,
|
||||
optimized_tokens=10,
|
||||
tokens_saved=0,
|
||||
transforms_applied=[],
|
||||
tags={},
|
||||
optimization_latency=0.0,
|
||||
)
|
||||
async def _stream(proxy: HeadroomProxy, **overrides: Any):
|
||||
arguments: dict[str, Any] = {
|
||||
"url": "https://chatgpt.com/backend-api/codex/responses",
|
||||
"headers": {"authorization": "Bearer test"},
|
||||
"body": {"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.4",
|
||||
"request_id": "test-hangup",
|
||||
"original_tokens": 10,
|
||||
"optimized_tokens": 10,
|
||||
"tokens_saved": 0,
|
||||
"transforms_applied": [],
|
||||
"tags": {},
|
||||
"optimization_latency": 0.0,
|
||||
}
|
||||
if overrides.get("memory_user_id"):
|
||||
proxy.memory_handler = MagicMock()
|
||||
response = await proxy._stream_response(**{**arguments, **overrides})
|
||||
return response.body_iterator
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])
|
||||
async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):
|
||||
proxy = _proxy(asyncio.Event())
|
||||
proxy = _proxy(upstream_eof=asyncio.Event())
|
||||
body = await _stream(proxy)
|
||||
seen = b""
|
||||
if hangup == "closed_at_terminal_yield":
|
||||
@@ -117,10 +175,53 @@ async def test_client_hangup_after_terminal_event_records_success_with_usage(han
|
||||
|
||||
|
||||
async def test_client_hangup_before_terminal_event_is_still_a_failure():
|
||||
proxy = _proxy(asyncio.Event())
|
||||
proxy = _proxy(upstream_eof=asyncio.Event())
|
||||
body = await _stream(proxy)
|
||||
assert b"response.created" in await body.__anext__()
|
||||
await body.aclose()
|
||||
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 502
|
||||
|
||||
|
||||
@pytest.mark.parametrize("frames_read", [1, 2, 3, 4, 5])
|
||||
async def test_memory_filtered_hangup_before_the_client_gets_message_stop_is_a_failure(
|
||||
frames_read,
|
||||
):
|
||||
"""Frames 1-4 come from the one upstream chunk; frame 5 is the held message_delta."""
|
||||
proxy = _proxy(MESSAGE)
|
||||
body = await _stream(proxy, **MEMORY_STREAM)
|
||||
frames = [await body.__anext__() for _ in range(frames_read)]
|
||||
assert b"message_start" in frames[0]
|
||||
assert not any(b"message_stop" in frame for frame in frames)
|
||||
await body.aclose()
|
||||
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 502
|
||||
|
||||
|
||||
async def test_memory_filtered_hangup_after_message_stop_is_forwarded_records_success():
|
||||
proxy = _proxy(MESSAGE)
|
||||
body = await _stream(proxy, **MEMORY_STREAM)
|
||||
while b"message_stop" not in await body.__anext__():
|
||||
pass
|
||||
await body.aclose()
|
||||
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 200
|
||||
assert outcome.provider_input_tokens == 1_000
|
||||
assert outcome.output_tokens == 5
|
||||
|
||||
|
||||
async def test_malformed_event_after_a_valid_one_still_forwards_the_chunk():
|
||||
"""Invalid UTF-8 ends the stream with an error, after the upstream bytes."""
|
||||
valid = _event("response.output_text.delta", {"type": "response.output_text.delta"})
|
||||
malformed = b'event: response.output_text.delta\ndata: {"delta": "\xff"}\n\n'
|
||||
proxy = _proxy(START, valid + malformed)
|
||||
body = await _stream(proxy)
|
||||
frames = [frame async for frame in body]
|
||||
|
||||
assert frames[:2] == [START, valid + malformed]
|
||||
assert b"error" in frames[2]
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 502
|
||||
|
||||
Reference in New Issue
Block a user