fix(proxy): count a hang-up as completed only after the client got the terminal event

The hang-up handler read stream_state["stream_terminal"], which the usage
parser sets from the whole upstream chunk before any of it is forwarded.
With the memory tool filter on, one chunk holding a whole message sets it
before the client gets its first frame, and the filter holds message_delta
and message_stop until after the stream. A client that closed at the first
frame was recorded as a 200.

Track the terminal event at the client-facing yield instead. Unfiltered,
each yielded chunk carries every event parsed so far, so the parser's flag
is reused. Filtered frames are whole events, so they are classified as they
go out, including the closing and continuation frames.

Moving the parse ahead of the yield also meant a chunk with a valid event
followed by invalid UTF-8 was dropped and replaced by the error event. A
parse failure is now raised after the chunk is forwarded, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-19 23:31:48 +02:00
co-authored by Claude Opus 5.5
parent 4eeb461999
commit ef0773a706
2 changed files with 158 additions and 28 deletions
+36 -7
View File
@@ -20,6 +20,7 @@ from headroom.proxy.helpers import (
RETRYABLE_OVERLOAD_STATUSES,
jitter_delay_ms,
overload_retry_is_futile,
parse_sse_events_from_byte_buffer,
retry_after_ms,
)
from headroom.proxy.provider_usage import (
@@ -1656,6 +1657,9 @@ class StreamingMixin:
# the outcome only, so pending messages are not drained into a
# stream nobody reads.
client_left_after_terminal = False
# A terminal event the client was handed, not just one parsed from
# upstream: the memory filter holds message_stop until the end.
client_got_terminal = False
pending_messages: list[dict] = []
# Proxy-injected memory tools are invisible to the client, so their
# tool_use blocks must never reach it (GH #2195).
@@ -1665,6 +1669,23 @@ class StreamingMixin:
else None
)
def _client_frame(frame: bytes) -> bytes:
"""``_guard_client_bytes``, noting when the terminal event goes out."""
nonlocal client_got_terminal
guarded = _guard_client_bytes(frame)
if guarded and not client_got_terminal:
if memory_filter is None:
# Unfiltered, the client gets each whole chunk once it
# is parsed, so every event the parser has seen.
client_got_terminal = stream_state["stream_terminal"]
else:
# Filtered frames are complete events the filter decoded.
client_got_terminal = any(
_sse_event_outcome(name, data, stream_state["sse_dialect"])[1]
for name, data in parse_sse_events_from_byte_buffer(bytearray(frame))
)
return guarded
try:
async with contextlib.aclosing(upstream_response) as response:
sse_chunk_index = 0
@@ -1726,8 +1747,14 @@ class StreamingMixin:
)
memory_enabled = False
# Parse complete SSE events from buffer
usage = self._parse_sse_usage_from_buffer(stream_state, provider)
# Parse complete SSE events from buffer. A failure (e.g.
# invalid UTF-8) is raised once the chunk is forwarded:
# malformed upstream content still reaches the client.
parse_error: Exception | None = None
try:
usage = self._parse_sse_usage_from_buffer(stream_state, provider)
except Exception as e:
usage, parse_error = None, e
if usage:
if "input_tokens" in usage:
stream_state["input_tokens"] = usage["input_tokens"]
@@ -1762,9 +1789,11 @@ class StreamingMixin:
[chunk] if memory_filter is None else memory_filter.feed(chunk)
)
for frame in client_frames:
guarded_frame = _guard_client_bytes(frame)
guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
if parse_error is not None:
raise parse_error
# Memory tool handling after stream completes
# Chunks were already yielded in real-time above, so we only
@@ -1787,7 +1816,7 @@ class StreamingMixin:
)
if memory_filter is not None:
for frame in memory_filter.closing_frames():
guarded_frame = _guard_client_bytes(frame)
guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
guarded_tail = _drain_context_guard()
@@ -1811,7 +1840,7 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
guarded_frame = _guard_client_bytes(frame)
guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
memory_filter = None
@@ -1856,7 +1885,7 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
guarded_frame = _guard_client_bytes(frame)
guarded_frame = _client_frame(frame)
if guarded_frame:
yield guarded_frame
@@ -1911,7 +1940,7 @@ class StreamingMixin:
# Codex closes the connection as soon as it reads
# response.completed, before the upstream EOF ends the loop.
client_left_after_terminal = (
stream_state["stream_terminal"] and not stream_state["stream_failed"]
client_got_terminal and not stream_state["stream_failed"]
)
raise
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
+122 -21
View File
@@ -4,6 +4,8 @@ Codex CLI closes the connection as soon as it reads ``response.completed``,
before the upstream EOF ends the proxy's read loop. The outcome was decided
only after that loop, so every such turn recorded as a 502 failure with no
usage and no cache metrics, although the client got a complete 200 response.
Success needs the terminal event the client was handed, not one only parsed
from upstream: the memory filter holds ``message_stop`` until after the stream.
"""
from __future__ import annotations
@@ -40,8 +42,60 @@ COMPLETED = _event(
},
)
# A whole Anthropic message in one upstream chunk: the parser sees message_stop
# before the client gets the first frame the memory filter splits it into.
MESSAGE = b"".join(
[
_event(
"message_start",
{
"type": "message_start",
"message": {
"id": "m1",
"type": "message",
"role": "assistant",
"model": "claude-test",
"content": [],
"usage": {"input_tokens": 1_000, "output_tokens": 1},
},
},
),
_event(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text"}},
),
_event(
"content_block_delta",
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "ok"},
},
),
_event("content_block_stop", {"type": "content_block_stop", "index": 0}),
_event(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 5},
},
),
_event("message_stop", {"type": "message_stop"}),
]
)
MEMORY_STREAM = {
"url": "https://api.anthropic.com/v1/messages",
"headers": {"x-api-key": "sk-test"},
"body": {"model": "claude-test", "max_tokens": 100, "stream": True, "messages": []},
"provider": "anthropic",
"model": "claude-test",
"memory_user_id": "user-1",
"server_memory_tool_names": frozenset({"memory_save"}),
}
def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
def _proxy(*chunks: bytes, upstream_eof: asyncio.Event | None = None) -> HeadroomProxy:
proxy = object.__new__(HeadroomProxy)
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
proxy._config = MagicMock()
@@ -55,9 +109,10 @@ def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
response.status_code = 200
async def aiter_bytes():
yield START
yield COMPLETED
await upstream_eof.wait() # the upstream EOF the client never waits for
for chunk in chunks or (START, COMPLETED):
yield chunk
if upstream_eof is not None:
await upstream_eof.wait() # the upstream EOF the client never waits for
response.aiter_bytes = aiter_bytes
response.aclose = AsyncMock()
@@ -66,27 +121,30 @@ def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
return proxy
async def _stream(proxy: HeadroomProxy):
response = await proxy._stream_response(
url="https://chatgpt.com/backend-api/codex/responses",
headers={"authorization": "Bearer test"},
body={"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
provider="openai",
model="gpt-5.4",
request_id="test-hangup",
original_tokens=10,
optimized_tokens=10,
tokens_saved=0,
transforms_applied=[],
tags={},
optimization_latency=0.0,
)
async def _stream(proxy: HeadroomProxy, **overrides: Any):
arguments: dict[str, Any] = {
"url": "https://chatgpt.com/backend-api/codex/responses",
"headers": {"authorization": "Bearer test"},
"body": {"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
"provider": "openai",
"model": "gpt-5.4",
"request_id": "test-hangup",
"original_tokens": 10,
"optimized_tokens": 10,
"tokens_saved": 0,
"transforms_applied": [],
"tags": {},
"optimization_latency": 0.0,
}
if overrides.get("memory_user_id"):
proxy.memory_handler = MagicMock()
response = await proxy._stream_response(**{**arguments, **overrides})
return response.body_iterator
@pytest.mark.parametrize("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])
async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):
proxy = _proxy(asyncio.Event())
proxy = _proxy(upstream_eof=asyncio.Event())
body = await _stream(proxy)
seen = b""
if hangup == "closed_at_terminal_yield":
@@ -117,10 +175,53 @@ async def test_client_hangup_after_terminal_event_records_success_with_usage(han
async def test_client_hangup_before_terminal_event_is_still_a_failure():
proxy = _proxy(asyncio.Event())
proxy = _proxy(upstream_eof=asyncio.Event())
body = await _stream(proxy)
assert b"response.created" in await body.__anext__()
await body.aclose()
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 502
@pytest.mark.parametrize("frames_read", [1, 2, 3, 4, 5])
async def test_memory_filtered_hangup_before_the_client_gets_message_stop_is_a_failure(
frames_read,
):
"""Frames 1-4 come from the one upstream chunk; frame 5 is the held message_delta."""
proxy = _proxy(MESSAGE)
body = await _stream(proxy, **MEMORY_STREAM)
frames = [await body.__anext__() for _ in range(frames_read)]
assert b"message_start" in frames[0]
assert not any(b"message_stop" in frame for frame in frames)
await body.aclose()
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 502
async def test_memory_filtered_hangup_after_message_stop_is_forwarded_records_success():
proxy = _proxy(MESSAGE)
body = await _stream(proxy, **MEMORY_STREAM)
while b"message_stop" not in await body.__anext__():
pass
await body.aclose()
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 200
assert outcome.provider_input_tokens == 1_000
assert outcome.output_tokens == 5
async def test_malformed_event_after_a_valid_one_still_forwards_the_chunk():
"""Invalid UTF-8 ends the stream with an error, after the upstream bytes."""
valid = _event("response.output_text.delta", {"type": "response.output_text.delta"})
malformed = b'event: response.output_text.delta\ndata: {"delta": "\xff"}\n\n'
proxy = _proxy(START, valid + malformed)
body = await _stream(proxy)
frames = [frame async for frame in body]
assert frames[:2] == [START, valid + malformed]
assert b"error" in frames[2]
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 502