fix(proxy): record a stream the client closes after its terminal event as completed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-19 15:03:27 +02:00
co-authored by Claude Opus 5.5
parent fbf7bc3278
commit 4eeb461999
3 changed files with 232 additions and 15 deletions
+42 -15
View File
@@ -1652,6 +1652,10 @@ class StreamingMixin:
full_sse_bytes = bytearray()
parsed_response = None # Set by memory block; used by CCR + prefix tracker
completed_normally = False
# Set when the client hangs up after the terminal event; it marks
# the outcome only, so pending messages are not drained into a
# stream nobody reads.
client_left_after_terminal = False
pending_messages: list[dict] = []
# Proxy-injected memory tools are invisible to the client, so their
# tool_use blocks must never reach it (GH #2195).
@@ -1690,19 +1694,6 @@ class StreamingMixin:
tail = bytes(stream_state["sse_buffer"][-MAX_SSE_BUFFER_SIZE // 2 :])
stream_state["sse_buffer"] = bytearray(tail)
# Always stream immediately — buffering breaks
# real-time clients (LangGraph, LangChain, etc.)
# The context guard runs last, on exactly what the
# client receives (after the memory filter), and may
# hold back bytes only until the first complete event.
client_frames = (
[chunk] if memory_filter is None else memory_filter.feed(chunk)
)
for frame in client_frames:
guarded_frame = _guard_client_bytes(frame)
if guarded_frame:
yield guarded_frame
if _codex_wire_debug:
capture_codex_wire_debug(
"http_stream_upstream_chunk",
@@ -1759,6 +1750,22 @@ class StreamingMixin:
"cache_creation_ephemeral_1h_input_tokens"
]
# Always stream immediately — buffering breaks
# real-time clients (LangGraph, LangChain, etc.)
# The context guard runs last, on exactly what the
# client receives (after the memory filter), and may
# hold back bytes only until the first complete event.
# Yielded only after the chunk is parsed: a client may
# hang up on the terminal event (Codex does), closing
# this generator at the yield.
client_frames = (
[chunk] if memory_filter is None else memory_filter.feed(chunk)
)
for frame in client_frames:
guarded_frame = _guard_client_bytes(frame)
if guarded_frame:
yield guarded_frame
# Memory tool handling after stream completes
# Chunks were already yielded in real-time above, so we only
# do silent background processing here — no yielding.
@@ -1900,6 +1907,13 @@ class StreamingMixin:
stream_state["stream_terminal"] and not stream_state["stream_failed"]
)
except (GeneratorExit, asyncio.CancelledError):
# Codex closes the connection as soon as it reads
# response.completed, before the upstream EOF ends the loop.
client_left_after_terminal = (
stream_state["stream_terminal"] and not stream_state["stream_failed"]
)
raise
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
guarded_tail = _drain_context_guard()
@@ -1985,7 +1999,8 @@ class StreamingMixin:
client=client,
waste_signals=waste_signals,
status_code=_stream_outcome_status(
upstream_response.status_code, completed_normally
upstream_response.status_code,
completed_normally or client_left_after_terminal,
),
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
@@ -2143,7 +2158,6 @@ class StreamingMixin:
chunk_bytes = sse_line.encode()
if prefix_tracker is not None:
full_sse_bytes.extend(chunk_bytes)
yield chunk_bytes
# Track usage from message_start event
if event.event_type == "message_start":
@@ -2188,9 +2202,15 @@ class StreamingMixin:
stream_terminal = stream_terminal or terminal
if failed:
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
# After the bookkeeping: the client may hang up on the terminal event.
yield chunk_bytes
completed_normally = stream_terminal and not stream_failed
except (GeneratorExit, asyncio.CancelledError):
# A client that hung up after the terminal event got the whole stream.
completed_normally = stream_terminal and not stream_failed
raise
except Exception as e:
error_message = format_exception_message(e)
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
@@ -2410,6 +2430,13 @@ class StreamingMixin:
completed_normally = (
stream_state.get("stream_terminal", False) and not stream_failed
)
except (GeneratorExit, asyncio.CancelledError):
# A client that hung up after the terminal event got the whole stream.
stream_failed = stream_state.get("stream_failed", False)
completed_normally = (
stream_state.get("stream_terminal", False) and not stream_failed
)
raise
except Exception as e:
logger.error(f"[{request_id}] Backend streaming error: {e}")
error_data = public_errors.openai_error_body(
+64
View File
@@ -405,6 +405,70 @@ def test_backend_clean_eof_requires_protocol_completion(provider, complete):
assert outcomes[0].status_code == (200 if complete else 502)
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
def test_backend_client_hangup_after_terminal_event_is_completed(provider):
"""A client may close on the terminal event, before the backend's EOF."""
from types import MethodType
from headroom.proxy.handlers.streaming import StreamingMixin
class Backend(_Backend):
async def stream_message(self, body, headers):
yield SimpleNamespace(
event_type="message_stop",
raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
data={"type": "message_stop"},
)
await asyncio.Event().wait()
async def stream_openai_message(self, body, headers):
yield "data: [DONE]\n\n"
await asyncio.Event().wait()
outcomes = []
async def record(outcome):
outcomes.append(outcome)
handler = _handler(Backend(provider))
handler._record_request_outcome = record
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
handler._parse_sse_usage_from_buffer = MethodType(
StreamingMixin._parse_sse_usage_from_buffer, handler
)
async def drive():
arguments = {
"body": {"messages": []},
"headers": {},
"model": "m",
"request_id": "rid",
"original_tokens": 0,
"optimized_tokens": 0,
"tokens_saved": 0,
"transforms_applied": [],
"tags": {},
"optimization_latency": 0.0,
"backend": handler.anthropic_backend,
}
if provider == "anthropic":
response = await StreamingMixin._stream_response_bedrock(
handler, provider=provider, **arguments
)
else:
response = await StreamingMixin._stream_openai_via_backend(
handler, start_time=0.0, **arguments
)
body = response.body_iterator
while b"message_stop" not in (chunk := await body.__anext__()) and b"[DONE]" not in chunk:
pass
await body.aclose()
asyncio.run(drive())
assert len(outcomes) == 1
assert outcomes[0].status_code == 200
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
@pytest.mark.parametrize("complete", [False, True])
def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):
@@ -0,0 +1,126 @@
"""A client that hangs up after the terminal event still got a complete stream.
Codex CLI closes the connection as soon as it reads ``response.completed``,
before the upstream EOF ends the proxy's read loop. The outcome was decided
only after that loop, so every such turn recorded as a 502 failure with no
usage and no cache metrics, although the client got a complete 200 response.
"""
from __future__ import annotations
import asyncio
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from headroom.proxy.server import HeadroomProxy
def _event(name: str, payload: dict[str, Any]) -> bytes:
return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode()
START = _event("response.created", {"type": "response.created", "response": {"id": "r1"}})
COMPLETED = _event(
"response.completed",
{
"type": "response.completed",
"response": {
"id": "r1",
"status": "completed",
"usage": {
"input_tokens": 1_000,
"input_tokens_details": {"cached_tokens": 600, "cache_write_tokens": 0},
"output_tokens": 5,
},
},
},
)
def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
proxy = object.__new__(HeadroomProxy)
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
proxy._config = MagicMock()
proxy._config.ccr_inject_tool = False
proxy._config.retry_max_attempts = 1
proxy.config = proxy._config
proxy._record_request_outcome = AsyncMock()
response = MagicMock()
response.headers = httpx.Headers({"content-type": "text/event-stream"})
response.status_code = 200
async def aiter_bytes():
yield START
yield COMPLETED
await upstream_eof.wait() # the upstream EOF the client never waits for
response.aiter_bytes = aiter_bytes
response.aclose = AsyncMock()
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
proxy.http_client.send = AsyncMock(return_value=response)
return proxy
async def _stream(proxy: HeadroomProxy):
response = await proxy._stream_response(
url="https://chatgpt.com/backend-api/codex/responses",
headers={"authorization": "Bearer test"},
body={"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
provider="openai",
model="gpt-5.4",
request_id="test-hangup",
original_tokens=10,
optimized_tokens=10,
tokens_saved=0,
transforms_applied=[],
tags={},
optimization_latency=0.0,
)
return response.body_iterator
@pytest.mark.parametrize("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])
async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):
proxy = _proxy(asyncio.Event())
body = await _stream(proxy)
seen = b""
if hangup == "closed_at_terminal_yield":
# The generator is suspended at the yield of the terminal chunk.
while b"response.completed" not in seen:
seen += await body.__anext__()
await body.aclose()
else:
# The generator read past the terminal chunk and waits for EOF.
async def read_all() -> None:
nonlocal seen
async for chunk in body:
seen += chunk
reader = asyncio.create_task(read_all())
while b"response.completed" not in seen:
await asyncio.sleep(0)
reader.cancel()
with pytest.raises(asyncio.CancelledError):
await reader
proxy._record_request_outcome.assert_awaited_once()
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 200
assert outcome.provider_input_tokens == 1_000
assert outcome.cache_read_tokens == 600
assert outcome.output_tokens == 5
async def test_client_hangup_before_terminal_event_is_still_a_failure():
proxy = _proxy(asyncio.Event())
body = await _stream(proxy)
assert b"response.created" in await body.__anext__()
await body.aclose()
outcome = proxy._record_request_outcome.await_args.args[0]
assert outcome.status_code == 502