mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): record a stream the client closes after its terminal event as completed
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1652,6 +1652,10 @@ class StreamingMixin:
|
||||
full_sse_bytes = bytearray()
|
||||
parsed_response = None # Set by memory block; used by CCR + prefix tracker
|
||||
completed_normally = False
|
||||
# Set when the client hangs up after the terminal event; it marks
|
||||
# the outcome only, so pending messages are not drained into a
|
||||
# stream nobody reads.
|
||||
client_left_after_terminal = False
|
||||
pending_messages: list[dict] = []
|
||||
# Proxy-injected memory tools are invisible to the client, so their
|
||||
# tool_use blocks must never reach it (GH #2195).
|
||||
@@ -1690,19 +1694,6 @@ class StreamingMixin:
|
||||
tail = bytes(stream_state["sse_buffer"][-MAX_SSE_BUFFER_SIZE // 2 :])
|
||||
stream_state["sse_buffer"] = bytearray(tail)
|
||||
|
||||
# Always stream immediately — buffering breaks
|
||||
# real-time clients (LangGraph, LangChain, etc.)
|
||||
# The context guard runs last, on exactly what the
|
||||
# client receives (after the memory filter), and may
|
||||
# hold back bytes only until the first complete event.
|
||||
client_frames = (
|
||||
[chunk] if memory_filter is None else memory_filter.feed(chunk)
|
||||
)
|
||||
for frame in client_frames:
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
|
||||
if _codex_wire_debug:
|
||||
capture_codex_wire_debug(
|
||||
"http_stream_upstream_chunk",
|
||||
@@ -1759,6 +1750,22 @@ class StreamingMixin:
|
||||
"cache_creation_ephemeral_1h_input_tokens"
|
||||
]
|
||||
|
||||
# Always stream immediately — buffering breaks
|
||||
# real-time clients (LangGraph, LangChain, etc.)
|
||||
# The context guard runs last, on exactly what the
|
||||
# client receives (after the memory filter), and may
|
||||
# hold back bytes only until the first complete event.
|
||||
# Yielded only after the chunk is parsed: a client may
|
||||
# hang up on the terminal event (Codex does), closing
|
||||
# this generator at the yield.
|
||||
client_frames = (
|
||||
[chunk] if memory_filter is None else memory_filter.feed(chunk)
|
||||
)
|
||||
for frame in client_frames:
|
||||
guarded_frame = _guard_client_bytes(frame)
|
||||
if guarded_frame:
|
||||
yield guarded_frame
|
||||
|
||||
# Memory tool handling after stream completes
|
||||
# Chunks were already yielded in real-time above, so we only
|
||||
# do silent background processing here — no yielding.
|
||||
@@ -1900,6 +1907,13 @@ class StreamingMixin:
|
||||
stream_state["stream_terminal"] and not stream_state["stream_failed"]
|
||||
)
|
||||
|
||||
except (GeneratorExit, asyncio.CancelledError):
|
||||
# Codex closes the connection as soon as it reads
|
||||
# response.completed, before the upstream EOF ends the loop.
|
||||
client_left_after_terminal = (
|
||||
stream_state["stream_terminal"] and not stream_state["stream_failed"]
|
||||
)
|
||||
raise
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
|
||||
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
|
||||
guarded_tail = _drain_context_guard()
|
||||
@@ -1985,7 +1999,8 @@ class StreamingMixin:
|
||||
client=client,
|
||||
waste_signals=waste_signals,
|
||||
status_code=_stream_outcome_status(
|
||||
upstream_response.status_code, completed_normally
|
||||
upstream_response.status_code,
|
||||
completed_normally or client_left_after_terminal,
|
||||
),
|
||||
conversation_key=conversation_key,
|
||||
conversation_tokens_saved=conversation_tokens_saved,
|
||||
@@ -2143,7 +2158,6 @@ class StreamingMixin:
|
||||
chunk_bytes = sse_line.encode()
|
||||
if prefix_tracker is not None:
|
||||
full_sse_bytes.extend(chunk_bytes)
|
||||
yield chunk_bytes
|
||||
|
||||
# Track usage from message_start event
|
||||
if event.event_type == "message_start":
|
||||
@@ -2188,9 +2202,15 @@ class StreamingMixin:
|
||||
stream_terminal = stream_terminal or terminal
|
||||
if failed:
|
||||
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
|
||||
# After the bookkeeping: the client may hang up on the terminal event.
|
||||
yield chunk_bytes
|
||||
|
||||
completed_normally = stream_terminal and not stream_failed
|
||||
|
||||
except (GeneratorExit, asyncio.CancelledError):
|
||||
# A client that hung up after the terminal event got the whole stream.
|
||||
completed_normally = stream_terminal and not stream_failed
|
||||
raise
|
||||
except Exception as e:
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
|
||||
@@ -2410,6 +2430,13 @@ class StreamingMixin:
|
||||
completed_normally = (
|
||||
stream_state.get("stream_terminal", False) and not stream_failed
|
||||
)
|
||||
except (GeneratorExit, asyncio.CancelledError):
|
||||
# A client that hung up after the terminal event got the whole stream.
|
||||
stream_failed = stream_state.get("stream_failed", False)
|
||||
completed_normally = (
|
||||
stream_state.get("stream_terminal", False) and not stream_failed
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Backend streaming error: {e}")
|
||||
error_data = public_errors.openai_error_body(
|
||||
|
||||
@@ -405,6 +405,70 @@ def test_backend_clean_eof_requires_protocol_completion(provider, complete):
|
||||
assert outcomes[0].status_code == (200 if complete else 502)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
|
||||
def test_backend_client_hangup_after_terminal_event_is_completed(provider):
|
||||
"""A client may close on the terminal event, before the backend's EOF."""
|
||||
from types import MethodType
|
||||
|
||||
from headroom.proxy.handlers.streaming import StreamingMixin
|
||||
|
||||
class Backend(_Backend):
|
||||
async def stream_message(self, body, headers):
|
||||
yield SimpleNamespace(
|
||||
event_type="message_stop",
|
||||
raw_sse='event: message_stop\ndata: {"type":"message_stop"}\n\n',
|
||||
data={"type": "message_stop"},
|
||||
)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def stream_openai_message(self, body, headers):
|
||||
yield "data: [DONE]\n\n"
|
||||
await asyncio.Event().wait()
|
||||
|
||||
outcomes = []
|
||||
|
||||
async def record(outcome):
|
||||
outcomes.append(outcome)
|
||||
|
||||
handler = _handler(Backend(provider))
|
||||
handler._record_request_outcome = record
|
||||
handler._extract_anthropic_cache_ttl_metrics = lambda usage: (0, 0)
|
||||
handler._parse_sse_usage_from_buffer = MethodType(
|
||||
StreamingMixin._parse_sse_usage_from_buffer, handler
|
||||
)
|
||||
|
||||
async def drive():
|
||||
arguments = {
|
||||
"body": {"messages": []},
|
||||
"headers": {},
|
||||
"model": "m",
|
||||
"request_id": "rid",
|
||||
"original_tokens": 0,
|
||||
"optimized_tokens": 0,
|
||||
"tokens_saved": 0,
|
||||
"transforms_applied": [],
|
||||
"tags": {},
|
||||
"optimization_latency": 0.0,
|
||||
"backend": handler.anthropic_backend,
|
||||
}
|
||||
if provider == "anthropic":
|
||||
response = await StreamingMixin._stream_response_bedrock(
|
||||
handler, provider=provider, **arguments
|
||||
)
|
||||
else:
|
||||
response = await StreamingMixin._stream_openai_via_backend(
|
||||
handler, start_time=0.0, **arguments
|
||||
)
|
||||
body = response.body_iterator
|
||||
while b"message_stop" not in (chunk := await body.__anext__()) and b"[DONE]" not in chunk:
|
||||
pass
|
||||
await body.aclose()
|
||||
|
||||
asyncio.run(drive())
|
||||
assert len(outcomes) == 1
|
||||
assert outcomes[0].status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "openai"])
|
||||
@pytest.mark.parametrize("complete", [False, True])
|
||||
def test_backend_prefix_tracker_advances_only_after_completion(provider, complete):
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""A client that hangs up after the terminal event still got a complete stream.
|
||||
|
||||
Codex CLI closes the connection as soon as it reads ``response.completed``,
|
||||
before the upstream EOF ends the proxy's read loop. The outcome was decided
|
||||
only after that loop, so every such turn recorded as a 502 failure with no
|
||||
usage and no cache metrics, although the client got a complete 200 response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from headroom.proxy.server import HeadroomProxy
|
||||
|
||||
|
||||
def _event(name: str, payload: dict[str, Any]) -> bytes:
|
||||
return f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
START = _event("response.created", {"type": "response.created", "response": {"id": "r1"}})
|
||||
COMPLETED = _event(
|
||||
"response.completed",
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "r1",
|
||||
"status": "completed",
|
||||
"usage": {
|
||||
"input_tokens": 1_000,
|
||||
"input_tokens_details": {"cached_tokens": 600, "cache_write_tokens": 0},
|
||||
"output_tokens": 5,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _proxy(upstream_eof: asyncio.Event) -> HeadroomProxy:
|
||||
proxy = object.__new__(HeadroomProxy)
|
||||
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
|
||||
proxy._config = MagicMock()
|
||||
proxy._config.ccr_inject_tool = False
|
||||
proxy._config.retry_max_attempts = 1
|
||||
proxy.config = proxy._config
|
||||
proxy._record_request_outcome = AsyncMock()
|
||||
|
||||
response = MagicMock()
|
||||
response.headers = httpx.Headers({"content-type": "text/event-stream"})
|
||||
response.status_code = 200
|
||||
|
||||
async def aiter_bytes():
|
||||
yield START
|
||||
yield COMPLETED
|
||||
await upstream_eof.wait() # the upstream EOF the client never waits for
|
||||
|
||||
response.aiter_bytes = aiter_bytes
|
||||
response.aclose = AsyncMock()
|
||||
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
|
||||
proxy.http_client.send = AsyncMock(return_value=response)
|
||||
return proxy
|
||||
|
||||
|
||||
async def _stream(proxy: HeadroomProxy):
|
||||
response = await proxy._stream_response(
|
||||
url="https://chatgpt.com/backend-api/codex/responses",
|
||||
headers={"authorization": "Bearer test"},
|
||||
body={"model": "gpt-5.4", "stream": True, "input": "Reply with: ok"},
|
||||
provider="openai",
|
||||
model="gpt-5.4",
|
||||
request_id="test-hangup",
|
||||
original_tokens=10,
|
||||
optimized_tokens=10,
|
||||
tokens_saved=0,
|
||||
transforms_applied=[],
|
||||
tags={},
|
||||
optimization_latency=0.0,
|
||||
)
|
||||
return response.body_iterator
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hangup", ["closed_at_terminal_yield", "cancelled_awaiting_eof"])
|
||||
async def test_client_hangup_after_terminal_event_records_success_with_usage(hangup):
|
||||
proxy = _proxy(asyncio.Event())
|
||||
body = await _stream(proxy)
|
||||
seen = b""
|
||||
if hangup == "closed_at_terminal_yield":
|
||||
# The generator is suspended at the yield of the terminal chunk.
|
||||
while b"response.completed" not in seen:
|
||||
seen += await body.__anext__()
|
||||
await body.aclose()
|
||||
else:
|
||||
# The generator read past the terminal chunk and waits for EOF.
|
||||
async def read_all() -> None:
|
||||
nonlocal seen
|
||||
async for chunk in body:
|
||||
seen += chunk
|
||||
|
||||
reader = asyncio.create_task(read_all())
|
||||
while b"response.completed" not in seen:
|
||||
await asyncio.sleep(0)
|
||||
reader.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await reader
|
||||
|
||||
proxy._record_request_outcome.assert_awaited_once()
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 200
|
||||
assert outcome.provider_input_tokens == 1_000
|
||||
assert outcome.cache_read_tokens == 600
|
||||
assert outcome.output_tokens == 5
|
||||
|
||||
|
||||
async def test_client_hangup_before_terminal_event_is_still_a_failure():
|
||||
proxy = _proxy(asyncio.Event())
|
||||
body = await _stream(proxy)
|
||||
assert b"response.created" in await body.__anext__()
|
||||
await body.aclose()
|
||||
|
||||
outcome = proxy._record_request_outcome.await_args.args[0]
|
||||
assert outcome.status_code == 502
|
||||
Reference in New Issue
Block a user