mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): mark a passthrough stream cut mid-character as failed
The recovered usage is still recorded, but the outcome now carries the shared stream-failure status (502) instead of the default 200, so the funnel books it as failed rather than as a completed success. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
parent
7642a42539
commit
a76fd81121
@@ -113,6 +113,7 @@ from headroom.proxy.buffered_ccr_response import (
|
||||
from headroom.proxy.compression_decision import CompressionDecision
|
||||
from headroom.proxy.cost import header_safe_transforms
|
||||
from headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value
|
||||
from headroom.proxy.handlers.streaming import _stream_outcome_status
|
||||
from headroom.proxy.image_isolation import run_image_compression_isolated
|
||||
from headroom.proxy.outcome import RequestOutcome
|
||||
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
|
||||
@@ -11709,16 +11710,20 @@ class OpenAIHandlerMixin:
|
||||
)
|
||||
yield chunk
|
||||
finally:
|
||||
completed_normally = True
|
||||
buf = stream_state["sse_buffer"]
|
||||
if len(buf) > 0:
|
||||
buf.extend(b"\n\n")
|
||||
# A truncated stream can end inside a multi-byte character;
|
||||
# the strict decoder then raises and would skip the outcome
|
||||
# below. Drop that unusable tail instead.
|
||||
with contextlib.suppress(UnicodeDecodeError):
|
||||
# below. Keep the usage already parsed, drop the unusable
|
||||
# tail, and report the stream as failed.
|
||||
try:
|
||||
_absorb_usage(
|
||||
self._parse_sse_usage_from_buffer(stream_state, stream_provider)
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
completed_normally = False
|
||||
|
||||
output_tokens = stream_state["output_tokens"] or 0
|
||||
cache_read_tokens = stream_state["cache_read_input_tokens"] or 0
|
||||
@@ -11759,6 +11764,9 @@ class OpenAIHandlerMixin:
|
||||
total_latency_ms=(time.time() - start_time) * 1000,
|
||||
ttfb_ms=stream_state["ttfb_ms"] or 0,
|
||||
tags=tags,
|
||||
status_code=_stream_outcome_status(
|
||||
upstream_response.status_code, completed_normally
|
||||
),
|
||||
client=client,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from headroom.proxy.helpers import (
|
||||
_headroom_bypass_enabled,
|
||||
relocate_system_messages_to_top_level,
|
||||
)
|
||||
from headroom.proxy.outcome import emit_request_outcome
|
||||
from headroom.proxy.server import HeadroomProxy
|
||||
|
||||
|
||||
@@ -850,13 +851,19 @@ def test_vertex_stream_passthrough_preserves_chunks_and_records_usage() -> None:
|
||||
|
||||
|
||||
def test_vertex_stream_passthrough_records_outcome_when_stream_ends_mid_character() -> None:
|
||||
closed: list[bool] = []
|
||||
|
||||
class _CountingChunks(_AsyncChunks):
|
||||
async def aclose(self) -> None:
|
||||
closed.append(True)
|
||||
|
||||
class _TruncatedClient(_VertexStreamClient):
|
||||
async def send(self, request, stream=False): # noqa: ANN001, ANN201
|
||||
return httpx.Response(
|
||||
200,
|
||||
request=request,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_AsyncChunks(
|
||||
stream=_CountingChunks(
|
||||
[
|
||||
b'data: {"usageMetadata":{"promptTokenCount":13,'
|
||||
b'"candidatesTokenCount":5}}\n\n',
|
||||
@@ -892,8 +899,27 @@ def test_vertex_stream_passthrough_records_outcome_when_stream_ends_mid_characte
|
||||
|
||||
assert len(asyncio.run(collect())) == 2
|
||||
assert len(outcomes) == 1
|
||||
assert closed == [True]
|
||||
# Usage already received is kept, but a stream cut mid-character did not
|
||||
# complete: it takes the same failure status as other post-start stream
|
||||
# failures, so the outcome funnel books it as failed, not as a success.
|
||||
assert outcomes[0].optimized_tokens == 13
|
||||
assert outcomes[0].output_tokens == 5
|
||||
assert outcomes[0].status_code == 502
|
||||
|
||||
failed: list[str] = []
|
||||
requested: list[dict] = []
|
||||
|
||||
class _Metrics:
|
||||
async def record_failed(self, provider): # noqa: ANN001, ANN202
|
||||
failed.append(provider)
|
||||
|
||||
async def record_request(self, **kwargs): # noqa: ANN003, ANN202
|
||||
requested.append(kwargs)
|
||||
|
||||
asyncio.run(emit_request_outcome(SimpleNamespace(metrics=_Metrics()), outcomes[0]))
|
||||
assert failed == [outcomes[0].provider]
|
||||
assert requested == []
|
||||
|
||||
|
||||
def test_stream_finalizer_records_vertex_provider_for_dashboard() -> None:
|
||||
|
||||
Reference in New Issue
Block a user