mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
## Description Closes #3405. When an SSE stream is interrupted in the middle of an event or UTF-8 character, the streaming finalizer appended `\n\n` to the residual byte buffer. That synthetic terminator made the incomplete event look complete and sent it through the strict UTF-8 decoder, causing a secondary `UnicodeDecodeError` during cleanup. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (breaking change that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Stop synthesizing an SSE event terminator during Anthropic streaming finalization; discard the unterminated residual tail instead. - Apply the same rule to the OpenAI-via-backend streaming cleanup path. - Keep complete events parsed during normal chunk processing, while preventing partial SSE/UTF-8 data from being decoded as a complete event. - Replace the old truncated-buffer recovery test with a regression covering a truncated UTF-8 tail and asserting finalization completes without raising. ## Testing - [x] Focused unit tests pass - [x] Linting passes - [x] Formatting passes - [x] New regression coverage added - [ ] Full test suite - [ ] Live upstream streaming run ### Test Output ```text python -m pytest tests/test_proxy_streaming_request_logger.py -q 9 passed in 71.37s python -m pytest tests/test_sse_byte_buffer_policy.py tests/test_sse_utf8_split.py -q 8 passed in 10.78s python -m ruff check headroom/proxy/handlers/streaming.py tests/test_proxy_streaming_request_logger.py All checks passed! python -m ruff format --check headroom/proxy/handlers/streaming.py tests/test_proxy_streaming_request_logger.py 2 files already formatted ``` The focused proxy test used the published `headroom-ai==0.37.0` wheel only for its compatible Rust extension; the source changes and tests ran from this checkout. ## Real Behavior Proof - Environment: Windows 10, Python 3.11.9, pytest 9.1.1. - Exact command / steps: ran the new regression with the production finalizer reverted; it failed with `UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe5 ... unexpected end of data`. Restored the fix and ran the regression plus the existing SSE parser tests. - Observed result: the old implementation fails when finalization appends a terminator to an unterminated event ending in a partial UTF-8 byte; the fixed implementation clears the incomplete tail without raising, and the focused proxy suite passes 9 tests. - Not tested: a live upstream provider disconnect; the regression exercises the exact residual-buffer/finalizer path with a partial UTF-8 sequence. ## Runtime Rollout Safety - Rollout-managed feature(s): None; this is an internal parser/finalization bug fix with no feature flag or staged rollout. - Minimum rollout channel: Standard release containing this bug fix; no separate rollout channel is required. - Stable/default behavior changed: No; complete SSE events remain parsed during normal streaming, while only unterminated residual tails are discarded during finalization. - Kill switch / disable path: Not applicable; revert this PR or the release containing it. - Unsafe override required: No. - Qualification impact: None; there are no API or configuration changes, and focused regression coverage was added. - Rollback path: Revert this change; the prior behavior may reintroduce a secondary UnicodeDecodeError on interrupted partial UTF-8 SSE tails. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective - [x] Focused unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` ## Additional Notes Complete SSE events continue to be parsed during streaming. Only an unterminated residual tail is discarded at finalization, so interrupted cleanup cannot turn incomplete wire data into a parser exception. --------- Signed-off-by: UniversePeak <UniversePeak@users.noreply.github.com> Co-authored-by: UniversePeak <UniversePeak@users.noreply.github.com>
342 lines
12 KiB
Python
342 lines
12 KiB
Python
"""Tests that the Anthropic streaming finalizer logs requests for the feed.
|
|
|
|
Without this, the streaming Anthropic path (which is what Claude Code uses)
|
|
silently bypassed the request logger, leaving `/stats.recent_requests` and
|
|
`/transformations/feed` permanently empty even when `--log-messages` was set.
|
|
The non-streaming Anthropic path and the Bedrock streaming path were the
|
|
only ones that called `self.logger.log(...)`.
|
|
"""
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from headroom.proxy.request_logger import RequestLogger
|
|
from headroom.proxy.server import HeadroomProxy
|
|
|
|
|
|
def _build_proxy_with_real_logger(*, log_full_messages: bool) -> HeadroomProxy:
|
|
"""Build a HeadroomProxy with mocks for everything except the request logger,
|
|
so we can assert what actually gets recorded."""
|
|
proxy = object.__new__(HeadroomProxy)
|
|
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
|
|
proxy.metrics = MagicMock()
|
|
proxy.metrics.record_request = AsyncMock(return_value=None)
|
|
proxy.cost_tracker = MagicMock()
|
|
proxy.cost_tracker.record_tokens.return_value = None
|
|
proxy.memory_manager = None
|
|
proxy.memory_handler = None
|
|
proxy._config = MagicMock()
|
|
proxy._config.log_full_messages = log_full_messages
|
|
proxy._config.ccr_inject_tool = False
|
|
proxy.config = proxy._config
|
|
proxy.logger = RequestLogger(log_file=None, log_full_messages=log_full_messages)
|
|
return proxy
|
|
|
|
|
|
def _stream_state(output_tokens: int = 42) -> dict:
|
|
return {
|
|
"output_tokens": output_tokens,
|
|
"total_bytes": 200,
|
|
"ttfb_ms": 35.0,
|
|
"input_tokens": 1000,
|
|
"cache_read_input_tokens": 0,
|
|
"cache_creation_input_tokens": 0,
|
|
"cache_creation_ephemeral_5m_input_tokens": 0,
|
|
"cache_creation_ephemeral_1h_input_tokens": 0,
|
|
"sse_buffer": "",
|
|
}
|
|
|
|
|
|
def test_parse_openai_responses_completed_usage_from_sse_buffer():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
completed = {
|
|
"type": "response.completed",
|
|
"response": {
|
|
"id": "resp_1",
|
|
"usage": {
|
|
"input_tokens": 844_000,
|
|
"input_tokens_details": {"cached_tokens": 657_400},
|
|
"output_tokens": 6_635,
|
|
},
|
|
},
|
|
}
|
|
state = {
|
|
"sse_buffer": bytearray(
|
|
f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode()
|
|
)
|
|
}
|
|
|
|
usage = proxy._parse_sse_usage_from_buffer(state, "openai")
|
|
|
|
assert usage == {
|
|
"input_tokens": 844_000,
|
|
"output_tokens": 6_635,
|
|
"cache_read_input_tokens": 657_400,
|
|
}
|
|
assert state["sse_buffer"] == bytearray()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_logs_request_for_feed():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
request_tags = {"stack": "wrap_claude"}
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": [{"role": "user", "content": "hi"}]},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-1",
|
|
original_tokens=1000,
|
|
optimized_tokens=600,
|
|
tokens_saved=400,
|
|
transforms_applied=["smart_crusher"],
|
|
optimization_latency=12.0,
|
|
stream_state=_stream_state(),
|
|
start_time=0.0,
|
|
tags=request_tags,
|
|
)
|
|
|
|
entries = proxy.logger.get_recent(10)
|
|
assert len(entries) == 1, "streaming finalizer must log exactly one entry per request"
|
|
entry = entries[0]
|
|
assert entry["request_id"] == "req-stream-1"
|
|
assert entry["provider"] == "anthropic"
|
|
assert entry["model"] == "claude-sonnet-4-6"
|
|
assert entry["input_tokens_original"] == 1000
|
|
assert entry["input_tokens_optimized"] == 600
|
|
assert entry["tokens_saved"] == 400
|
|
assert entry["savings_percent"] == pytest.approx(40.0)
|
|
assert entry["transforms_applied"] == ["smart_crusher"]
|
|
assert entry["tags"] == {
|
|
"stack": "wrap_claude",
|
|
"output_tokens_source": "provider",
|
|
}
|
|
assert request_tags == {"stack": "wrap_claude"}
|
|
assert entry["cache_hit"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_marks_estimated_output_tokens() -> None:
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
state = _stream_state()
|
|
state["output_tokens"] = None
|
|
state["total_bytes"] = 200
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": [{"role": "user", "content": "hi"}]},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-estimated",
|
|
original_tokens=10,
|
|
optimized_tokens=10,
|
|
tokens_saved=0,
|
|
transforms_applied=[],
|
|
optimization_latency=1.0,
|
|
stream_state=state,
|
|
start_time=0.0,
|
|
)
|
|
|
|
entry = proxy.logger.get_recent(1)[0]
|
|
assert entry["output_tokens"] == 5
|
|
assert entry["tags"]["output_tokens_source"] == "estimated_bytes"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_logs_original_and_compressed_messages():
|
|
"""With log_full_messages enabled, both sides of the compression are
|
|
recorded: `request_messages` is the pre-compression snapshot the caller
|
|
threads in via `original_messages`, `compressed_messages` is what was
|
|
actually sent upstream (i.e. `body["messages"]` after in-place mutation)."""
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=True)
|
|
# `body["messages"]` models the post-compression list - the proxy mutates
|
|
# `body` in place before calling `_finalize_stream_response`, so this is
|
|
# already what was shipped over the wire.
|
|
body = {"messages": [{"role": "user", "content": "[compressed]"}]}
|
|
original = [{"role": "user", "content": "[original, pre-compression]"}]
|
|
|
|
await proxy._finalize_stream_response(
|
|
body=body,
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-2",
|
|
original_tokens=10,
|
|
optimized_tokens=8,
|
|
tokens_saved=2,
|
|
transforms_applied=[],
|
|
optimization_latency=1.0,
|
|
stream_state=_stream_state(output_tokens=5),
|
|
start_time=0.0,
|
|
original_messages=original,
|
|
)
|
|
|
|
entries = proxy.logger.get_recent_with_messages(10)
|
|
assert len(entries) == 1
|
|
assert entries[0]["request_messages"] == original
|
|
assert entries[0]["compressed_messages"] == body["messages"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_omits_messages_when_log_full_messages_disabled():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": [{"role": "user", "content": "hello"}]},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-3",
|
|
original_tokens=10,
|
|
optimized_tokens=8,
|
|
tokens_saved=2,
|
|
transforms_applied=[],
|
|
optimization_latency=1.0,
|
|
stream_state=_stream_state(output_tokens=5),
|
|
start_time=0.0,
|
|
original_messages=[{"role": "user", "content": "dropped"}],
|
|
)
|
|
|
|
entries = proxy.logger.get_recent_with_messages(10)
|
|
assert len(entries) == 1
|
|
# Both sides share the same gate - neither leaks when log_full_messages
|
|
# is off.
|
|
assert entries[0]["request_messages"] is None
|
|
assert entries[0]["compressed_messages"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_handles_zero_original_tokens():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": []},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-4",
|
|
original_tokens=0,
|
|
optimized_tokens=0,
|
|
tokens_saved=0,
|
|
transforms_applied=[],
|
|
optimization_latency=0.0,
|
|
stream_state=_stream_state(output_tokens=0),
|
|
start_time=0.0,
|
|
)
|
|
|
|
entries = proxy.logger.get_recent(10)
|
|
assert len(entries) == 1
|
|
assert entries[0]["savings_percent"] == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_openai_responses_stream_uses_provider_usage_for_dashboard():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
state = _stream_state(output_tokens=6_635)
|
|
state["input_tokens"] = 844_000
|
|
state["cache_read_input_tokens"] = 657_400
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"model": "gpt-5.5", "input": [{"type": "message", "role": "user"}]},
|
|
provider="openai",
|
|
model="gpt-5.5",
|
|
request_id="req-openai-responses-stream",
|
|
original_tokens=0,
|
|
optimized_tokens=0,
|
|
tokens_saved=663_000,
|
|
transforms_applied=["openai_responses_live_zone"],
|
|
optimization_latency=26.0,
|
|
stream_state=state,
|
|
start_time=0.0,
|
|
)
|
|
|
|
entries = proxy.logger.get_recent(10)
|
|
assert len(entries) == 1
|
|
entry = entries[0]
|
|
assert entry["input_tokens_optimized"] == 844_000
|
|
assert entry["input_tokens_original"] == 1_507_000
|
|
assert entry["tokens_saved"] == 663_000
|
|
assert entry["savings_percent"] == pytest.approx(663_000 / 1_507_000 * 100)
|
|
assert entry["output_tokens"] == 6_635
|
|
|
|
proxy.metrics.record_request.assert_awaited_once()
|
|
metrics_kwargs = proxy.metrics.record_request.await_args.kwargs
|
|
assert metrics_kwargs["input_tokens"] == 844_000
|
|
assert metrics_kwargs["output_tokens"] == 6_635
|
|
assert metrics_kwargs["tokens_saved"] == 663_000
|
|
assert metrics_kwargs["cache_read_tokens"] == 657_400
|
|
assert metrics_kwargs["uncached_input_tokens"] == 186_600
|
|
|
|
proxy.cost_tracker.record_tokens.assert_called_once()
|
|
cost_args, cost_kwargs = proxy.cost_tracker.record_tokens.call_args
|
|
assert cost_args[:3] == ("gpt-5.5", 663_000, 844_000)
|
|
assert cost_kwargs["cache_read_tokens"] == 657_400
|
|
assert cost_kwargs["uncached_tokens"] == 186_600
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_discards_truncated_utf8_tail() -> None:
|
|
"""An interrupted SSE event must not be made complete by finalization."""
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
|
|
partial_message_start = (
|
|
b"event: message_start\n"
|
|
b'data: {"type":"message_start","message":{"id":"msg_x",'
|
|
b'"type":"message","role":"assistant","model":"claude-sonnet-4-6",'
|
|
b'"content":[],"stop_reason":null,"usage":{'
|
|
b'"input_tokens":1234,"cache_read_input_tokens":50000,'
|
|
b'"cache_creation_input_tokens":2500,"output_tokens":1}}}'
|
|
b"\xe5"
|
|
)
|
|
|
|
state = {
|
|
"output_tokens": None,
|
|
"total_bytes": len(partial_message_start),
|
|
"ttfb_ms": 35.0,
|
|
"input_tokens": None,
|
|
"cache_read_input_tokens": 0,
|
|
"cache_creation_input_tokens": 0,
|
|
"cache_creation_ephemeral_5m_input_tokens": 0,
|
|
"cache_creation_ephemeral_1h_input_tokens": 0,
|
|
"sse_buffer": bytearray(partial_message_start),
|
|
}
|
|
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": [{"role": "user", "content": "hi"}]},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-truncated",
|
|
original_tokens=2000,
|
|
optimized_tokens=1800,
|
|
tokens_saved=200,
|
|
transforms_applied=[],
|
|
optimization_latency=5.0,
|
|
stream_state=state,
|
|
start_time=0.0,
|
|
)
|
|
|
|
assert state["input_tokens"] is None
|
|
assert state["cache_read_input_tokens"] == 0
|
|
assert state["cache_creation_input_tokens"] == 0
|
|
assert state["sse_buffer"] == bytearray()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_stream_response_no_op_when_logger_disabled():
|
|
proxy = _build_proxy_with_real_logger(log_full_messages=False)
|
|
proxy.logger = None # `--no-log-requests` would put us here
|
|
|
|
# Should not raise.
|
|
await proxy._finalize_stream_response(
|
|
body={"messages": []},
|
|
provider="anthropic",
|
|
model="claude-sonnet-4-6",
|
|
request_id="req-stream-5",
|
|
original_tokens=10,
|
|
optimized_tokens=8,
|
|
tokens_saved=2,
|
|
transforms_applied=[],
|
|
optimization_latency=1.0,
|
|
stream_state=_stream_state(),
|
|
start_time=0.0,
|
|
)
|