fix(proxy): preserve empty transport error context (#3675)

## Description

Transport exceptions with empty string representations were returned as
Anthropic `api_error` responses with an empty message. Preserve the
exception type in the response so clients and operators can identify
dropped connections and timeouts.

Closes #3674

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Add a shared exception-message formatter with a type-name fallback for
empty messages.
- Use the fallback in LiteLLM Anthropic and Bedrock streaming error
responses and add regression coverage for read errors and timeouts.

## Testing

- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)
- [x] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [ ] Manual testing performed

### Test Output

```text
$ .venv/bin/pytest tests/test_litellm_transport_errors.py tests/test_proxy/test_bedrock_sse_ping.py -q
7 passed in 8.48s

$ .venv/bin/pytest tests/test_litellm_transport_errors.py tests/test_proxy/test_bedrock_sse_ping.py tests/test_route_advice.py tests/test_bedrock_streaming_input_tokens.py tests/test_backend_streaming_cache_metrics.py -q
33 passed in 9.14s

$ .venv/bin/ruff check tests/test_litellm_transport_errors.py headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py
All checks passed!

$ .venv/bin/ruff format --check tests/test_litellm_transport_errors.py headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py
5 files already formatted

$ .venv/bin/mypy headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py --ignore-missing-imports
Success: no issues found in 4 source files
```

## Real Behavior Proof

- Environment: Linux x86_64, Python 3.11.16, Headroom checkout at commit
`78575deb`
- Exact command / steps: Run the focused LiteLLM Bedrock regression
suite with an empty-message `httpx.ReadError` and `httpx.ReadTimeout`.
- Observed result: Non-stream and stream error envelopes now contain
`ReadError (no message)` and `ReadTimeout (no message)` instead of an
empty message; the handler-level paths are covered as well as the
backend formatter.
- Not tested: Live AWS Bedrock credentials or an external upstream
transport failure. The full repository shard still has an unrelated
DeepSeek pricing assertion failure on the base test suite.

## Runtime Rollout Safety

- Rollout-managed feature(s): None; this is an error-message fallback
only.
- Minimum rollout channel: Stable; no new rollout configuration is
required.
- Stable/default behavior changed: Empty transport exceptions now
identify their exception type instead of returning an empty error
message.
- Kill switch / disable path: Revert this PR; no persisted state or
migration is involved.
- Unsafe override required: No.
- Qualification impact: None; the fallback is covered by focused backend
and proxy tests.
- Rollback path: Revert this PR.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally
- [x] I did **not** edit `CHANGELOG.md` - it is generated by
release-please from my Conventional Commit PR title (a CI guard enforces
this)

## Screenshots (if applicable)

N/A

---------

Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
AxelRay
2026-09-25 10:22:43 -05:00
committed by GitHub
co-authored by JD Davis
parent dfdc72511e
commit ca02f28c67
6 changed files with 251 additions and 23 deletions
+14 -9
View File
@@ -12,6 +12,8 @@ import uuid
from collections.abc import AsyncIterator
from typing import Any, cast
from headroom.utils import format_exception_message
from .base import Backend, BackendResponse, StreamEvent
logger = logging.getLogger(__name__)
@@ -305,7 +307,7 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
logger.error(f"any-llm error: {e}")
logger.error(f"any-llm error: {format_exception_message(e)}")
return self._error_response(e)
async def stream_message(
@@ -497,12 +499,13 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
logger.error(f"any-llm streaming error: {e}")
error_message = format_exception_message(e)
logger.error(f"any-llm streaming error: {error_message}")
yield StreamEvent(
event_type="error",
data={
"type": "error",
"error": {"type": "api_error", "message": str(e)},
"error": {"type": "api_error", "message": error_message},
},
)
@@ -590,7 +593,7 @@ class AnyLLMBackend(Backend):
)
except Exception as e:
logger.error(f"any-llm OpenAI error: {e}")
logger.error(f"any-llm OpenAI error: {format_exception_message(e)}")
return self._error_response(e, openai_format=True)
def _error_response(self, e: Exception, openai_format: bool = False) -> BackendResponse:
@@ -598,6 +601,7 @@ class AnyLLMBackend(Backend):
error_type = "api_error"
status_code = 500
error_message = format_exception_message(e)
error_str = str(e).lower()
if "authentication" in error_str or "api_key" in error_str or "api key" in error_str:
error_type = "invalid_api_key" if openai_format else "authentication_error"
@@ -611,11 +615,11 @@ class AnyLLMBackend(Backend):
body: dict[str, Any]
if openai_format:
body = {"error": {"message": str(e), "type": error_type, "code": error_type}}
body = {"error": {"message": error_message, "type": error_type, "code": error_type}}
else:
body = {"type": "error", "error": {"type": error_type, "message": str(e)}}
body = {"type": "error", "error": {"type": error_type, "message": error_message}}
return BackendResponse(body=body, status_code=status_code, error=str(e))
return BackendResponse(body=body, status_code=status_code, error=error_message)
async def stream_openai_message(
self,
@@ -661,10 +665,11 @@ class AnyLLMBackend(Backend):
yield "data: [DONE]\n\n"
except Exception as e:
logger.error(f"any-llm OpenAI streaming error: {e}")
error_message = format_exception_message(e)
logger.error(f"any-llm OpenAI streaming error: {error_message}")
error_data = {
"error": {
"message": str(e),
"message": error_message,
"type": "api_error",
"code": "backend_error",
}
+16 -10
View File
@@ -21,6 +21,8 @@ from collections.abc import AsyncIterator
from dataclasses import dataclass, field
from typing import Any
from headroom.utils import format_exception_message
from .base import Backend, BackendResponse, StreamEvent
logger = logging.getLogger(__name__)
@@ -1261,7 +1263,8 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
logger.error(f"LiteLLM error: {e}")
error_message = format_exception_message(e)
logger.error(f"LiteLLM error: {error_message}")
# Map to Anthropic error format
error_type = "api_error"
@@ -1281,10 +1284,10 @@ class LiteLLMBackend(Backend):
return BackendResponse(
body={
"type": "error",
"error": {"type": error_type, "message": str(e)},
"error": {"type": error_type, "message": error_message},
},
status_code=status_code,
error=str(e),
error=error_message,
)
async def stream_message(
@@ -1671,12 +1674,13 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
logger.error(f"LiteLLM streaming error: {e}")
error_message = format_exception_message(e)
logger.error(f"LiteLLM streaming error: {error_message}")
yield StreamEvent(
event_type="error",
data={
"type": "error",
"error": {"type": "api_error", "message": str(e)},
"error": {"type": "api_error", "message": error_message},
},
)
@@ -1871,7 +1875,8 @@ class LiteLLMBackend(Backend):
)
except Exception as e:
logger.error(f"LiteLLM OpenAI error: {e}")
error_message = format_exception_message(e)
logger.error(f"LiteLLM OpenAI error: {error_message}")
# Map to OpenAI error format
error_type = "api_error"
@@ -1891,13 +1896,13 @@ class LiteLLMBackend(Backend):
return BackendResponse(
body={
"error": {
"message": str(e),
"message": error_message,
"type": error_type,
"code": error_type,
}
},
status_code=status_code,
error=str(e),
error=error_message,
)
async def stream_openai_message(
@@ -1970,10 +1975,11 @@ class LiteLLMBackend(Backend):
yield "data: [DONE]\n\n"
except Exception as e:
logger.error(f"LiteLLM OpenAI streaming error: {e}")
error_message = format_exception_message(e)
logger.error(f"LiteLLM OpenAI streaming error: {error_message}")
error_data = {
"error": {
"message": str(e),
"message": error_message,
"type": "api_error",
"code": "backend_error",
}
+6 -2
View File
@@ -64,6 +64,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
from headroom.proxy.outcome import RequestOutcome
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
@@ -3694,12 +3695,15 @@ class AnthropicHandlerMixin:
content=backend_response.body,
)
except Exception as e:
logger.error(f"[{request_id}] Bedrock backend error: {e}")
error_message = format_exception_message(e)
logger.error(f"[{request_id}] Bedrock backend error: {error_message}")
# Unit 4: release the pre-upstream semaphore on error.
await _finalize_pre_upstream()
return JSONResponse(
status_code=500,
content={
"type": "error",
"error": {"type": "api_error", "message": str(e)},
"error": {"type": "api_error", "message": error_message},
},
)
+4 -2
View File
@@ -30,6 +30,7 @@ import httpx
from headroom.copilot_auth import apply_copilot_api_auth
from headroom.proxy.stream_output_tokens import estimate_output_tokens
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
@@ -2018,10 +2019,11 @@ class StreamingMixin:
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
except Exception as e:
logger.error(f"[{request_id}] Bedrock streaming error: {e}")
error_message = format_exception_message(e)
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
error_event = {
"type": "error",
"error": {"type": "api_error", "message": str(e)},
"error": {"type": "api_error", "message": error_message},
}
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+6
View File
@@ -20,6 +20,12 @@ def generate_request_id() -> str:
return str(uuid.uuid4())
def format_exception_message(exc: BaseException) -> str:
"""Return exception text with a useful fallback for empty messages."""
message = str(exc)
return message or f"{type(exc).__name__} (no message)"
def compute_hash(data: str | bytes) -> str:
"""Compute SHA256 hash, returning hex string."""
if isinstance(data, str):
+205
View File
@@ -0,0 +1,205 @@
"""Regression tests for transport errors returned by the LiteLLM backend."""
from __future__ import annotations
from collections.abc import AsyncIterator
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from tests._dotenv import importorskip_no_env_leak
importorskip_no_env_leak("litellm")
from fastapi.testclient import TestClient # noqa: E402
from headroom.backends.anyllm import AnyLLMBackend # noqa: E402
from headroom.backends.base import BackendResponse # noqa: E402
from headroom.backends.litellm import LiteLLMBackend # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
_BODY = {
"model": "claude-sonnet-4-20250514",
"messages": [{"role": "user", "content": "hello"}],
}
@pytest.mark.asyncio
async def test_send_message_names_transport_error_without_message() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadError(""),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
result = await backend.send_message(_BODY, {})
assert result.status_code == 500
assert result.error == "ReadError (no message)"
assert result.body["error"]["message"] == "ReadError (no message)"
@pytest.mark.asyncio
async def test_stream_message_names_transport_error_without_message() -> None:
with (
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
):
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
events = [event async for event in backend.stream_message(_BODY, {})]
error_event = next(event for event in events if event.event_type == "error")
assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
@pytest.mark.asyncio
async def test_anyllm_backend_names_transport_error_without_message() -> None:
pytest.importorskip("any_llm")
fake_llm = MagicMock()
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
backend = AnyLLMBackend(provider="anthropic")
result = await backend.send_message(_BODY, {})
assert result.status_code == 500
assert result.error == "ReadError (no message)"
assert result.body["error"]["message"] == "ReadError (no message)"
@pytest.mark.asyncio
async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:
pytest.importorskip("any_llm")
fake_llm = MagicMock()
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
backend = AnyLLMBackend(provider="anthropic")
events = [event async for event in backend.stream_message(_BODY, {})]
error_event = next(event for event in events if event.event_type == "error")
assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
@pytest.mark.asyncio
async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:
pytest.importorskip("any_llm")
with (
patch(
"headroom.backends.anyllm.AnyLLM.create",
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
),
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
):
anyllm_backend = AnyLLMBackend(provider="openai")
anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})
litellm_backend = LiteLLMBackend(provider="openrouter")
litellm_result = await litellm_backend.send_openai_message(_BODY, {})
assert anyllm_result.body["error"]["message"] == "ReadError (no message)"
assert anyllm_result.error == "ReadError (no message)"
assert litellm_result.body["error"]["message"] == "ReadTimeout (no message)"
assert litellm_result.error == "ReadTimeout (no message)"
@pytest.mark.asyncio
async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:
pytest.importorskip("any_llm")
with (
patch(
"headroom.backends.anyllm.AnyLLM.create",
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
),
patch(
"headroom.backends.litellm.acompletion",
new_callable=AsyncMock,
side_effect=httpx.ReadTimeout(""),
),
):
anyllm_backend = AnyLLMBackend(provider="openai")
anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]
litellm_backend = LiteLLMBackend(provider="openrouter")
litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]
assert '"message": "ReadError (no message)"' in anyllm_chunks[0]
assert '"message": "ReadTimeout (no message)"' in litellm_chunks[0]
def _erroring_anthropic_backend() -> MagicMock:
"""Raise blank-message transport errors through both proxy backend paths."""
async def send_message(body: dict, headers: dict) -> BackendResponse:
raise httpx.ReadError("")
async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:
raise httpx.ReadTimeout("")
yield # pragma: no cover - keeps this function an async generator
backend = MagicMock()
backend.name = "anyllm-anthropic"
backend.send_message = send_message
backend.stream_message = stream_message
backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
backend.supports_model = MagicMock(return_value=True)
return backend
def _proxy_config() -> ProxyConfig:
return ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
backend="anyllm",
anyllm_provider="anthropic",
)
def _messages_request(*, stream: bool) -> dict:
return {
"model": "claude-3-5-sonnet-20241022",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 32,
"stream": stream,
}
def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:
backend = _erroring_anthropic_backend()
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
app = create_app(_proxy_config())
with TestClient(app) as client:
response = client.post(
"/v1/messages",
json=_messages_request(stream=False),
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
)
assert response.status_code == 500
assert response.json()["error"]["message"] == "ReadError (no message)"
def test_bedrock_stream_names_transport_error_without_message() -> None:
backend = _erroring_anthropic_backend()
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
app = create_app(_proxy_config())
with TestClient(app) as client:
response = client.post(
"/v1/messages",
json=_messages_request(stream=True),
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
)
assert response.status_code == 200
assert '"message": "ReadTimeout (no message)"' in response.text