mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): preserve empty transport error context (#3675)
## Description Transport exceptions with empty string representations were returned as Anthropic `api_error` responses with an empty message. Preserve the exception type in the response so clients and operators can identify dropped connections and timeouts. Closes #3674 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Add a shared exception-message formatter with a type-name fallback for empty messages. - Use the fallback in LiteLLM Anthropic and Bedrock streaming error responses and add regression coverage for read errors and timeouts. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ .venv/bin/pytest tests/test_litellm_transport_errors.py tests/test_proxy/test_bedrock_sse_ping.py -q 7 passed in 8.48s $ .venv/bin/pytest tests/test_litellm_transport_errors.py tests/test_proxy/test_bedrock_sse_ping.py tests/test_route_advice.py tests/test_bedrock_streaming_input_tokens.py tests/test_backend_streaming_cache_metrics.py -q 33 passed in 9.14s $ .venv/bin/ruff check tests/test_litellm_transport_errors.py headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py All checks passed! $ .venv/bin/ruff format --check tests/test_litellm_transport_errors.py headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py 5 files already formatted $ .venv/bin/mypy headroom/utils.py headroom/backends/litellm.py headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/streaming.py --ignore-missing-imports Success: no issues found in 4 source files ``` ## Real Behavior Proof - Environment: Linux x86_64, Python 3.11.16, Headroom checkout at commit `78575deb` - Exact command / steps: Run the focused LiteLLM Bedrock regression suite with an empty-message `httpx.ReadError` and `httpx.ReadTimeout`. - Observed result: Non-stream and stream error envelopes now contain `ReadError (no message)` and `ReadTimeout (no message)` instead of an empty message; the handler-level paths are covered as well as the backend formatter. - Not tested: Live AWS Bedrock credentials or an external upstream transport failure. The full repository shard still has an unrelated DeepSeek pricing assertion failure on the base test suite. ## Runtime Rollout Safety - Rollout-managed feature(s): None; this is an error-message fallback only. - Minimum rollout channel: Stable; no new rollout configuration is required. - Stable/default behavior changed: Empty transport exceptions now identify their exception type instead of returning an empty error message. - Kill switch / disable path: Revert this PR; no persisted state or migration is involved. - Unsafe override required: No. - Qualification impact: None; the fallback is covered by focused backend and proxy tests. - Rollback path: Revert this PR. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally - [x] I did **not** edit `CHANGELOG.md` - it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A --------- Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
@@ -12,6 +12,8 @@ import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
|
||||
from headroom.utils import format_exception_message
|
||||
|
||||
from .base import Backend, BackendResponse, StreamEvent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -305,7 +307,7 @@ class AnyLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"any-llm error: {e}")
|
||||
logger.error(f"any-llm error: {format_exception_message(e)}")
|
||||
return self._error_response(e)
|
||||
|
||||
async def stream_message(
|
||||
@@ -497,12 +499,13 @@ class AnyLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"any-llm streaming error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"any-llm streaming error: {error_message}")
|
||||
yield StreamEvent(
|
||||
event_type="error",
|
||||
data={
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": str(e)},
|
||||
"error": {"type": "api_error", "message": error_message},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -590,7 +593,7 @@ class AnyLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"any-llm OpenAI error: {e}")
|
||||
logger.error(f"any-llm OpenAI error: {format_exception_message(e)}")
|
||||
return self._error_response(e, openai_format=True)
|
||||
|
||||
def _error_response(self, e: Exception, openai_format: bool = False) -> BackendResponse:
|
||||
@@ -598,6 +601,7 @@ class AnyLLMBackend(Backend):
|
||||
error_type = "api_error"
|
||||
status_code = 500
|
||||
|
||||
error_message = format_exception_message(e)
|
||||
error_str = str(e).lower()
|
||||
if "authentication" in error_str or "api_key" in error_str or "api key" in error_str:
|
||||
error_type = "invalid_api_key" if openai_format else "authentication_error"
|
||||
@@ -611,11 +615,11 @@ class AnyLLMBackend(Backend):
|
||||
|
||||
body: dict[str, Any]
|
||||
if openai_format:
|
||||
body = {"error": {"message": str(e), "type": error_type, "code": error_type}}
|
||||
body = {"error": {"message": error_message, "type": error_type, "code": error_type}}
|
||||
else:
|
||||
body = {"type": "error", "error": {"type": error_type, "message": str(e)}}
|
||||
body = {"type": "error", "error": {"type": error_type, "message": error_message}}
|
||||
|
||||
return BackendResponse(body=body, status_code=status_code, error=str(e))
|
||||
return BackendResponse(body=body, status_code=status_code, error=error_message)
|
||||
|
||||
async def stream_openai_message(
|
||||
self,
|
||||
@@ -661,10 +665,11 @@ class AnyLLMBackend(Backend):
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"any-llm OpenAI streaming error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"any-llm OpenAI streaming error: {error_message}")
|
||||
error_data = {
|
||||
"error": {
|
||||
"message": str(e),
|
||||
"message": error_message,
|
||||
"type": "api_error",
|
||||
"code": "backend_error",
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from headroom.utils import format_exception_message
|
||||
|
||||
from .base import Backend, BackendResponse, StreamEvent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1261,7 +1263,8 @@ class LiteLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LiteLLM error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"LiteLLM error: {error_message}")
|
||||
|
||||
# Map to Anthropic error format
|
||||
error_type = "api_error"
|
||||
@@ -1281,10 +1284,10 @@ class LiteLLMBackend(Backend):
|
||||
return BackendResponse(
|
||||
body={
|
||||
"type": "error",
|
||||
"error": {"type": error_type, "message": str(e)},
|
||||
"error": {"type": error_type, "message": error_message},
|
||||
},
|
||||
status_code=status_code,
|
||||
error=str(e),
|
||||
error=error_message,
|
||||
)
|
||||
|
||||
async def stream_message(
|
||||
@@ -1671,12 +1674,13 @@ class LiteLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LiteLLM streaming error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"LiteLLM streaming error: {error_message}")
|
||||
yield StreamEvent(
|
||||
event_type="error",
|
||||
data={
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": str(e)},
|
||||
"error": {"type": "api_error", "message": error_message},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1871,7 +1875,8 @@ class LiteLLMBackend(Backend):
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LiteLLM OpenAI error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"LiteLLM OpenAI error: {error_message}")
|
||||
|
||||
# Map to OpenAI error format
|
||||
error_type = "api_error"
|
||||
@@ -1891,13 +1896,13 @@ class LiteLLMBackend(Backend):
|
||||
return BackendResponse(
|
||||
body={
|
||||
"error": {
|
||||
"message": str(e),
|
||||
"message": error_message,
|
||||
"type": error_type,
|
||||
"code": error_type,
|
||||
}
|
||||
},
|
||||
status_code=status_code,
|
||||
error=str(e),
|
||||
error=error_message,
|
||||
)
|
||||
|
||||
async def stream_openai_message(
|
||||
@@ -1970,10 +1975,11 @@ class LiteLLMBackend(Backend):
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"LiteLLM OpenAI streaming error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"LiteLLM OpenAI streaming error: {error_message}")
|
||||
error_data = {
|
||||
"error": {
|
||||
"message": str(e),
|
||||
"message": error_message,
|
||||
"type": "api_error",
|
||||
"code": "backend_error",
|
||||
}
|
||||
|
||||
@@ -64,6 +64,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
|
||||
from headroom.proxy.outcome import RequestOutcome
|
||||
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
|
||||
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
|
||||
from headroom.utils import format_exception_message
|
||||
|
||||
logger = logging.getLogger("headroom.proxy")
|
||||
|
||||
@@ -3694,12 +3695,15 @@ class AnthropicHandlerMixin:
|
||||
content=backend_response.body,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Bedrock backend error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"[{request_id}] Bedrock backend error: {error_message}")
|
||||
# Unit 4: release the pre-upstream semaphore on error.
|
||||
await _finalize_pre_upstream()
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": str(e)},
|
||||
"error": {"type": "api_error", "message": error_message},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ import httpx
|
||||
from headroom.copilot_auth import apply_copilot_api_auth
|
||||
from headroom.proxy.stream_output_tokens import estimate_output_tokens
|
||||
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
|
||||
from headroom.utils import format_exception_message
|
||||
|
||||
logger = logging.getLogger("headroom.proxy")
|
||||
|
||||
@@ -2018,10 +2019,11 @@ class StreamingMixin:
|
||||
logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[{request_id}] Bedrock streaming error: {e}")
|
||||
error_message = format_exception_message(e)
|
||||
logger.error(f"[{request_id}] Bedrock streaming error: {error_message}")
|
||||
error_event = {
|
||||
"type": "error",
|
||||
"error": {"type": "api_error", "message": str(e)},
|
||||
"error": {"type": "api_error", "message": error_message},
|
||||
}
|
||||
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
|
||||
|
||||
|
||||
@@ -20,6 +20,12 @@ def generate_request_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def format_exception_message(exc: BaseException) -> str:
|
||||
"""Return exception text with a useful fallback for empty messages."""
|
||||
message = str(exc)
|
||||
return message or f"{type(exc).__name__} (no message)"
|
||||
|
||||
|
||||
def compute_hash(data: str | bytes) -> str:
|
||||
"""Compute SHA256 hash, returning hex string."""
|
||||
if isinstance(data, str):
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Regression tests for transport errors returned by the LiteLLM backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests._dotenv import importorskip_no_env_leak
|
||||
|
||||
importorskip_no_env_leak("litellm")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from headroom.backends.anyllm import AnyLLMBackend # noqa: E402
|
||||
from headroom.backends.base import BackendResponse # noqa: E402
|
||||
from headroom.backends.litellm import LiteLLMBackend # noqa: E402
|
||||
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
||||
|
||||
_BODY = {
|
||||
"model": "claude-sonnet-4-20250514",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_message_names_transport_error_without_message() -> None:
|
||||
with (
|
||||
patch(
|
||||
"headroom.backends.litellm.acompletion",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=httpx.ReadError(""),
|
||||
),
|
||||
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
||||
):
|
||||
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
||||
result = await backend.send_message(_BODY, {})
|
||||
|
||||
assert result.status_code == 500
|
||||
assert result.error == "ReadError (no message)"
|
||||
assert result.body["error"]["message"] == "ReadError (no message)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_message_names_transport_error_without_message() -> None:
|
||||
with (
|
||||
patch(
|
||||
"headroom.backends.litellm.acompletion",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=httpx.ReadTimeout(""),
|
||||
),
|
||||
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
||||
):
|
||||
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
||||
events = [event async for event in backend.stream_message(_BODY, {})]
|
||||
|
||||
error_event = next(event for event in events if event.event_type == "error")
|
||||
assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anyllm_backend_names_transport_error_without_message() -> None:
|
||||
pytest.importorskip("any_llm")
|
||||
fake_llm = MagicMock()
|
||||
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))
|
||||
|
||||
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
|
||||
backend = AnyLLMBackend(provider="anthropic")
|
||||
result = await backend.send_message(_BODY, {})
|
||||
|
||||
assert result.status_code == 500
|
||||
assert result.error == "ReadError (no message)"
|
||||
assert result.body["error"]["message"] == "ReadError (no message)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:
|
||||
pytest.importorskip("any_llm")
|
||||
fake_llm = MagicMock()
|
||||
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))
|
||||
|
||||
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
|
||||
backend = AnyLLMBackend(provider="anthropic")
|
||||
events = [event async for event in backend.stream_message(_BODY, {})]
|
||||
|
||||
error_event = next(event for event in events if event.event_type == "error")
|
||||
assert error_event.data["error"]["message"] == "ReadTimeout (no message)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:
|
||||
pytest.importorskip("any_llm")
|
||||
with (
|
||||
patch(
|
||||
"headroom.backends.anyllm.AnyLLM.create",
|
||||
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
|
||||
),
|
||||
patch(
|
||||
"headroom.backends.litellm.acompletion",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=httpx.ReadTimeout(""),
|
||||
),
|
||||
):
|
||||
anyllm_backend = AnyLLMBackend(provider="openai")
|
||||
anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})
|
||||
litellm_backend = LiteLLMBackend(provider="openrouter")
|
||||
litellm_result = await litellm_backend.send_openai_message(_BODY, {})
|
||||
|
||||
assert anyllm_result.body["error"]["message"] == "ReadError (no message)"
|
||||
assert anyllm_result.error == "ReadError (no message)"
|
||||
assert litellm_result.body["error"]["message"] == "ReadTimeout (no message)"
|
||||
assert litellm_result.error == "ReadTimeout (no message)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:
|
||||
pytest.importorskip("any_llm")
|
||||
with (
|
||||
patch(
|
||||
"headroom.backends.anyllm.AnyLLM.create",
|
||||
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
|
||||
),
|
||||
patch(
|
||||
"headroom.backends.litellm.acompletion",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=httpx.ReadTimeout(""),
|
||||
),
|
||||
):
|
||||
anyllm_backend = AnyLLMBackend(provider="openai")
|
||||
anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]
|
||||
litellm_backend = LiteLLMBackend(provider="openrouter")
|
||||
litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]
|
||||
|
||||
assert '"message": "ReadError (no message)"' in anyllm_chunks[0]
|
||||
assert '"message": "ReadTimeout (no message)"' in litellm_chunks[0]
|
||||
|
||||
|
||||
def _erroring_anthropic_backend() -> MagicMock:
|
||||
"""Raise blank-message transport errors through both proxy backend paths."""
|
||||
|
||||
async def send_message(body: dict, headers: dict) -> BackendResponse:
|
||||
raise httpx.ReadError("")
|
||||
|
||||
async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:
|
||||
raise httpx.ReadTimeout("")
|
||||
yield # pragma: no cover - keeps this function an async generator
|
||||
|
||||
backend = MagicMock()
|
||||
backend.name = "anyllm-anthropic"
|
||||
backend.send_message = send_message
|
||||
backend.stream_message = stream_message
|
||||
backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
|
||||
backend.supports_model = MagicMock(return_value=True)
|
||||
return backend
|
||||
|
||||
|
||||
def _proxy_config() -> ProxyConfig:
|
||||
return ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=False,
|
||||
rate_limit_enabled=False,
|
||||
backend="anyllm",
|
||||
anyllm_provider="anthropic",
|
||||
)
|
||||
|
||||
|
||||
def _messages_request(*, stream: bool) -> dict:
|
||||
return {
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 32,
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
|
||||
def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:
|
||||
backend = _erroring_anthropic_backend()
|
||||
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
||||
app = create_app(_proxy_config())
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/v1/messages",
|
||||
json=_messages_request(stream=False),
|
||||
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
|
||||
)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert response.json()["error"]["message"] == "ReadError (no message)"
|
||||
|
||||
|
||||
def test_bedrock_stream_names_transport_error_without_message() -> None:
|
||||
backend = _erroring_anthropic_backend()
|
||||
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
||||
app = create_app(_proxy_config())
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/v1/messages",
|
||||
json=_messages_request(stream=True),
|
||||
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert '"message": "ReadTimeout (no message)"' in response.text
|
||||
Reference in New Issue
Block a user