mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:07:29 +08:00
## Description Fixes #3943. The LiteLLM backend turned any upstream error it did not recognise by message text into HTTP 500, so an upstream 400 reached the client as a 500 and its retry-on-400 fallback never ran. The Anthropic and OpenAI error paths now use the exception's own 4xx `status_code` (400, 401, 403, 404, 413, 422, 429) with a matching error type, and keep the old message matching otherwise. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Map 4xx status; add tests. ## Testing - [x] Unit tests pass (`pytest`) (new file only) - [x] Linting passes (`ruff check .`) (changed files) - [ ] Type checking passes (`mypy headroom`) (not run) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text # Paste relevant command output or artifact links here tests/test_litellm_transport_errors.py: 22 passed, 4 skipped ruff check (changed files): All checks passed! ``` ## Real Behavior Proof - Environment: local venv, mocked upstream - Exact command / steps: the pytest run above - Observed result: BadRequestError gives 400, was 500 - Not tested: real Bedrock ## Runtime Rollout Safety - Rollout-managed feature(s): none - Minimum rollout channel: n/a - Stable/default behavior changed: upstream 4xx keep their status - Kill switch / disable path: none - Unsafe override required: no - Qualification impact: none - Rollback path: revert the commit ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [ ] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A ## Additional Notes N/A <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/3944" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: JD Davis <mxjerrett@gmail.com>
377 lines
13 KiB
Python
377 lines
13 KiB
Python
"""Regression tests for transport errors returned by the LiteLLM backend."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import AsyncIterator
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from tests._dotenv import importorskip_no_env_leak
|
|
|
|
importorskip_no_env_leak("litellm")
|
|
|
|
from fastapi.testclient import TestClient # noqa: E402
|
|
|
|
from headroom.backends.anyllm import AnyLLMBackend # noqa: E402
|
|
from headroom.backends.base import BackendResponse # noqa: E402
|
|
from headroom.backends.litellm import LiteLLMBackend # noqa: E402
|
|
from headroom.proxy.public_errors import ( # noqa: E402
|
|
UPSTREAM_PROTOCOL_ERROR,
|
|
UPSTREAM_TIMEOUT,
|
|
public_message,
|
|
)
|
|
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
|
|
|
# Transport exceptions never surface their own text (it names the upstream host);
|
|
# an empty-message ReadError/ReadTimeout maps to the fixed vocabulary instead.
|
|
_PROTOCOL = public_message(UPSTREAM_PROTOCOL_ERROR)
|
|
_TIMEOUT = public_message(UPSTREAM_TIMEOUT)
|
|
|
|
_BODY = {
|
|
"model": "claude-sonnet-4-20250514",
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_send_message_names_transport_error_without_message() -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=httpx.ReadError(""),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await backend.send_message(_BODY, {})
|
|
|
|
assert result.status_code == 500
|
|
assert result.error == _PROTOCOL
|
|
assert result.body["error"]["message"] == _PROTOCOL
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stream_message_names_transport_error_without_message() -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=httpx.ReadTimeout(""),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
events = [event async for event in backend.stream_message(_BODY, {})]
|
|
|
|
error_event = next(event for event in events if event.event_type == "error")
|
|
assert error_event.data["error"]["message"] == _TIMEOUT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anyllm_backend_names_transport_error_without_message() -> None:
|
|
pytest.importorskip("any_llm")
|
|
fake_llm = MagicMock()
|
|
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadError(""))
|
|
|
|
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
|
|
backend = AnyLLMBackend(provider="anthropic")
|
|
result = await backend.send_message(_BODY, {})
|
|
|
|
assert result.status_code == 500
|
|
assert result.error == _PROTOCOL
|
|
assert result.body["error"]["message"] == _PROTOCOL
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anyllm_stream_backend_names_transport_error_without_message() -> None:
|
|
pytest.importorskip("any_llm")
|
|
fake_llm = MagicMock()
|
|
fake_llm.acompletion = AsyncMock(side_effect=httpx.ReadTimeout(""))
|
|
|
|
with patch("headroom.backends.anyllm.AnyLLM.create", return_value=fake_llm):
|
|
backend = AnyLLMBackend(provider="anthropic")
|
|
events = [event async for event in backend.stream_message(_BODY, {})]
|
|
|
|
error_event = next(event for event in events if event.event_type == "error")
|
|
assert error_event.data["error"]["message"] == _TIMEOUT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openai_backend_boundaries_name_transport_errors_without_message() -> None:
|
|
pytest.importorskip("any_llm")
|
|
with (
|
|
patch(
|
|
"headroom.backends.anyllm.AnyLLM.create",
|
|
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
|
|
),
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=httpx.ReadTimeout(""),
|
|
),
|
|
):
|
|
anyllm_backend = AnyLLMBackend(provider="openai")
|
|
anyllm_result = await anyllm_backend.send_openai_message(_BODY, {})
|
|
litellm_backend = LiteLLMBackend(provider="openrouter")
|
|
litellm_result = await litellm_backend.send_openai_message(_BODY, {})
|
|
|
|
assert anyllm_result.body["error"]["message"] == _PROTOCOL
|
|
assert anyllm_result.error == _PROTOCOL
|
|
assert litellm_result.body["error"]["message"] == _TIMEOUT
|
|
assert litellm_result.error == _TIMEOUT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openai_stream_boundaries_name_transport_errors_without_message() -> None:
|
|
pytest.importorskip("any_llm")
|
|
with (
|
|
patch(
|
|
"headroom.backends.anyllm.AnyLLM.create",
|
|
return_value=MagicMock(acompletion=AsyncMock(side_effect=httpx.ReadError(""))),
|
|
),
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=httpx.ReadTimeout(""),
|
|
),
|
|
):
|
|
anyllm_backend = AnyLLMBackend(provider="openai")
|
|
anyllm_chunks = [chunk async for chunk in anyllm_backend.stream_openai_message(_BODY, {})]
|
|
litellm_backend = LiteLLMBackend(provider="openrouter")
|
|
litellm_chunks = [chunk async for chunk in litellm_backend.stream_openai_message(_BODY, {})]
|
|
|
|
assert f'"message": {json.dumps(_PROTOCOL)}' in anyllm_chunks[0]
|
|
assert f'"message": {json.dumps(_TIMEOUT)}' in litellm_chunks[0]
|
|
|
|
|
|
def _erroring_anthropic_backend() -> MagicMock:
|
|
"""Raise blank-message transport errors through both proxy backend paths."""
|
|
|
|
async def send_message(body: dict, headers: dict) -> BackendResponse:
|
|
raise httpx.ReadError("")
|
|
|
|
async def stream_message(body: dict, headers: dict) -> AsyncIterator[object]:
|
|
raise httpx.ReadTimeout("")
|
|
yield # pragma: no cover - keeps this function an async generator
|
|
|
|
backend = MagicMock()
|
|
backend.name = "anyllm-anthropic"
|
|
backend.send_message = send_message
|
|
backend.stream_message = stream_message
|
|
backend.map_model_id = MagicMock(return_value="claude-3-5-sonnet-20241022")
|
|
backend.supports_model = MagicMock(return_value=True)
|
|
return backend
|
|
|
|
|
|
def _proxy_config() -> ProxyConfig:
|
|
return ProxyConfig(
|
|
optimize=False,
|
|
cache_enabled=False,
|
|
rate_limit_enabled=False,
|
|
backend="anyllm",
|
|
anyllm_provider="anthropic",
|
|
)
|
|
|
|
|
|
def _messages_request(*, stream: bool) -> dict:
|
|
return {
|
|
"model": "claude-3-5-sonnet-20241022",
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
"max_tokens": 32,
|
|
"stream": stream,
|
|
}
|
|
|
|
|
|
def test_anthropic_proxy_names_nonstream_transport_error_without_message() -> None:
|
|
backend = _erroring_anthropic_backend()
|
|
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
|
app = create_app(_proxy_config())
|
|
with TestClient(app) as client:
|
|
response = client.post(
|
|
"/v1/messages",
|
|
json=_messages_request(stream=False),
|
|
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
|
|
)
|
|
|
|
assert response.status_code == 500
|
|
# Proxy-level replies append the request id for log correlation.
|
|
assert response.json()["error"]["message"].startswith(_PROTOCOL)
|
|
assert response.json()["error"]["code"] == UPSTREAM_PROTOCOL_ERROR
|
|
|
|
|
|
def test_bedrock_stream_names_transport_error_without_message() -> None:
|
|
backend = _erroring_anthropic_backend()
|
|
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
|
app = create_app(_proxy_config())
|
|
with TestClient(app) as client:
|
|
response = client.post(
|
|
"/v1/messages",
|
|
json=_messages_request(stream=True),
|
|
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert _TIMEOUT in response.text
|
|
assert '"code": "upstream_timeout"' in response.text
|
|
|
|
|
|
def _upstream_error(status_code: int, message: str) -> Exception:
|
|
"""A LiteLLM-style exception: carries the upstream HTTP status as an attribute."""
|
|
import litellm
|
|
|
|
classes = {
|
|
400: litellm.BadRequestError,
|
|
401: litellm.AuthenticationError,
|
|
403: litellm.PermissionDeniedError,
|
|
404: litellm.NotFoundError,
|
|
422: litellm.UnprocessableEntityError,
|
|
429: litellm.RateLimitError,
|
|
500: litellm.InternalServerError,
|
|
}
|
|
if status_code not in classes:
|
|
# Unmapped 4xx: litellm bakes the status into each class, so use a bare
|
|
# exception carrying only the ``status_code`` attribute.
|
|
err = Exception(message)
|
|
err.status_code = status_code # type: ignore[attr-defined]
|
|
return err
|
|
response = httpx.Response(status_code, request=httpx.Request("POST", "https://upstream.test"))
|
|
return classes[status_code](
|
|
message=message, model="bedrock/claude", llm_provider="bedrock", response=response
|
|
)
|
|
|
|
|
|
_BAD_THINKING = (
|
|
"BedrockException - thinking.adaptive.display: Input should be 'summarized', 'omitted'"
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("status", "error_type"),
|
|
[
|
|
(400, "invalid_request_error"),
|
|
(403, "permission_error"),
|
|
(422, "invalid_request_error"),
|
|
(401, "authentication_error"),
|
|
(404, "not_found_error"),
|
|
(429, "rate_limit_error"),
|
|
(402, "invalid_request_error"),
|
|
(408, "invalid_request_error"),
|
|
(409, "invalid_request_error"),
|
|
(500, "api_error"),
|
|
],
|
|
)
|
|
async def test_send_message_keeps_upstream_litellm_status(status: int, error_type: str) -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=_upstream_error(status, _BAD_THINKING),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await backend.send_message(_BODY, {})
|
|
|
|
assert result.status_code == status
|
|
assert result.body["error"]["type"] == error_type
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("status", "error_type"),
|
|
[
|
|
(400, "invalid_request_error"),
|
|
(403, "permission_error"),
|
|
(422, "invalid_request_error"),
|
|
(401, "invalid_api_key"),
|
|
(404, "model_not_found"),
|
|
(429, "rate_limit_exceeded"),
|
|
(402, "invalid_request_error"),
|
|
(408, "invalid_request_error"),
|
|
(409, "invalid_request_error"),
|
|
(500, "api_error"),
|
|
],
|
|
)
|
|
async def test_send_openai_message_keeps_upstream_litellm_status(
|
|
status: int, error_type: str
|
|
) -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=_upstream_error(status, _BAD_THINKING),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await backend.send_openai_message(_BODY, {})
|
|
|
|
assert result.status_code == status
|
|
assert result.body["error"]["type"] == error_type
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_send_message_substring_fallback_without_status_attribute() -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=RuntimeError("Authentication failed: bad credentials"),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await backend.send_message(_BODY, {})
|
|
|
|
assert result.status_code == 401
|
|
assert result.body["error"]["type"] == "authentication_error"
|
|
|
|
|
|
class _StatusError(Exception):
|
|
def __init__(self, status_code: object) -> None:
|
|
super().__init__("payload too large")
|
|
self.status_code = status_code
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("method", "error_type"),
|
|
[("send_message", "request_too_large"), ("send_openai_message", "invalid_request_error")],
|
|
)
|
|
async def test_413_status_attribute_is_kept(method: str, error_type: str) -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=_StatusError(413),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await getattr(backend, method)(_BODY, {})
|
|
|
|
assert result.status_code == 413
|
|
assert result.body["error"]["type"] == error_type
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_int_status_attribute_falls_back_to_substring() -> None:
|
|
with (
|
|
patch(
|
|
"headroom.backends.litellm.acompletion",
|
|
new_callable=AsyncMock,
|
|
side_effect=_StatusError("429"),
|
|
),
|
|
patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
|
|
):
|
|
backend = LiteLLMBackend(provider="bedrock", region="us-east-1")
|
|
result = await backend.send_message(_BODY, {})
|
|
|
|
assert result.status_code == 500
|
|
assert result.body["error"]["type"] == "api_error"
|