mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): preserve Bedrock body-limit error dialect (#3871)
## Description Bedrock InvokeModel requests that exceed the request-body ceiling were returned with `error.type=invalid_request_error`, even though the Bedrock handler reports the same condition as `request_too_large`. This makes the middleware response inconsistent with the route's wire contract. Closes #3867 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature - [ ] Breaking change - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring ## Changes Made - Added explicit dialect handling for `/model/{id}/invoke` and `/model/{id}/invoke-with-response-stream`. - Preserved the existing Anthropic Messages and OpenAI payload formats. - Added regression coverage for both Bedrock routes, the Anthropic route, and the OpenAI fallback, including response-once behavior. ## Testing - [x] Unit tests pass - [x] Linting passes - [x] Type checking passes - [x] New tests added - [ ] Manual testing performed ### Test Output ```text 41 passed: tests/test_proxy/test_bedrock_passthrough.py tests/test_proxy_extension_ordering.py 55 passed, 2 skipped: related body-limit suites ruff check: passed mypy headroom/proxy/request_body_limit.py: passed ``` ## Real Behavior Proof On main, the oversized Bedrock request returned: ```json {"error":{"message":"Request body too large...","type":"invalid_request_error","code":"request_too_large"}} ``` With this change, the same request returns: ```json {"error":{"type":"request_too_large","message":"Request body too large..."}} ``` The request is rejected before any upstream call. ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Stable/default behavior changed: only the 413 error payload for Bedrock InvokeModel routes. - Kill switch / disable path: Not applicable. - Unsafe override required: No. - Rollback path: Revert this PR. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review - [x] I have added tests proving the fix - [x] New and existing tests pass locally - [x] I did not edit `CHANGELOG.md` ## Screenshots (if applicable) Not applicable.
This commit is contained in:
@@ -48,6 +48,21 @@ Send = Callable[[MutableMapping[str, Any]], Awaitable[None]]
|
||||
# the OpenAI-style payload, which is what the handlers' own 413s use.
|
||||
_ANTHROPIC_PATH_MARKERS = ("/v1/messages",)
|
||||
|
||||
# The AWS Bedrock InvokeModel passthrough speaks a third dialect: its handler
|
||||
# (``handle_bedrock_invoke``) answers a too-large body with
|
||||
# ``{"error": {"type": "request_too_large", "message": ...}}``. Its routes are
|
||||
# ``/model/{model_id:path}/invoke`` and
|
||||
# ``/model/{model_id:path}/invoke-with-response-stream``
|
||||
# (``headroom/providers/proxy_routes.py``), and model ids may contain slashes
|
||||
# (ARN-style inference profiles), so the match is a suffix check under the
|
||||
# ``/model/`` prefix rather than a fixed-path table.
|
||||
_BEDROCK_INVOKE_PREFIX = "/model/"
|
||||
_BEDROCK_INVOKE_SUFFIXES = ("/invoke", "/invoke-with-response-stream")
|
||||
|
||||
|
||||
def _is_bedrock_invoke_path(path: str) -> bool:
|
||||
return path.startswith(_BEDROCK_INVOKE_PREFIX) and path.endswith(_BEDROCK_INVOKE_SUFFIXES)
|
||||
|
||||
|
||||
def _too_large_payload(path: str, limit: int) -> bytes:
|
||||
"""Mirror the handlers' own 413 payloads so clients see one shape per dialect."""
|
||||
@@ -57,6 +72,13 @@ def _too_large_payload(path: str, limit: int) -> bytes:
|
||||
"type": "error",
|
||||
"error": {"type": "request_too_large", "message": message},
|
||||
}
|
||||
elif _is_bedrock_invoke_path(path):
|
||||
payload = {
|
||||
"error": {
|
||||
"type": "request_too_large",
|
||||
"message": message,
|
||||
}
|
||||
}
|
||||
else:
|
||||
payload = {
|
||||
"error": {
|
||||
|
||||
@@ -352,6 +352,81 @@ def test_oversized_body_returns_413_and_never_forwards(monkeypatch):
|
||||
http.send.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route", [INVOKE, INVOKE_STREAM], ids=["invoke", "invoke-with-response-stream"]
|
||||
)
|
||||
def test_body_ceiling_413_uses_bedrock_handler_dialect(monkeypatch, route):
|
||||
"""The body ceiling is enforced outside the handler, but its 413 must keep
|
||||
the route's own wire dialect: exactly the payload ``handle_bedrock_invoke``
|
||||
returns for ``RequestBodyTooLarge`` —
|
||||
``{"error": {"type": "request_too_large", "message": ...}}`` — with the
|
||||
request refused before any upstream call.
|
||||
"""
|
||||
import headroom.proxy.helpers as helpers_mod
|
||||
|
||||
small_cap = 1024
|
||||
monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
|
||||
|
||||
app = create_app(_make_config())
|
||||
with TestClient(app) as client:
|
||||
proxy = client.app.state.proxy
|
||||
http = _install_fake_client(proxy, _FakeUpstream())
|
||||
proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
|
||||
body = {"messages": [{"role": "user", "content": "x" * (small_cap * 4)}], "max_tokens": 8}
|
||||
resp = client.post(route, json=body)
|
||||
|
||||
assert resp.status_code == 413
|
||||
assert resp.json() == {
|
||||
"error": {
|
||||
"type": "request_too_large",
|
||||
"message": f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB",
|
||||
}
|
||||
}
|
||||
http.build_request.assert_not_called()
|
||||
http.send.assert_not_called()
|
||||
|
||||
|
||||
def test_body_ceiling_413_leaves_other_dialects_unchanged(monkeypatch):
|
||||
"""The dialect fix is scoped to the Bedrock InvokeModel routes: ``/v1/messages``
|
||||
keeps the Anthropic envelope and OpenAI chat keeps
|
||||
``{"error": {"type": "invalid_request_error", "code": "request_too_large"}}``.
|
||||
"""
|
||||
import headroom.proxy.helpers as helpers_mod
|
||||
|
||||
small_cap = 1024
|
||||
monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
|
||||
|
||||
app = create_app(_make_config())
|
||||
with TestClient(app) as client:
|
||||
proxy = client.app.state.proxy
|
||||
http = _install_fake_client(proxy, _FakeUpstream())
|
||||
proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
|
||||
oversized = b"x" * (small_cap * 4)
|
||||
messages = client.post(
|
||||
"/v1/messages",
|
||||
content=oversized,
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
chat = client.post(
|
||||
"/v1/chat/completions",
|
||||
content=oversized,
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
|
||||
message = f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB"
|
||||
assert messages.status_code == 413
|
||||
assert messages.json() == {
|
||||
"type": "error",
|
||||
"error": {"type": "request_too_large", "message": message},
|
||||
}
|
||||
assert chat.status_code == 413
|
||||
assert chat.json() == {
|
||||
"error": {"message": message, "type": "invalid_request_error", "code": "request_too_large"}
|
||||
}
|
||||
http.build_request.assert_not_called()
|
||||
http.send.assert_not_called()
|
||||
|
||||
|
||||
# ── metrics ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ monkeypatching discovery, so the tests exercise the app exactly as
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -247,6 +248,42 @@ class TestBodyCeilingAppliesBeforeExtensionMiddleware:
|
||||
},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
[
|
||||
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
|
||||
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
|
||||
],
|
||||
ids=["invoke", "invoke-with-response-stream"],
|
||||
)
|
||||
def test_bedrock_invoke_dialect_payload_on_model_paths(self, monkeypatch, route):
|
||||
"""The two Bedrock InvokeModel routes keep handle_bedrock_invoke's wire
|
||||
dialect on the body ceiling: exactly
|
||||
``{"error": {"type": "request_too_large", "message": ...}}`` — not the
|
||||
OpenAI-style payload — and the extension never buffers the body.
|
||||
"""
|
||||
monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
|
||||
app = _make_app(
|
||||
monkeypatch,
|
||||
_install_buffering,
|
||||
proxy_token=TOKEN,
|
||||
bedrock_api_url="http://127.0.0.1:4000",
|
||||
)
|
||||
with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
|
||||
resp = c.post(
|
||||
route,
|
||||
content=b"x" * 4096,
|
||||
headers={"Authorization": f"Bearer {TOKEN}", "content-type": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 413
|
||||
assert resp.json() == {
|
||||
"error": {
|
||||
"type": "request_too_large",
|
||||
"message": "Request body too large. Maximum size is 0MB",
|
||||
}
|
||||
}
|
||||
assert _BufferingExtension.max_seen == 0
|
||||
|
||||
def test_body_within_ceiling_reaches_extension_intact(self, monkeypatch):
|
||||
monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
|
||||
app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
|
||||
@@ -369,6 +406,68 @@ class TestRequestBodyLimitMiddlewareDirectly:
|
||||
assert statuses == [413] # exactly one refusal; the downstream 400 was dropped
|
||||
assert downstream_sent == [{"attempted": True}]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
|
||||
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
|
||||
],
|
||||
ids=["invoke", "invoke-with-response-stream"],
|
||||
)
|
||||
async def test_stream_trip_answers_once_in_bedrock_dialect(self, path):
|
||||
"""On a metered-stream trip over a Bedrock InvokeModel route the single
|
||||
refusal carries handle_bedrock_invoke's dialect, and the handler's own
|
||||
413 answer (its RequestBodyTooLarge response) is discarded by the
|
||||
response-once state: exactly one response reaches the wire.
|
||||
"""
|
||||
downstream_sent: list[dict] = []
|
||||
|
||||
async def downstream(scope, receive, send):
|
||||
try:
|
||||
while True:
|
||||
m = await receive()
|
||||
if m["type"] != "http.request":
|
||||
break
|
||||
if not m.get("more_body"):
|
||||
break
|
||||
except proxy_helpers.RequestBodyTooLarge:
|
||||
# handle_bedrock_invoke answers RequestBodyTooLarge with its own
|
||||
# 413; response-once must drop it because the ceiling already answered.
|
||||
body = b'{"error": {"type": "request_too_large", "message": "from handler"}}'
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(body)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
downstream_sent.append({"attempted": True})
|
||||
return
|
||||
raise AssertionError("reader should have been stopped")
|
||||
|
||||
sent: list[dict] = []
|
||||
|
||||
async def send(m):
|
||||
sent.append(m)
|
||||
|
||||
mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
|
||||
await mw(self._http_scope(path=path), self._chunked_receive([b"a" * 400] * 5), send)
|
||||
|
||||
statuses = [m["status"] for m in sent if m["type"] == "http.response.start"]
|
||||
assert statuses == [413] # exactly one response; the handler's 413 was dropped
|
||||
body = b"".join(m.get("body") or b"" for m in sent if m["type"] == "http.response.body")
|
||||
assert json.loads(body) == {
|
||||
"error": {
|
||||
"type": "request_too_large",
|
||||
"message": "Request body too large. Maximum size is 0MB",
|
||||
}
|
||||
}
|
||||
assert downstream_sent == [{"attempted": True}]
|
||||
|
||||
async def test_body_under_ceiling_is_delivered_untouched(self):
|
||||
held = bytearray()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user