fix(proxy): preserve Bedrock body-limit error dialect (#3871)

## Description

Bedrock InvokeModel requests that exceed the request-body ceiling were
returned with `error.type=invalid_request_error`, even though the
Bedrock handler reports the same condition as `request_too_large`. This
makes the middleware response inconsistent with the route's wire
contract.

Closes #3867

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature
- [ ] Breaking change
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring

## Changes Made

- Added explicit dialect handling for `/model/{id}/invoke` and
`/model/{id}/invoke-with-response-stream`.
- Preserved the existing Anthropic Messages and OpenAI payload formats.
- Added regression coverage for both Bedrock routes, the Anthropic
route, and the OpenAI fallback, including response-once behavior.

## Testing

- [x] Unit tests pass
- [x] Linting passes
- [x] Type checking passes
- [x] New tests added
- [ ] Manual testing performed

### Test Output

```text
41 passed: tests/test_proxy/test_bedrock_passthrough.py tests/test_proxy_extension_ordering.py
55 passed, 2 skipped: related body-limit suites
ruff check: passed
mypy headroom/proxy/request_body_limit.py: passed
```

## Real Behavior Proof

On main, the oversized Bedrock request returned:

```json
{"error":{"message":"Request body too large...","type":"invalid_request_error","code":"request_too_large"}}
```

With this change, the same request returns:

```json
{"error":{"type":"request_too_large","message":"Request body too large..."}}
```

The request is rejected before any upstream call.

## Runtime Rollout Safety

- Rollout-managed feature(s): None.
- Stable/default behavior changed: only the 413 error payload for
Bedrock InvokeModel routes.
- Kill switch / disable path: Not applicable.
- Unsafe override required: No.
- Rollback path: Revert this PR.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review
- [x] I have added tests proving the fix
- [x] New and existing tests pass locally
- [x] I did not edit `CHANGELOG.md`

## Screenshots (if applicable)

Not applicable.
This commit is contained in:
charlieanna
2026-09-29 22:29:46 -07:00
committed by GitHub
parent 5e0435d215
commit ffc6edb4ba
3 changed files with 196 additions and 0 deletions
+22
View File
@@ -48,6 +48,21 @@ Send = Callable[[MutableMapping[str, Any]], Awaitable[None]]
# the OpenAI-style payload, which is what the handlers' own 413s use.
_ANTHROPIC_PATH_MARKERS = ("/v1/messages",)
# The AWS Bedrock InvokeModel passthrough speaks a third dialect: its handler
# (``handle_bedrock_invoke``) answers a too-large body with
# ``{"error": {"type": "request_too_large", "message": ...}}``. Its routes are
# ``/model/{model_id:path}/invoke`` and
# ``/model/{model_id:path}/invoke-with-response-stream``
# (``headroom/providers/proxy_routes.py``), and model ids may contain slashes
# (ARN-style inference profiles), so the match is a suffix check under the
# ``/model/`` prefix rather than a fixed-path table.
_BEDROCK_INVOKE_PREFIX = "/model/"
_BEDROCK_INVOKE_SUFFIXES = ("/invoke", "/invoke-with-response-stream")
def _is_bedrock_invoke_path(path: str) -> bool:
return path.startswith(_BEDROCK_INVOKE_PREFIX) and path.endswith(_BEDROCK_INVOKE_SUFFIXES)
def _too_large_payload(path: str, limit: int) -> bytes:
"""Mirror the handlers' own 413 payloads so clients see one shape per dialect."""
@@ -57,6 +72,13 @@ def _too_large_payload(path: str, limit: int) -> bytes:
"type": "error",
"error": {"type": "request_too_large", "message": message},
}
elif _is_bedrock_invoke_path(path):
payload = {
"error": {
"type": "request_too_large",
"message": message,
}
}
else:
payload = {
"error": {
@@ -352,6 +352,81 @@ def test_oversized_body_returns_413_and_never_forwards(monkeypatch):
http.send.assert_not_called()
@pytest.mark.parametrize(
"route", [INVOKE, INVOKE_STREAM], ids=["invoke", "invoke-with-response-stream"]
)
def test_body_ceiling_413_uses_bedrock_handler_dialect(monkeypatch, route):
"""The body ceiling is enforced outside the handler, but its 413 must keep
the route's own wire dialect: exactly the payload ``handle_bedrock_invoke``
returns for ``RequestBodyTooLarge`` —
``{"error": {"type": "request_too_large", "message": ...}}`` — with the
request refused before any upstream call.
"""
import headroom.proxy.helpers as helpers_mod
small_cap = 1024
monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
app = create_app(_make_config())
with TestClient(app) as client:
proxy = client.app.state.proxy
http = _install_fake_client(proxy, _FakeUpstream())
proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
body = {"messages": [{"role": "user", "content": "x" * (small_cap * 4)}], "max_tokens": 8}
resp = client.post(route, json=body)
assert resp.status_code == 413
assert resp.json() == {
"error": {
"type": "request_too_large",
"message": f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB",
}
}
http.build_request.assert_not_called()
http.send.assert_not_called()
def test_body_ceiling_413_leaves_other_dialects_unchanged(monkeypatch):
"""The dialect fix is scoped to the Bedrock InvokeModel routes: ``/v1/messages``
keeps the Anthropic envelope and OpenAI chat keeps
``{"error": {"type": "invalid_request_error", "code": "request_too_large"}}``.
"""
import headroom.proxy.helpers as helpers_mod
small_cap = 1024
monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
app = create_app(_make_config())
with TestClient(app) as client:
proxy = client.app.state.proxy
http = _install_fake_client(proxy, _FakeUpstream())
proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
oversized = b"x" * (small_cap * 4)
messages = client.post(
"/v1/messages",
content=oversized,
headers={"content-type": "application/json"},
)
chat = client.post(
"/v1/chat/completions",
content=oversized,
headers={"content-type": "application/json"},
)
message = f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB"
assert messages.status_code == 413
assert messages.json() == {
"type": "error",
"error": {"type": "request_too_large", "message": message},
}
assert chat.status_code == 413
assert chat.json() == {
"error": {"message": message, "type": "invalid_request_error", "code": "request_too_large"}
}
http.build_request.assert_not_called()
http.send.assert_not_called()
# ── metrics ───────────────────────────────────────────────────────────
+99
View File
@@ -17,6 +17,7 @@ monkeypatching discovery, so the tests exercise the app exactly as
from __future__ import annotations
import json
from typing import Any
import pytest
@@ -247,6 +248,42 @@ class TestBodyCeilingAppliesBeforeExtensionMiddleware:
},
}
@pytest.mark.parametrize(
"route",
[
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
],
ids=["invoke", "invoke-with-response-stream"],
)
def test_bedrock_invoke_dialect_payload_on_model_paths(self, monkeypatch, route):
"""The two Bedrock InvokeModel routes keep handle_bedrock_invoke's wire
dialect on the body ceiling: exactly
``{"error": {"type": "request_too_large", "message": ...}}`` — not the
OpenAI-style payload — and the extension never buffers the body.
"""
monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
app = _make_app(
monkeypatch,
_install_buffering,
proxy_token=TOKEN,
bedrock_api_url="http://127.0.0.1:4000",
)
with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
resp = c.post(
route,
content=b"x" * 4096,
headers={"Authorization": f"Bearer {TOKEN}", "content-type": "application/json"},
)
assert resp.status_code == 413
assert resp.json() == {
"error": {
"type": "request_too_large",
"message": "Request body too large. Maximum size is 0MB",
}
}
assert _BufferingExtension.max_seen == 0
def test_body_within_ceiling_reaches_extension_intact(self, monkeypatch):
monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
@@ -369,6 +406,68 @@ class TestRequestBodyLimitMiddlewareDirectly:
assert statuses == [413] # exactly one refusal; the downstream 400 was dropped
assert downstream_sent == [{"attempted": True}]
@pytest.mark.parametrize(
"path",
[
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
"/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
],
ids=["invoke", "invoke-with-response-stream"],
)
async def test_stream_trip_answers_once_in_bedrock_dialect(self, path):
"""On a metered-stream trip over a Bedrock InvokeModel route the single
refusal carries handle_bedrock_invoke's dialect, and the handler's own
413 answer (its RequestBodyTooLarge response) is discarded by the
response-once state: exactly one response reaches the wire.
"""
downstream_sent: list[dict] = []
async def downstream(scope, receive, send):
try:
while True:
m = await receive()
if m["type"] != "http.request":
break
if not m.get("more_body"):
break
except proxy_helpers.RequestBodyTooLarge:
# handle_bedrock_invoke answers RequestBodyTooLarge with its own
# 413; response-once must drop it because the ceiling already answered.
body = b'{"error": {"type": "request_too_large", "message": "from handler"}}'
await send(
{
"type": "http.response.start",
"status": 413,
"headers": [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode()),
],
}
)
await send({"type": "http.response.body", "body": body})
downstream_sent.append({"attempted": True})
return
raise AssertionError("reader should have been stopped")
sent: list[dict] = []
async def send(m):
sent.append(m)
mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
await mw(self._http_scope(path=path), self._chunked_receive([b"a" * 400] * 5), send)
statuses = [m["status"] for m in sent if m["type"] == "http.response.start"]
assert statuses == [413] # exactly one response; the handler's 413 was dropped
body = b"".join(m.get("body") or b"" for m in sent if m["type"] == "http.response.body")
assert json.loads(body) == {
"error": {
"type": "request_too_large",
"message": "Request body too large. Maximum size is 0MB",
}
}
assert downstream_sent == [{"attempted": True}]
async def test_body_under_ceiling_is_delivered_untouched(self):
held = bytearray()