fix(proxy): enforce configured TPM limits (#3350)

## Description

Consume the configured token-per-minute bucket after input token
counting and before upstream dispatch. Previously handlers checked only
the request-per-minute bucket, so `rate_limit_tokens_per_minute` created
a token bucket that was never used.

Closes #3347

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Enforce input TPM in Anthropic Messages.
- Enforce input TPM in OpenAI Chat Completions and Responses.
- Enforce input TPM in Gemini generate-content requests.
- Return 429 before upstream dispatch and record rate-limit metrics.
- Add integration coverage for all four handler paths.

## Testing

- [x] Unit tests pass (`pytest`)
- [ ] Linting passes (`ruff check .`)
- [ ] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [x] Manual testing performed

### Test Output

```text
$ .venv/bin/pytest -q tests/test_proxy_tpm_enforcement.py
4 passed, 1 warning in 1.48s

$ .venv/bin/pytest -q tests/test_proxy_tpm_enforcement.py tests/test_proxy_config_rate_limit.py tests/test_cli_proxy_env.py -k 'tpm or rate_limit'
11 passed, 67 deselected, 1 warning

$ .venv/bin/ruff check <changed files>
All checks passed!

$ .venv/bin/ruff format --check <changed files>
4 files already formatted

$ .venv/bin/mypy headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/openai.py headroom/proxy/handlers/gemini.py
Success: no issues found in 3 source files
```

## Real Behavior Proof

- Environment: Linux, Python 3.14, real FastAPI proxy app and tokenizer,
real local HTTP upstream, `rate_limit_tokens_per_minute=1`.
- Exact command / steps: started an allowlisted local
Anthropic-compatible HTTP server with a request counter, configured the
proxy with TPM 1 and RPM 1000, then sent a normal `/v1/messages` request
containing more than one input token.
- Observed result: the proxy returned HTTP 429 before making any
upstream request.

```text
HTTP Request: POST http://testserver/v1/messages "HTTP/1.1 429 Too Many Requests"
status=429
detail=Token rate limited. Retry after 1200.0s
upstream_calls=0
```

- Not tested: external/cloud providers; streaming-specific response
behavior; distributed rate-limit backends; full repository test suite.

## Runtime Rollout Safety

- Rollout-managed feature(s): proxy rate limiting.
- Minimum rollout channel: N/A.
- Stable/default behavior changed: only when rate limiting is enabled
and the configured TPM budget is exceeded.
- Kill switch / disable path: disable rate limiting or increase
`rate_limit_tokens_per_minute`.
- Unsafe override required: No.
- Qualification impact: deployments that configured a low TPM value will
now receive the intended 429 responses.
- Rollback path: revert this commit.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md` — it is generated by
release-please from my Conventional Commit PR title (a CI guard enforces
this)

## Screenshots (if applicable)

N/A.

## Additional Notes

Documentation is unchanged because the existing TPM configuration now
behaves as documented. The warning above is the existing Starlette
`TestClient` deprecation warning.
This commit is contained in:
C0sm0cats
2026-09-24 23:46:10 -05:00
committed by GitHub
parent ed08069ba0
commit 138736c96f
4 changed files with 154 additions and 0 deletions
+13
View File
@@ -1432,6 +1432,19 @@ class AnthropicHandlerMixin:
# conversation is CPU-bound — on-loop it froze the server (#1701).
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
if self.rate_limiter:
allowed, wait_seconds = await self.rate_limiter.check_tokens(
rate_key, original_tokens
)
if not allowed:
await self.metrics.record_rate_limited(provider=provider_name)
await _finalize_pre_upstream()
raise HTTPException(
status_code=429,
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
headers={"Retry-After": str(int(wait_seconds) + 1)},
)
# Enterprise Security: scan request before compression
_security_ctx = None
if self.security:
+9
View File
@@ -532,6 +532,15 @@ class GeminiHandlerMixin:
# Token counting (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
if self.rate_limiter:
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
if not allowed:
await self.metrics.record_rate_limited(provider=provider_name)
raise HTTPException(
status_code=429,
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
)
# Optimization
transforms_applied: list[str] = []
waste_signals_dict: dict[str, int] | None = None
+18
View File
@@ -3689,6 +3689,15 @@ class OpenAIHandlerMixin:
# Token counting (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
if self.rate_limiter:
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
if not allowed:
await self.metrics.record_rate_limited(provider=openai_chat_outcome_provider)
raise HTTPException(
status_code=429,
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
)
# Hook: pre_compress
_hook_biases = None
# Hard per-message veto. Separate from ``_hook_biases`` because a bias is
@@ -5735,6 +5744,15 @@ class OpenAIHandlerMixin:
# shift when it does.
message_input_tokens = original_tokens
if self.rate_limiter:
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
if not allowed:
await self.metrics.record_rate_limited(provider="openai")
raise HTTPException(
status_code=429,
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
)
# Defaults below feed downstream telemetry and memory injection.
# If optimization remains enabled, the Responses payload is compressed
# later through `_compress_openai_responses_payload`.
+114
View File
@@ -0,0 +1,114 @@
"""The configured token-per-minute bucket must be consumed by handlers."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
import pytest
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=True,
rate_limit_requests_per_minute=1_000,
rate_limit_tokens_per_minute=1,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
)
with TestClient(create_app(config)) as client:
proxy = client.app.state.proxy
tokenizer = MagicMock()
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
response = client.post(
"/v1/messages",
headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
json={
"model": "claude-haiku-4-5",
"max_tokens": 64,
"messages": [{"role": "user", "content": "large request"}],
"stream": False,
},
)
assert response.status_code == 429
assert response.json()["detail"].startswith("Token rate limited.")
proxy._retry_request.assert_not_awaited()
assert len(proxy.rate_limiter._token_buckets) == 1
@pytest.mark.parametrize(
("path", "headers", "body"),
[
(
"/v1/chat/completions",
{"authorization": "Bearer test-key"},
{
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "large request"}],
"stream": False,
},
),
(
"/v1/responses",
{"authorization": "Bearer test-key"},
{
"model": "gpt-4o-mini",
"input": "large request",
"stream": False,
},
),
(
"/v1beta/models/gemini-2.5-flash:generateContent",
{"x-goog-api-key": "test-key"},
{
"contents": [{"role": "user", "parts": [{"text": "large request"}]}],
},
),
],
)
def test_other_handlers_reject_request_over_tpm_before_upstream(
monkeypatch, path, headers, body
) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=True,
rate_limit_requests_per_minute=1_000,
rate_limit_tokens_per_minute=1,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
)
with TestClient(create_app(config)) as client:
proxy = client.app.state.proxy
tokenizer = MagicMock()
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
response = client.post(path, headers=headers, json=body)
assert response.status_code == 429
assert response.json()["detail"].startswith("Token rate limited.")
proxy._retry_request.assert_not_awaited()
assert len(proxy.rate_limiter._token_buckets) == 1