mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): enforce configured TPM limits (#3350)
## Description Consume the configured token-per-minute bucket after input token counting and before upstream dispatch. Previously handlers checked only the request-per-minute bucket, so `rate_limit_tokens_per_minute` created a token bucket that was never used. Closes #3347 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Enforce input TPM in Anthropic Messages. - Enforce input TPM in OpenAI Chat Completions and Responses. - Enforce input TPM in Gemini generate-content requests. - Return 429 before upstream dispatch and record rate-limit metrics. - Add integration coverage for all four handler paths. ## Testing - [x] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ .venv/bin/pytest -q tests/test_proxy_tpm_enforcement.py 4 passed, 1 warning in 1.48s $ .venv/bin/pytest -q tests/test_proxy_tpm_enforcement.py tests/test_proxy_config_rate_limit.py tests/test_cli_proxy_env.py -k 'tpm or rate_limit' 11 passed, 67 deselected, 1 warning $ .venv/bin/ruff check <changed files> All checks passed! $ .venv/bin/ruff format --check <changed files> 4 files already formatted $ .venv/bin/mypy headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/openai.py headroom/proxy/handlers/gemini.py Success: no issues found in 3 source files ``` ## Real Behavior Proof - Environment: Linux, Python 3.14, real FastAPI proxy app and tokenizer, real local HTTP upstream, `rate_limit_tokens_per_minute=1`. - Exact command / steps: started an allowlisted local Anthropic-compatible HTTP server with a request counter, configured the proxy with TPM 1 and RPM 1000, then sent a normal `/v1/messages` request containing more than one input token. - Observed result: the proxy returned HTTP 429 before making any upstream request. ```text HTTP Request: POST http://testserver/v1/messages "HTTP/1.1 429 Too Many Requests" status=429 detail=Token rate limited. Retry after 1200.0s upstream_calls=0 ``` - Not tested: external/cloud providers; streaming-specific response behavior; distributed rate-limit backends; full repository test suite. ## Runtime Rollout Safety - Rollout-managed feature(s): proxy rate limiting. - Minimum rollout channel: N/A. - Stable/default behavior changed: only when rate limiting is enabled and the configured TPM budget is exceeded. - Kill switch / disable path: disable rate limiting or increase `rate_limit_tokens_per_minute`. - Unsafe override required: No. - Qualification impact: deployments that configured a low TPM value will now receive the intended 429 responses. - Rollback path: revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A. ## Additional Notes Documentation is unchanged because the existing TPM configuration now behaves as documented. The warning above is the existing Starlette `TestClient` deprecation warning.
This commit is contained in:
@@ -1432,6 +1432,19 @@ class AnthropicHandlerMixin:
|
||||
# conversation is CPU-bound — on-loop it froze the server (#1701).
|
||||
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
|
||||
|
||||
if self.rate_limiter:
|
||||
allowed, wait_seconds = await self.rate_limiter.check_tokens(
|
||||
rate_key, original_tokens
|
||||
)
|
||||
if not allowed:
|
||||
await self.metrics.record_rate_limited(provider=provider_name)
|
||||
await _finalize_pre_upstream()
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
|
||||
headers={"Retry-After": str(int(wait_seconds) + 1)},
|
||||
)
|
||||
|
||||
# Enterprise Security: scan request before compression
|
||||
_security_ctx = None
|
||||
if self.security:
|
||||
|
||||
@@ -532,6 +532,15 @@ class GeminiHandlerMixin:
|
||||
# Token counting (offloaded off the event loop — GH #1701)
|
||||
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
|
||||
|
||||
if self.rate_limiter:
|
||||
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
|
||||
if not allowed:
|
||||
await self.metrics.record_rate_limited(provider=provider_name)
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
|
||||
)
|
||||
|
||||
# Optimization
|
||||
transforms_applied: list[str] = []
|
||||
waste_signals_dict: dict[str, int] | None = None
|
||||
|
||||
@@ -3689,6 +3689,15 @@ class OpenAIHandlerMixin:
|
||||
# Token counting (offloaded off the event loop — GH #1701)
|
||||
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
|
||||
|
||||
if self.rate_limiter:
|
||||
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
|
||||
if not allowed:
|
||||
await self.metrics.record_rate_limited(provider=openai_chat_outcome_provider)
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
|
||||
)
|
||||
|
||||
# Hook: pre_compress
|
||||
_hook_biases = None
|
||||
# Hard per-message veto. Separate from ``_hook_biases`` because a bias is
|
||||
@@ -5735,6 +5744,15 @@ class OpenAIHandlerMixin:
|
||||
# shift when it does.
|
||||
message_input_tokens = original_tokens
|
||||
|
||||
if self.rate_limiter:
|
||||
allowed, wait_seconds = await self.rate_limiter.check_tokens(rate_key, original_tokens)
|
||||
if not allowed:
|
||||
await self.metrics.record_rate_limited(provider="openai")
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"Token rate limited. Retry after {wait_seconds:.1f}s",
|
||||
)
|
||||
|
||||
# Defaults below feed downstream telemetry and memory injection.
|
||||
# If optimization remains enabled, the Responses payload is compressed
|
||||
# later through `_compress_openai_responses_payload`.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""The configured token-per-minute bucket must be consumed by handlers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
||||
|
||||
|
||||
def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:
|
||||
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
||||
config = ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=False,
|
||||
rate_limit_enabled=True,
|
||||
rate_limit_requests_per_minute=1_000,
|
||||
rate_limit_tokens_per_minute=1,
|
||||
cost_tracking_enabled=False,
|
||||
log_requests=False,
|
||||
ccr_inject_tool=False,
|
||||
ccr_handle_responses=False,
|
||||
ccr_context_tracking=False,
|
||||
image_optimize=False,
|
||||
)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
proxy = client.app.state.proxy
|
||||
tokenizer = MagicMock()
|
||||
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
|
||||
proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
|
||||
|
||||
response = client.post(
|
||||
"/v1/messages",
|
||||
headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
|
||||
json={
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "large request"}],
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 429
|
||||
assert response.json()["detail"].startswith("Token rate limited.")
|
||||
proxy._retry_request.assert_not_awaited()
|
||||
assert len(proxy.rate_limiter._token_buckets) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "headers", "body"),
|
||||
[
|
||||
(
|
||||
"/v1/chat/completions",
|
||||
{"authorization": "Bearer test-key"},
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "large request"}],
|
||||
"stream": False,
|
||||
},
|
||||
),
|
||||
(
|
||||
"/v1/responses",
|
||||
{"authorization": "Bearer test-key"},
|
||||
{
|
||||
"model": "gpt-4o-mini",
|
||||
"input": "large request",
|
||||
"stream": False,
|
||||
},
|
||||
),
|
||||
(
|
||||
"/v1beta/models/gemini-2.5-flash:generateContent",
|
||||
{"x-goog-api-key": "test-key"},
|
||||
{
|
||||
"contents": [{"role": "user", "parts": [{"text": "large request"}]}],
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_other_handlers_reject_request_over_tpm_before_upstream(
|
||||
monkeypatch, path, headers, body
|
||||
) -> None:
|
||||
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
||||
config = ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=False,
|
||||
rate_limit_enabled=True,
|
||||
rate_limit_requests_per_minute=1_000,
|
||||
rate_limit_tokens_per_minute=1,
|
||||
cost_tracking_enabled=False,
|
||||
log_requests=False,
|
||||
ccr_inject_tool=False,
|
||||
ccr_handle_responses=False,
|
||||
ccr_context_tracking=False,
|
||||
image_optimize=False,
|
||||
)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
proxy = client.app.state.proxy
|
||||
tokenizer = MagicMock()
|
||||
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
|
||||
proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
|
||||
|
||||
response = client.post(path, headers=headers, json=body)
|
||||
|
||||
assert response.status_code == 429
|
||||
assert response.json()["detail"].startswith("Token rate limited.")
|
||||
proxy._retry_request.assert_not_awaited()
|
||||
assert len(proxy.rate_limiter._token_buckets) == 1
|
||||
Reference in New Issue
Block a user