mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): partition response cache by upstream (#3349)
## Description Include the resolved custom upstream base URL in semantic response-cache identity for Anthropic and OpenAI chat handlers. Identical prompts sent to different compatible gateways can legitimately produce different responses and must not share cached output. Closes #3346 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Add `upstream_base_url` to Anthropic response-cache key fields. - Add `upstream_base_url` to OpenAI chat response-cache key fields. - Add handler-level regressions for two custom gateways receiving identical requests. ## Testing - [x] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ .venv/bin/pytest -q tests/test_proxy_semantic_cache_key_integration.py tests/test_proxy_openai_cache_key_integration.py 8 passed, 1 warning in 2.76s $ .venv/bin/ruff check <changed files> All checks passed! $ .venv/bin/ruff format --check <changed files> 4 files already formatted $ .venv/bin/mypy headroom/proxy/handlers/anthropic.py headroom/proxy/handlers/openai.py Success: no issues found in 2 source files ``` ## Real Behavior Proof - Environment: Linux, Python 3.14, real FastAPI proxy app and real HTTP transport, two local OpenAI-compatible HTTP upstreams, semantic cache enabled. - Exact command / steps: allowlisted two loopback origins with `HEADROOM_ALLOWED_BASE_URLS`, started an HTTP server on each origin returning its gateway identity, then posted the same `/v1/chat/completions` body twice with only `x-headroom-base-url` changed. - Observed result: each request reached its selected upstream and returned that gateway's response. ```text HTTP Request: POST http://127.0.0.1:41521/v1/chat/completions "HTTP/1.0 200 OK" HTTP Request: POST http://127.0.0.1:37439/v1/chat/completions "HTTP/1.0 200 OK" responses=['gateway-A', 'gateway-B'] upstream_calls=[1, 1] ``` - Not tested: external/cloud OpenAI-compatible gateways; streaming requests; full repository test suite. ## Runtime Rollout Safety - Rollout-managed feature(s): Semantic response caching with custom upstream routing. - Minimum rollout channel: N/A. - Stable/default behavior changed: cache partitioning changes only when the resolved upstream differs. - Kill switch / disable path: disable semantic caching with `cache_enabled=False`. - Unsafe override required: No. - Qualification impact: existing cache entries remain readable only under their previous key; affected requests repopulate separate entries. - Rollback path: revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A. ## Additional Notes Documentation is unchanged because no public configuration or API contract changes. The warning above is the existing Starlette `TestClient` deprecation warning. The refreshed branch also updates the seeded cache-hit regression to include the new `upstream_base_url` key field; all hosted CI test shards passed on `6cc2d648`. --------- Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JerrettDavis
parent
ab62b9e0ad
commit
dfdc72511e
@@ -1336,6 +1336,10 @@ class AnthropicHandlerMixin:
|
||||
# semantics (#1473 review). Non-generation metadata (metadata,
|
||||
# service_tier) is intentionally excluded.
|
||||
cache_key_fields = {
|
||||
# A per-request x-headroom-base-url selects a different response
|
||||
# producer. Without it in the key, two gateways serving the same
|
||||
# model/messages can return each other's cached response (#3346).
|
||||
"upstream_base_url": upstream_base_url,
|
||||
"system": body.get("system"),
|
||||
"tools": body.get("tools"),
|
||||
"tool_choice": body.get("tool_choice"),
|
||||
|
||||
@@ -3598,6 +3598,10 @@ class OpenAIHandlerMixin:
|
||||
# review). Transport/metadata fields (stream, store, user, service_tier)
|
||||
# and the deprecated functions API are intentionally excluded.
|
||||
cache_key_fields = {
|
||||
# The resolved upstream is request-local when x-headroom-base-url
|
||||
# is present. Responses from separate gateways are not equivalent
|
||||
# even when their model/messages fields match (#3346).
|
||||
"upstream_base_url": upstream_base_url,
|
||||
"tools": body.get("tools"),
|
||||
"tool_choice": body.get("tool_choice"),
|
||||
"response_format": body.get("response_format"),
|
||||
|
||||
@@ -314,6 +314,7 @@ def test_cache_hit_never_replays_a_foreign_content_type() -> None:
|
||||
key = proxy.cache._compute_key(
|
||||
body["messages"],
|
||||
body["model"],
|
||||
upstream_base_url=None,
|
||||
system=None,
|
||||
tools=None,
|
||||
tool_choice=None,
|
||||
|
||||
@@ -114,3 +114,59 @@ def test_openai_differing_field_not_served_from_cache(field, a, b) -> None:
|
||||
assert ra2.status_code == 200
|
||||
assert _content(ra2) == "resp-1"
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_openai_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:
|
||||
"""Two OpenAI-compatible gateways must not share a response entry (#3346)."""
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.proxy.handlers.openai._resolve_openai_upstream_base",
|
||||
lambda headers: headers.get("x-headroom-base-url"),
|
||||
)
|
||||
|
||||
with _make_cached_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
calls.append(str(url))
|
||||
content = "gateway-A" if "gateway-a" in str(url) else "gateway-B"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": f"chatcmpl_{len(calls)}",
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": content},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
headers = {"authorization": "Bearer test-key"}
|
||||
body = _body()
|
||||
|
||||
first = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},
|
||||
json=body,
|
||||
)
|
||||
second = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},
|
||||
json=body,
|
||||
)
|
||||
|
||||
assert first.status_code == 200
|
||||
assert second.status_code == 200
|
||||
assert _content(first) == "gateway-A"
|
||||
assert _content(second) == "gateway-B"
|
||||
assert calls == [
|
||||
"https://gateway-a.example/v1/chat/completions",
|
||||
"https://gateway-b.example/v1/chat/completions",
|
||||
]
|
||||
|
||||
@@ -167,3 +167,62 @@ def test_different_thinking_not_served_from_cache() -> None:
|
||||
assert a2.status_code == 200
|
||||
assert _text(a2) == "resp-1"
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
def test_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:
|
||||
"""The response producer is part of cache identity (#3346)."""
|
||||
calls: list[str] = []
|
||||
|
||||
async def allow_example_upstream(_url: str) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.providers.proxy_routes.is_safe_upstream_url_async",
|
||||
allow_example_upstream,
|
||||
)
|
||||
|
||||
with _make_cached_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
calls.append(str(url))
|
||||
text = "gateway-A" if "gateway-a" in str(url) else "gateway-B"
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": f"msg_{len(calls)}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": text}],
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 3,
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
|
||||
body = _body("Answer with the gateway identity.")
|
||||
|
||||
first = client.post(
|
||||
"/v1/messages",
|
||||
headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},
|
||||
json=body,
|
||||
)
|
||||
second = client.post(
|
||||
"/v1/messages",
|
||||
headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},
|
||||
json=body,
|
||||
)
|
||||
|
||||
assert first.status_code == 200
|
||||
assert second.status_code == 200
|
||||
assert _text(first) == "gateway-A"
|
||||
assert _text(second) == "gateway-B"
|
||||
assert calls == [
|
||||
"https://gateway-a.example/v1/messages",
|
||||
"https://gateway-b.example/v1/messages",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user