mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): preserve response cache keys across in-place hooks (#4457)
## Description When `pre_compress` mutates messages in place, the OpenAI and Anthropic handlers could store a response under messages that no longer matched the original cache lookup. Snapshotting the cache-key messages before the hook keeps repeated identical non-streaming requests eligible for a cache hit while still forwarding the hook's rewritten messages upstream. Fixes #4275 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Snapshot cache-key messages in both OpenAI and Anthropic handlers when response caching is enabled for a non-streaming request. - Add a regression covering in-place hook mutation through both provider routes. ## Testing - [x] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text uv run --extra proxy --with pytest pytest tests/test_response_cache_in_place_hook.py tests/test_openai_response_cache_key.py tests/test_proxy_semantic_cache_key_integration.py tests/test_proxy_openai_cache_key_integration.py -q 11 passed, 2 warnings Ruff check and format check on the three changed files: passed. uv run --extra proxy --with mypy mypy headroom Success across 572 source files. ``` ## Real Behavior Proof - Environment: Linux x86_64, Python 3.12.13; the real local proxy with response caching enabled and a synthetic HTTP upstream bound to loopback. Synthetic credentials were used; no provider service was contacted. - Exact steps: start the proxy and loopback stub, send the same non-streaming request twice to `/v1/chat/completions` and `/v1/messages`, and have `pre_compress` mutate the user message in place. - Observed result: both requests returned HTTP 200 with the same response ID for each provider; each provider received one upstream call, and the upstream saw `rewritten in place`. - Not tested: real OpenAI or Anthropic services, streaming requests, full upstream CI, or maintainer review. ## Runtime Rollout Safety - Rollout-managed feature(s): none. - Minimum rollout channel: stable; the existing response-cache path is corrected for eligible requests. - Stable/default behavior changed: repeated identical non-streaming requests with an in-place hook mutation can now reuse the cached response. - Kill switch / disable path: disable response caching through the existing proxy configuration. - Unsafe override required: no. - Qualification impact: the local two-provider loopback proof passed; provider-service qualification was not run. - Rollback path: revert this change. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A. ## Additional Notes No documentation change is needed for this internal cache-key correction. The full test suite, remote CI, and human review remain unverified. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4457" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end -->
This commit is contained in:
@@ -1544,7 +1544,9 @@ class AnthropicHandlerMixin:
|
||||
# it was looked up by — the cache would never hit and would fill with
|
||||
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
|
||||
# (the same reason cache_key_fields is snapshotted here, #327).
|
||||
cache_lookup_messages = messages
|
||||
cache_lookup_messages = (
|
||||
copy.deepcopy(messages) if self.cache and not stream else messages
|
||||
)
|
||||
# Response-cache partition: a cached response is only ever replayed to a
|
||||
# caller presenting the same provider credentials and principal (01-F15).
|
||||
# Snapshotted with the key fields so lookup and store agree. None means
|
||||
|
||||
@@ -3701,7 +3701,7 @@ class OpenAIHandlerMixin:
|
||||
# `messages`, but it runs before this snapshot, so its output is already
|
||||
# captured — keep this snapshot after image compression, or a reorder
|
||||
# silently reintroduces the drift.
|
||||
cache_lookup_messages = messages
|
||||
cache_lookup_messages = copy.deepcopy(messages) if self.cache and not stream else messages
|
||||
# Response-cache partition: a cached response is only ever replayed to a
|
||||
# caller presenting the same provider credentials and principal (01-F15).
|
||||
# Snapshotted with the key fields so lookup and store agree. None means
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Response-cache keys must survive in-place pre_compress mutations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from headroom.hooks import CompressionHooks # noqa: E402
|
||||
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
||||
|
||||
|
||||
class _InPlaceHooks(CompressionHooks):
|
||||
def pre_compress(self, messages, ctx):
|
||||
for message in messages:
|
||||
if message.get("role") == "user":
|
||||
message["content"] = "rewritten in place"
|
||||
return messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "path", "headers", "body", "response"),
|
||||
[
|
||||
(
|
||||
"openai",
|
||||
"/v1/chat/completions",
|
||||
{"authorization": "Bearer test-key"},
|
||||
{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "original"}]},
|
||||
{
|
||||
"id": "chatcmpl_1",
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "reply"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
|
||||
},
|
||||
),
|
||||
(
|
||||
"anthropic",
|
||||
"/v1/messages",
|
||||
{"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
|
||||
{
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "original"}],
|
||||
},
|
||||
{
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": "reply"}],
|
||||
"usage": {
|
||||
"input_tokens": 5,
|
||||
"output_tokens": 1,
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_in_place_pre_compress_mutation_keeps_response_cache_hit(
|
||||
provider: str, path: str, headers: dict, body: dict, response: dict
|
||||
) -> None:
|
||||
calls = {"n": 0}
|
||||
config = ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=True,
|
||||
rate_limit_enabled=False,
|
||||
cost_tracking_enabled=False,
|
||||
log_requests=False,
|
||||
ccr_inject_tool=False,
|
||||
ccr_handle_responses=False,
|
||||
ccr_context_tracking=False,
|
||||
image_optimize=False,
|
||||
hooks=_InPlaceHooks(),
|
||||
)
|
||||
|
||||
with TestClient(create_app(config)) as client:
|
||||
proxy = client.app.state.proxy
|
||||
|
||||
async def _fake_retry(method, url, request_headers, request_body, stream=False, **kwargs):
|
||||
calls["n"] += 1
|
||||
result = dict(response)
|
||||
result["id"] = f"{provider}_{calls['n']}"
|
||||
return httpx.Response(200, json=result)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
first = client.post(path, headers=headers, json=body)
|
||||
second = client.post(path, headers=headers, json=body)
|
||||
|
||||
assert first.status_code == second.status_code == 200
|
||||
assert calls["n"] == 1
|
||||
assert first.json() == second.json()
|
||||
Reference in New Issue
Block a user