fix(proxy): preserve response cache keys across in-place hooks (#4457)

## Description

When `pre_compress` mutates messages in place, the OpenAI and Anthropic
handlers could store a response under messages that no longer matched
the original cache lookup. Snapshotting the cache-key messages before
the hook keeps repeated identical non-streaming requests eligible for a
cache hit while still forwarding the hook's rewritten messages upstream.

Fixes #4275

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Snapshot cache-key messages in both OpenAI and Anthropic handlers when
response caching is enabled for a non-streaming request.
- Add a regression covering in-place hook mutation through both provider
routes.

## Testing

- [x] Unit tests pass (`pytest`)
- [ ] Linting passes (`ruff check .`)
- [x] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [x] Manual testing performed

### Test Output

```text
uv run --extra proxy --with pytest pytest tests/test_response_cache_in_place_hook.py tests/test_openai_response_cache_key.py tests/test_proxy_semantic_cache_key_integration.py tests/test_proxy_openai_cache_key_integration.py -q
11 passed, 2 warnings

Ruff check and format check on the three changed files: passed.
uv run --extra proxy --with mypy mypy headroom
Success across 572 source files.
```

## Real Behavior Proof

- Environment: Linux x86_64, Python 3.12.13; the real local proxy with
response caching enabled and a synthetic HTTP upstream bound to
loopback. Synthetic credentials were used; no provider service was
contacted.
- Exact steps: start the proxy and loopback stub, send the same
non-streaming request twice to `/v1/chat/completions` and
`/v1/messages`, and have `pre_compress` mutate the user message in
place.
- Observed result: both requests returned HTTP 200 with the same
response ID for each provider; each provider received one upstream call,
and the upstream saw `rewritten in place`.
- Not tested: real OpenAI or Anthropic services, streaming requests,
full upstream CI, or maintainer review.

## Runtime Rollout Safety

- Rollout-managed feature(s): none.
- Minimum rollout channel: stable; the existing response-cache path is
corrected for eligible requests.
- Stable/default behavior changed: repeated identical non-streaming
requests with an in-place hook mutation can now reuse the cached
response.
- Kill switch / disable path: disable response caching through the
existing proxy configuration.
- Unsafe override required: no.
- Qualification impact: the local two-provider loopback proof passed;
provider-service qualification was not run.
- Rollback path: revert this change.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md` — it is generated by
release-please from my Conventional Commit PR title (a CI guard enforces
this)

## Screenshots (if applicable)

N/A.

## Additional Notes

No documentation change is needed for this internal cache-key
correction. The full test suite, remote CI, and human review remain
unverified.
<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4457"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
This commit is contained in:
lorenzozane
2026-10-28 23:31:45 -05:00
committed by GitHub
parent 4a851ed254
commit 03590cd61e
3 changed files with 105 additions and 2 deletions
+3 -1
View File
@@ -1544,7 +1544,9 @@ class AnthropicHandlerMixin:
# it was looked up by — the cache would never hit and would fill with
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
# (the same reason cache_key_fields is snapshotted here, #327).
cache_lookup_messages = messages
cache_lookup_messages = (
copy.deepcopy(messages) if self.cache and not stream else messages
)
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
+1 -1
View File
@@ -3701,7 +3701,7 @@ class OpenAIHandlerMixin:
# `messages`, but it runs before this snapshot, so its output is already
# captured — keep this snapshot after image compression, or a reorder
# silently reintroduces the drift.
cache_lookup_messages = messages
cache_lookup_messages = copy.deepcopy(messages) if self.cache and not stream else messages
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
+101
View File
@@ -0,0 +1,101 @@
"""Response-cache keys must survive in-place pre_compress mutations."""
from __future__ import annotations
import httpx
import pytest
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
from headroom.hooks import CompressionHooks # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
class _InPlaceHooks(CompressionHooks):
def pre_compress(self, messages, ctx):
for message in messages:
if message.get("role") == "user":
message["content"] = "rewritten in place"
return messages
@pytest.mark.parametrize(
("provider", "path", "headers", "body", "response"),
[
(
"openai",
"/v1/chat/completions",
{"authorization": "Bearer test-key"},
{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "original"}]},
{
"id": "chatcmpl_1",
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "reply"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
},
),
(
"anthropic",
"/v1/messages",
{"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
{
"model": "claude-haiku-4-5",
"max_tokens": 64,
"messages": [{"role": "user", "content": "original"}],
},
{
"id": "msg_1",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "reply"}],
"usage": {
"input_tokens": 5,
"output_tokens": 1,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
},
},
),
],
)
def test_in_place_pre_compress_mutation_keeps_response_cache_hit(
provider: str, path: str, headers: dict, body: dict, response: dict
) -> None:
calls = {"n": 0}
config = ProxyConfig(
optimize=False,
cache_enabled=True,
rate_limit_enabled=False,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
hooks=_InPlaceHooks(),
)
with TestClient(create_app(config)) as client:
proxy = client.app.state.proxy
async def _fake_retry(method, url, request_headers, request_body, stream=False, **kwargs):
calls["n"] += 1
result = dict(response)
result["id"] = f"{provider}_{calls['n']}"
return httpx.Response(200, json=result)
proxy._retry_request = _fake_retry
first = client.post(path, headers=headers, json=body)
second = client.post(path, headers=headers, json=body)
assert first.status_code == second.status_code == 200
assert calls["n"] == 1
assert first.json() == second.json()