mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description When `pre_compress` mutates messages in place, the OpenAI and Anthropic handlers could store a response under messages that no longer matched the original cache lookup. Snapshotting the cache-key messages before the hook keeps repeated identical non-streaming requests eligible for a cache hit while still forwarding the hook's rewritten messages upstream. Fixes #4275 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Snapshot cache-key messages in both OpenAI and Anthropic handlers when response caching is enabled for a non-streaming request. - Add a regression covering in-place hook mutation through both provider routes. ## Testing - [x] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text uv run --extra proxy --with pytest pytest tests/test_response_cache_in_place_hook.py tests/test_openai_response_cache_key.py tests/test_proxy_semantic_cache_key_integration.py tests/test_proxy_openai_cache_key_integration.py -q 11 passed, 2 warnings Ruff check and format check on the three changed files: passed. uv run --extra proxy --with mypy mypy headroom Success across 572 source files. ``` ## Real Behavior Proof - Environment: Linux x86_64, Python 3.12.13; the real local proxy with response caching enabled and a synthetic HTTP upstream bound to loopback. Synthetic credentials were used; no provider service was contacted. - Exact steps: start the proxy and loopback stub, send the same non-streaming request twice to `/v1/chat/completions` and `/v1/messages`, and have `pre_compress` mutate the user message in place. - Observed result: both requests returned HTTP 200 with the same response ID for each provider; each provider received one upstream call, and the upstream saw `rewritten in place`. - Not tested: real OpenAI or Anthropic services, streaming requests, full upstream CI, or maintainer review. ## Runtime Rollout Safety - Rollout-managed feature(s): none. - Minimum rollout channel: stable; the existing response-cache path is corrected for eligible requests. - Stable/default behavior changed: repeated identical non-streaming requests with an in-place hook mutation can now reuse the cached response. - Kill switch / disable path: disable response caching through the existing proxy configuration. - Unsafe override required: no. - Qualification impact: the local two-provider loopback proof passed; provider-service qualification was not run. - Rollback path: revert this change. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A. ## Additional Notes No documentation change is needed for this internal cache-key correction. The full test suite, remote CI, and human review remain unverified. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4457" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end -->
102 lines
3.2 KiB
Python
102 lines
3.2 KiB
Python
"""Response-cache keys must survive in-place pre_compress mutations."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
pytest.importorskip("fastapi")
|
|
|
|
from fastapi.testclient import TestClient # noqa: E402
|
|
|
|
from headroom.hooks import CompressionHooks # noqa: E402
|
|
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
|
|
|
|
|
|
class _InPlaceHooks(CompressionHooks):
|
|
def pre_compress(self, messages, ctx):
|
|
for message in messages:
|
|
if message.get("role") == "user":
|
|
message["content"] = "rewritten in place"
|
|
return messages
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("provider", "path", "headers", "body", "response"),
|
|
[
|
|
(
|
|
"openai",
|
|
"/v1/chat/completions",
|
|
{"authorization": "Bearer test-key"},
|
|
{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "original"}]},
|
|
{
|
|
"id": "chatcmpl_1",
|
|
"object": "chat.completion",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": "reply"},
|
|
"finish_reason": "stop",
|
|
}
|
|
],
|
|
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
|
|
},
|
|
),
|
|
(
|
|
"anthropic",
|
|
"/v1/messages",
|
|
{"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
|
|
{
|
|
"model": "claude-haiku-4-5",
|
|
"max_tokens": 64,
|
|
"messages": [{"role": "user", "content": "original"}],
|
|
},
|
|
{
|
|
"id": "msg_1",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [{"type": "text", "text": "reply"}],
|
|
"usage": {
|
|
"input_tokens": 5,
|
|
"output_tokens": 1,
|
|
"cache_read_input_tokens": 0,
|
|
"cache_creation_input_tokens": 0,
|
|
},
|
|
},
|
|
),
|
|
],
|
|
)
|
|
def test_in_place_pre_compress_mutation_keeps_response_cache_hit(
|
|
provider: str, path: str, headers: dict, body: dict, response: dict
|
|
) -> None:
|
|
calls = {"n": 0}
|
|
config = ProxyConfig(
|
|
optimize=False,
|
|
cache_enabled=True,
|
|
rate_limit_enabled=False,
|
|
cost_tracking_enabled=False,
|
|
log_requests=False,
|
|
ccr_inject_tool=False,
|
|
ccr_handle_responses=False,
|
|
ccr_context_tracking=False,
|
|
image_optimize=False,
|
|
hooks=_InPlaceHooks(),
|
|
)
|
|
|
|
with TestClient(create_app(config)) as client:
|
|
proxy = client.app.state.proxy
|
|
|
|
async def _fake_retry(method, url, request_headers, request_body, stream=False, **kwargs):
|
|
calls["n"] += 1
|
|
result = dict(response)
|
|
result["id"] = f"{provider}_{calls['n']}"
|
|
return httpx.Response(200, json=result)
|
|
|
|
proxy._retry_request = _fake_retry
|
|
first = client.post(path, headers=headers, json=body)
|
|
second = client.post(path, headers=headers, json=body)
|
|
|
|
assert first.status_code == second.status_code == 200
|
|
assert calls["n"] == 1
|
|
assert first.json() == second.json()
|