mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 18:41:07 +08:00
## Description The proxy response-cache key policy strips `cache_control` recursively at every dict depth, with a single carve-out for JSON Schema `properties` names (#327). But `cache_control` is overloaded: on a content block, a `system` entry or a tool definition it is a transport prompt-cache directive, while inside tool arguments and JSON Schema literals it is ordinary application data the model actually sees. So two requests that differ only in that data get the same key. With `cache_enabled` on by default, the second caller is replayed the first caller's response for the full TTL. Reproduced on `main` (`e995eb9f`) against the real `SemanticCache` API, no mock: ```python msgs = [{"role": "assistant", "content": [{"type": "tool_use", "id": "c1", "name": "configure", "input": {"cache_control": "first"}}]}] changed = copy.deepcopy(msgs); changed[0]["content"][0]["input"]["cache_control"] = "second" c = SemanticCache() await c.set(msgs, "m", b'{"id":"first"}', {}, partition="caller") print("replayed:", (await c.get(changed, "m", partition="caller")) is not None) ``` Before (this branch's parent, `e995eb9f`): ```text changed tool payload replayed: True changed schema replayed: True ``` Closes #4272 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `headroom/proxy/semantic_cache_key_policy.py`: `strip_cache_control` no longer recurses into fields whose value is opaque payload — `input`, `arguments`, `input_schema`, `parameters`, `schema`, `properties`, `const`, `default`, `enum`, `example`, `examples`, `text` — so a `cache_control`-named field there stays in the key. Everywhere else (content blocks, `system`, tool definitions) it is still dropped, so a moved transport breakpoint still does not fragment the key. The narrower `properties`-only carve-out falls out of the general rule. - `tests/test_proxy_semantic_cache_key_policy.py`: covers the mandatory-partition precondition (`ValueError` when no partition is passed), which was the one uncovered line Codecov flagged in this change. - `tests/test_semantic_cache_literal_payloads.py`: new regression suite covering tool `input`, tool `input_schema` `default`, and `response_format` schema `const`, each through the real `SemanticCache` (changed request misses, original still hits), plus a guard for the decoded-dict `function.arguments` shape Headroom accepts (`handlers/openai.py` parses either), a guard that OpenAI's JSON-*string* `arguments` is passed through verbatim, and guards that transport `cache_control` is still ignored and nested transport annotations are still dropped. One pure module changed; no provider, no native rebuild. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed `mypy` is unchecked because the pre-commit hook fails on 11 pre-existing Windows-only `attr-defined` errors in files this PR does not touch (`headroom/fileperms.py`, `headroom/ccr/mcp_server.py`, `headroom/proxy/interceptors/astgrep.py`: `os.fchmod`, `os.O_DIRECTORY`, `fcntl`); neither changed file produces a mypy error. ### Test Output Regression tests fail before the fix. With `headroom/proxy/semantic_cache_key_policy.py` reverted and the new tests kept: ```text $ git show HEAD~1:headroom/proxy/semantic_cache_key_policy.py > headroom/proxy/semantic_cache_key_policy.py $ uv run pytest tests/test_semantic_cache_literal_payloads.py -q FAILED tests/test_semantic_cache_literal_payloads.py::test_literal_cache_control_does_not_alias_cached_responses[tool_input] FAILED ... [tool_arguments_dict] FAILED ... [tool_schema] FAILED ... [response_schema] FAILED tests/test_semantic_cache_literal_payloads.py::test_literal_cache_control_changes_the_key[tool_input] FAILED ... [tool_arguments_dict] FAILED ... [tool_schema] FAILED ... [response_schema] FAILED tests/test_semantic_cache_literal_payloads.py::test_strip_cache_control_still_drops_nested_transport_annotations 9 failed, 3 passed in 0.64s ``` The three that pass against the old policy are the guards: the OpenAI string-`arguments` case (a string is never rewritten), the schema-property-name case (#327), and the moved-transport-breakpoint case. After the fix: ```text $ uv run pytest tests/test_semantic_cache_literal_payloads.py tests/test_proxy_semantic_cache_key_policy.py \ tests/test_semantic_cache_key_policy.py tests/test_proxy_response_cache_partition.py \ tests/test_proxy_semantic_cache_key.py tests/test_proxy_semantic_cache_key_integration.py \ tests/test_openai_response_cache_key.py tests/test_proxy_openai_cache_key_integration.py -q 94 passed, 1 warning in 28.39s $ uv run pytest <the four policy/partition files> -q \ --cov=headroom.proxy.semantic_cache_key_policy --cov-report=term-missing --cov-branch Name Stmts Miss Branch BrPart Cover headroom\proxy\semantic_cache_key_policy.py 54 0 16 0 100% ``` $ uv run pytest tests/test_cache/test_semantic.py tests/test_openai_response_cache_key.py \ tests/test_proxy_openai_cache_key_integration.py tests/test_proxy_response_cache_store_guard.py \ tests/test_proxy_semantic_cache_key.py tests/test_proxy_semantic_cache_key_integration.py \ tests/test_shared_context.py -q 133 passed, 1 warning in 34.11s $ uv run pytest tests/test_cache_control_move_bust.py tests/test_anthropic_fork_lineage_cache.py \ tests/test_cross_turn_cache_safety.py tests/test_cache/test_prefix_tracker.py -q 113 passed, 1 warning in 6.21s $ uv run ruff check headroom/proxy/semantic_cache_key_policy.py tests/test_semantic_cache_literal_payloads.py All checks passed! $ uv run ruff format --check <both files> 1 file already formatted ``` ## Real Behavior Proof - Environment: Windows 11, `uv run` against the project venv (Python 3.13.11), branch `fix/4272-literal-payload-cache-keys` cut from upstream `main` `e995eb9f`. No mocked cache and no provider: the reproducer calls the real `SemanticCache.set`/`get` in `headroom/proxy/semantic_cache.py`. - Exact command / steps: run the reproducer above as a script under `uv run python` — store a response under a tool input containing `{"cache_control": "first"}`, deep-copy the messages, change that value to `"second"`, then `get` with the changed messages; repeat for a tool `input_schema` `default` and a `response_format` schema `const`. - Observed result: before the fix the changed request hit the cache in both cases (`changed tool payload replayed: True`, `changed schema replayed: True`). After the fix both print `False`, and the original request still hits (asserted in each regression test), so the key was not merely widened. - Not tested: the real OpenAI HTTP path with a decoded-dict `arguments` field, which is a nonstandard client encoding (`arguments` is normally a JSON string, covered by `test_openai_arguments_string_is_not_rewritten`); the full `make ci-precheck` (Rust/native rebuild) and the end-to-end proxy HTTP path; the change is confined to the pure key policy, which the tests above exercise directly through the real `SemanticCache`. `headroom/cache/prefix_tracker.py` has the same blunt recursive strip, but it serves prefix-content equality rather than response replay and is left as a separate follow-up. ## Runtime Rollout Safety - Rollout-managed feature(s): none; the proxy response cache (`cache_enabled`) is not a rollout-gated feature. - Minimum rollout channel: n/a - Stable/default behavior changed: yes, deliberately — for requests whose tool data or JSON Schema values contain a `cache_control`-named field, two distinct requests that used to share one key now get distinct keys. Requests without such a field produce the same key as before, so hit rate there is unchanged. - Kill switch / disable path: `cache_enabled = false`, or the existing proxy config for the response cache; no new switch. - Unsafe override required: no - Qualification impact: none; no rollout-managed surface touched. - Rollback path: revert this commit. Cache keys are process-local (`SemanticCache` is in-memory), so a revert needs no cache migration or invalidation. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation — not needed: the key policy's docstrings describe the new rule, and no user-facing docs claim `cache_control` is stripped from payloads. - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Additional Notes `headroom/cache/prefix_tracker.py:165` has the same blunt recursive strip, used for prefix-content equality rather than response replay. Worth a follow-up, deliberately out of scope here so the response-replay fix stays a one-module change. --------- Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
222 lines
8.3 KiB
Python
222 lines
8.3 KiB
Python
"""Literal ``cache_control`` fields inside model-visible payloads (#4272).
|
|
|
|
``cache_control`` is overloaded: a transport prompt-cache directive on content
|
|
blocks, and ordinary application data inside tool inputs, JSON Schema bodies and
|
|
their literal values. Stripping it everywhere made two requests that differ in
|
|
model-visible data share one key, so the second replayed the first's response.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
|
|
import pytest
|
|
|
|
from headroom.proxy.semantic_cache import SemanticCache
|
|
from headroom.proxy.semantic_cache_key_policy import (
|
|
compute_semantic_cache_key,
|
|
strip_cache_control,
|
|
)
|
|
|
|
MODEL = "fixture"
|
|
PARTITION = "caller"
|
|
RESPONSE = b'{"id":"first-response","choices":[{"message":{"content":"first"}}]}'
|
|
|
|
# Fields whose value Headroom may legitimately receive as a decoded dict and
|
|
# where ``cache_control`` is application data, not a transport directive. A JSON
|
|
# *string* value (what OpenAI actually sends for ``arguments``) was never at risk:
|
|
# the stripper leaves strings alone at any depth.
|
|
LITERAL_DICT_LOCATIONS = (
|
|
"tool_input", # Anthropic tool_use input
|
|
"tool_arguments_dict", # OpenAI function.arguments, decoded to a dict
|
|
"tool_schema", # Anthropic input_schema default
|
|
"response_schema", # OpenAI response_format json_schema const
|
|
)
|
|
|
|
|
|
def _payloads(location: str) -> tuple[list[dict], dict]:
|
|
"""(messages, key_fields) holding a literal ``cache_control`` at ``location``."""
|
|
messages: list[dict] = [{"role": "user", "content": "Configure the application."}]
|
|
fields: dict = {}
|
|
if location == "tool_input":
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": [
|
|
{
|
|
"type": "tool_use",
|
|
"id": "call1",
|
|
"name": "configure",
|
|
"input": {"cache_control": "first"},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
elif location == "tool_arguments_dict":
|
|
# OpenAI sends ``arguments`` as a JSON *string*, which the old stripper
|
|
# left alone (see the string guard below). Headroom also tolerates a
|
|
# decoded dict there -- ``handlers/openai.py`` parses either shape -- and
|
|
# for that shape the old stripper did erase the field, so this is the
|
|
# shape that demonstrates the OpenAI-path regression.
|
|
messages = [
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call1",
|
|
"type": "function",
|
|
"function": {"name": "configure", "arguments": {"cache_control": "first"}},
|
|
}
|
|
],
|
|
}
|
|
]
|
|
elif location == "tool_schema":
|
|
fields = {
|
|
"tools": [
|
|
{
|
|
"name": "configure",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"value": {"type": "object", "default": {"cache_control": "first"}}
|
|
},
|
|
},
|
|
}
|
|
]
|
|
}
|
|
elif location == "response_schema":
|
|
fields = {
|
|
"response_format": {
|
|
"type": "json_schema",
|
|
"json_schema": {"name": "answer", "schema": {"const": {"cache_control": "first"}}},
|
|
}
|
|
}
|
|
else: # pragma: no cover - guards the parametrization
|
|
raise AssertionError(f"unknown location {location}")
|
|
return messages, fields
|
|
|
|
|
|
def _with_changed_value(messages: list[dict], fields: dict, location: str) -> tuple[list, dict]:
|
|
messages, fields = copy.deepcopy((messages, fields))
|
|
if location == "tool_input":
|
|
messages[0]["content"][0]["input"]["cache_control"] = "second"
|
|
elif location == "tool_arguments_dict":
|
|
messages[0]["tool_calls"][0]["function"]["arguments"]["cache_control"] = "second"
|
|
elif location == "tool_schema":
|
|
fields["tools"][0]["input_schema"]["properties"]["value"]["default"]["cache_control"] = (
|
|
"second"
|
|
)
|
|
else:
|
|
fields["response_format"]["json_schema"]["schema"]["const"]["cache_control"] = "second"
|
|
return messages, fields
|
|
|
|
|
|
@pytest.mark.parametrize("location", LITERAL_DICT_LOCATIONS)
|
|
@pytest.mark.asyncio
|
|
async def test_literal_cache_control_does_not_alias_cached_responses(location: str) -> None:
|
|
messages, fields = _payloads(location)
|
|
changed_messages, changed_fields = _with_changed_value(messages, fields, location)
|
|
|
|
cache = SemanticCache()
|
|
await cache.set(messages, MODEL, RESPONSE, {}, partition=PARTITION, **fields)
|
|
|
|
# The changed request must miss: its tool data / schema really differs.
|
|
assert await cache.get(changed_messages, MODEL, partition=PARTITION, **changed_fields) is None
|
|
# ...and the original still hits, so the fix does not merely widen the key.
|
|
original = await cache.get(messages, MODEL, partition=PARTITION, **fields)
|
|
assert original is not None and original.response_body == RESPONSE
|
|
|
|
|
|
@pytest.mark.parametrize("location", LITERAL_DICT_LOCATIONS)
|
|
def test_literal_cache_control_changes_the_key(location: str) -> None:
|
|
messages, fields = _payloads(location)
|
|
changed_messages, changed_fields = _with_changed_value(messages, fields, location)
|
|
assert compute_semantic_cache_key(
|
|
messages, MODEL, partition=PARTITION, **fields
|
|
) != compute_semantic_cache_key(changed_messages, MODEL, partition=PARTITION, **changed_fields)
|
|
|
|
|
|
def test_openai_arguments_string_is_not_rewritten() -> None:
|
|
"""OpenAI sends ``arguments`` as a JSON string; it must pass through verbatim.
|
|
|
|
This shape was never aliased by the old stripper (strings are returned
|
|
as-is at any depth), so it is a regression guard, not a bug demonstration.
|
|
"""
|
|
first = '{"cache_control": "first"}'
|
|
second = '{"cache_control": "second"}'
|
|
payload = {"function": {"name": "configure", "arguments": first}}
|
|
assert strip_cache_control(payload) == payload
|
|
assert compute_semantic_cache_key(
|
|
[{"role": "assistant", "content": None, "tool_calls": [{"id": "c", **payload}]}],
|
|
MODEL,
|
|
partition=PARTITION,
|
|
) != compute_semantic_cache_key(
|
|
[
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [{"id": "c", "function": {"name": "configure", "arguments": second}}],
|
|
}
|
|
],
|
|
MODEL,
|
|
partition=PARTITION,
|
|
)
|
|
|
|
|
|
def test_transport_cache_control_is_still_ignored() -> None:
|
|
"""A moved breakpoint must not fragment the key (#327 intent)."""
|
|
block = {"type": "text", "text": "sys"}
|
|
with_block = [{**block, "cache_control": {"type": "ephemeral"}}]
|
|
assert compute_semantic_cache_key(
|
|
[{"role": "user", "content": "hello"}],
|
|
MODEL,
|
|
partition=PARTITION,
|
|
system=with_block,
|
|
tools=[{"name": "read", "cache_control": {"type": "ephemeral"}}],
|
|
) == compute_semantic_cache_key(
|
|
[{"role": "user", "content": "hello"}],
|
|
MODEL,
|
|
partition=PARTITION,
|
|
system=[block],
|
|
tools=[{"name": "read"}],
|
|
)
|
|
|
|
|
|
def test_strip_cache_control_keeps_schema_property_named_cache_control() -> None:
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"cache_control": {"type": "boolean", "description": "Flush the application cache"}
|
|
},
|
|
}
|
|
assert strip_cache_control(schema) == schema
|
|
|
|
|
|
def test_strip_cache_control_still_drops_nested_transport_annotations() -> None:
|
|
assert strip_cache_control(
|
|
{
|
|
"cache_control": "drop-root",
|
|
"messages": [
|
|
{
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "tool_use", "input": {"cache_control": "keep"}},
|
|
{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}},
|
|
],
|
|
"cache_control": {"type": "ephemeral"},
|
|
}
|
|
],
|
|
}
|
|
) == {
|
|
"messages": [
|
|
{
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "tool_use", "input": {"cache_control": "keep"}},
|
|
{"type": "text", "text": "hi"},
|
|
],
|
|
}
|
|
]
|
|
}
|