fix(proxy): cap cache_control breakpoints on OpenAI chat path (#4471) (#4491)

## Description

On the OpenAI `/chat/completions` path in cache mode, earlier messages
are replayed with old `cache_control` markers. The 4-marker budget guard
`enforce_cache_breakpoint_budget` now runs on that path after the mirror
step, skipped for `x-headroom-bypass`, with `client_messages=None` to
avoid the block normalizer inventing markers; `tool_calls` markers are
counted/dropped oldest-first.

Closes #4471

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- `handlers/openai.py`: run `enforce_cache_breakpoint_budget` as
last-stop guard right after the mirror step (skipped for
`x-headroom-bypass`), passing `client_messages=None` to avoid the block
normalizer inventing markers; same guard the gateway extension already
uses.
- `helpers.py`: `count_cache_breakpoints` counts
`tool_calls[*].cache_control`; the oldest-first drop pass also strips
those markers.
- Mirror step left as-is: its documented bail-out on count mismatch
stays, the budget guard now bounds the total with the bypass gate and
`client_messages=None`.

## Testing

- [x] Unit tests added/updated
- [x] Existing tests pass

```text
$ pytest tests/test_openai_message_level_cache_control.py tests/gateway/test_cache_breakpoint_budget.py tests/test_cache_breakpoint_diagnostics.py <prefix-tracker/mirror tests> -q
1 failed, 115 passed, 8 skipped in 10.69s
FAILED tests/test_cache_breakpoint_diagnostics.py::test_runtime_log_says_why_it_is_missing   # pre-existing on main: Windows WinError 183 on temp log dir
$ ruff check / ruff format --check: clean
```

## Real Behavior Proof

- Environment: Windows 11, Python uv tool env with headroom._core built
- Exact command / steps: unit-level repro of the issue body shape (4
client markers + replayed prefix carrying an old message-level and
tool_calls marker, counts differ) through
enforce_cache_breakpoint_budget
- Observed result: 6 markers in → ≤4 out, newest tail marker kept,
stats["repaired"] True
- Not tested: live LiteLLM + Vertex gateway end-to-end

## Runtime Rollout Safety

- Rollout-managed feature(s): none
- Minimum rollout channel: stable
- Stable/default behavior changed: only when a chat request exceeds 4
cache_control markers (previously a guaranteed upstream 400)
- Kill switch / disable path: none needed; under-limit requests return
by identity unchanged
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert this PR

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

🤖 Generated with [Claude Code](https://claude.com/claude-code)

<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4491"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
Parideboy
2026-10-28 23:38:31 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 JD Davis
parent 222f2d1df3
commit 55602fc1e2
3 changed files with 103 additions and 0 deletions
+18
View File
@@ -4334,6 +4334,24 @@ class OpenAIHandlerMixin:
optimized_messages, original_client_messages
)
from headroom.proxy.helpers import enforce_cache_breakpoint_budget
# Bypass (x-headroom-bypass: true, _bypass computed at ~L3458) opts out
# of ALL compression: never mutate those requests here.
# client_messages=None: passing the client list runs
# normalize_message_cache_control, whose block-level legacy fallback can
# add a marker to a list-content user message on chat requests and then
# cost a real system marker. Without it only the oldest-first drop runs
# — stale replayed markers are oldest, so they go first.
if not _bypass:
_, optimized_messages, tools, _ = enforce_cache_breakpoint_budget(
None,
optimized_messages,
tools,
client_messages=None,
request_id=request_id,
)
# Memory: inject context and tools for OpenAI requests.
#
# PR-A3 follow-up to A2: memory context now routes exclusively to
+16
View File
@@ -458,6 +458,12 @@ def count_cache_breakpoints(
if not isinstance(msg, dict):
continue
found = 1 if "cache_control" in msg else 0
# OpenAI chat shape: markers on an assistant's tool_calls entries.
tool_calls = msg.get("tool_calls")
if isinstance(tool_calls, list):
found += sum(
1 for tc in tool_calls if isinstance(tc, dict) and "cache_control" in tc
)
content = msg.get("content")
if isinstance(content, list):
for block in content:
@@ -847,6 +853,16 @@ def enforce_cache_breakpoint_budget(
budget["left"] -= 1
new_blocks.append(block)
msg = {**msg, "content": new_blocks}
tool_calls = msg.get("tool_calls")
if isinstance(tool_calls, list) and budget["left"] > 0:
new_calls = []
for tc in tool_calls:
if budget["left"] > 0 and isinstance(tc, dict) and "cache_control" in tc:
tc = {k: v for k, v in tc.items() if k != "cache_control"}
budget["left"] -= 1
new_calls.append(tc)
if new_calls != tool_calls:
msg = {**msg, "tool_calls": new_calls}
if budget["left"] > 0 and "cache_control" in msg:
msg = {k: v for k, v in msg.items() if k != "cache_control"}
budget["left"] -= 1
@@ -107,3 +107,72 @@ def test_gateway_body_is_held_to_four_markers(headroom_client) -> None:
)
assert data["messages"] == out["messages"]
assert "cache_breakpoint_budget" in data["transforms_applied"]
def test_openai_chat_replayed_markers_capped_oldest_first() -> None:
# Issue #4471: a replayed frozen prefix keeps an old message-level marker
# and the message count differs from the client's, so the mirror step
# bails; the budget guard must still cap OpenAI chat shapes at four.
client = [
{"role": "system", "content": "s1", "cache_control": CC},
{"role": "system", "content": "s2", "cache_control": CC},
{"role": "user", "content": "read f1"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "a", "type": "function", "function": {"name": "r", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "a", "content": "f1 body", "cache_control": CC},
]
tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]
replayed = [dict(m) for m in client]
replayed[3] = {
**replayed[3],
"tool_calls": [{**replayed[3]["tool_calls"][0], "cache_control": CC}],
}
replayed.insert(2, {"role": "user", "content": "earlier", "cache_control": CC})
assert count_cache_breakpoints(None, replayed, tools)["total"] == 6
_, messages, tools_out, stats = enforce_cache_breakpoint_budget(
None, replayed, tools, client_messages=None
)
assert stats["repaired"]
assert count_cache_breakpoints(None, messages, tools_out)["total"] <= 4
assert "cache_control" in messages[-1]
def test_openai_chat_budget_drops_stale_tool_marker_keeps_systems() -> None:
# Issue #4471: on the OpenAI chat path client_messages=None so
# normalize_message_cache_control's block-level legacy fallback cannot add
# a marker to a list-content user message and then cost a real system
# marker. Only the oldest-first drop runs, so stale replayed markers go
# first while system markers and the latest tool anchor are kept.
messages = [
# older replayed tool result carrying a marker: oldest, dropped first
{"role": "tool", "tool_call_id": "t1", "content": "stale", "cache_control": CC},
{"role": "system", "content": "system one", "cache_control": CC},
{"role": "system", "content": "system two", "cache_control": CC},
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
# latest tool result carrying a marker: kept
{"role": "tool", "tool_call_id": "t2", "content": "fresh", "cache_control": CC},
]
tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]
assert count_cache_breakpoints(None, messages, tools)["total"] == 5
_, out, tools_out, stats = enforce_cache_breakpoint_budget(
None, messages, tools, client_messages=None
)
assert stats["repaired"]
assert count_cache_breakpoints(None, out, tools_out)["total"] <= 4
assert "cache_control" in out[1]
assert "cache_control" in out[2]
assert "cache_control" in out[-1]
assert "cache_control" not in out[0]
assert out[3] == {"role": "user", "content": [{"type": "text", "text": "hi"}]}
def test_count_includes_tool_calls_markers() -> None:
msg = {"role": "assistant", "tool_calls": [{"id": "a", "cache_control": CC}]}
assert count_cache_breakpoints(None, [msg], None)["total"] == 1