mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cache): never batch-compress already-forwarded small Responses outputs in cache mode (#3756)
## Description
In cache mode, small Responses tool outputs below
`OPENAI_RESPONSES_ROUTER_MIN_BYTES` are forwarded raw until their batch
crosses the floor. When a later turn appends outputs and the batch
crosses the floor, `build_compression_batches` compresses the whole
batch, including outputs already forwarded raw on earlier turns. Those
bytes sit in the provider's cached prefix, so the rewrite breaks prompt
caching from the first old output onward.
Related: the batching comes from #2116. Same failure class as #3379
(fixed for Anthropic in #3380) and #3391. No open issue tracks this one.
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- In cache mode, small-output batches are skipped. Large units are
unchanged: they keep their per-unit result cache.
- Test: two turns in cache mode; turn 2 crosses the floor and turn 1's
outputs stay byte-identical.
Tradeoff: cache mode gives up small-batch savings. A narrower fix would
batch only units that are new this turn, but that needs per-conversation
forwarding state. We chose the stateless rule; happy to rework if you
prefer the narrower one.
## Testing
- [x] Unit tests pass (`pytest tests scripts/tests`): no new failures;
the 6 failures here also fail on `main`
- [x] Linting passes (`ruff check .`, `ruff format --check .`)
- [x] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
### Test Output
```text
before fix: AssertionError: assert ['{{HEADROOM_TAG_0}}unit0_0 ... unit5_11{{HEADROOM_TAG_11}}'] == []
after fix, branch merged with main (a8bd9bec), CI's lint and test steps in Docker (Python 3.12, maturin --profile ci wheel):
ruff check . All checks passed!
ruff format --check . 1648 files already formatted
mypy headroom Success: no issues found in 541 source files
pytest tests scripts/tests
main: 7 failed, 13236 passed, 613 skipped
branch: 6 failed, 13238 passed, 613 skipped
branch failures are a subset of main's
```
## Real Behavior Proof
- Environment: Linux, Python 3.12.3, `headroom-ai==0.38.0` wheel deps,
source at `main`.
- Exact command / steps: the two-turn sequence in the new test, calling
`_compress_openai_responses_live_text_units_with_router` with
`config.mode="cache"`.
- Observed result: before, turn 2 sent all six outputs (four already
forwarded) to the compressor; after, no compressor call and turn 1's
items are unchanged.
- Not tested: provider cache-read counters on a live session through
this branch. We have run this change in cache mode on Codex Responses
traffic since `aebe989`.
## Runtime Rollout Safety
- Rollout-managed feature(s): none
- Minimum rollout channel: stable
- Stable/default behavior changed: cache mode only; token mode unchanged
- Kill switch / disable path: token mode
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
## Checklist
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md`
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
@@ -33,6 +33,7 @@ from headroom.proxy.helpers import (
|
||||
)
|
||||
from headroom.proxy.identity import resolve_memory_identity
|
||||
from headroom.proxy.loopback_guard import is_loopback_host
|
||||
from headroom.proxy.modes import is_cache_mode
|
||||
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
|
||||
from headroom.proxy.upstream_guard import is_safe_upstream_url
|
||||
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
|
||||
@@ -2511,10 +2512,16 @@ class OpenAIHandlerMixin:
|
||||
)
|
||||
else:
|
||||
large_unit_indexes.append(unit_idx)
|
||||
small_batches, small_batch_skipped = build_compression_batches(
|
||||
small_batch_entries,
|
||||
min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,
|
||||
)
|
||||
if is_cache_mode(getattr(getattr(self, "config", None), "mode", "token")):
|
||||
# Appending outputs can push an old below-floor unit into a batch.
|
||||
# Preserve its first-forwarded bytes instead of retroactively
|
||||
# compressing it. Large units retain their per-unit result cache.
|
||||
small_batches, small_batch_skipped = [], small_batch_entries
|
||||
else:
|
||||
small_batches, small_batch_skipped = build_compression_batches(
|
||||
small_batch_entries,
|
||||
min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,
|
||||
)
|
||||
cache_misses: list[tuple[int, str, RoutedCompressionUnit]] = []
|
||||
cache_miss_followers: dict[str, list[int]] = {}
|
||||
for unit_idx in large_unit_indexes:
|
||||
|
||||
@@ -1300,3 +1300,53 @@ def test_openai_responses_adapter_excludes_custom_tool_beside_compressed_functio
|
||||
assert modified is True
|
||||
assert new_payload["input"][1]["output"] == excluded_output
|
||||
assert new_payload["input"][3]["output"] == "compressed tool output"
|
||||
|
||||
|
||||
def test_openai_responses_cache_mode_never_batches_old_small_outputs():
|
||||
"""Cache mode: an appended output must not pull earlier outputs into a batch.
|
||||
|
||||
Turn 1 forwards four small outputs raw (under the batch floor). Turn 2
|
||||
appends two more, crossing the floor. Batching then would rewrite the
|
||||
four outputs already in the provider's cached prefix.
|
||||
"""
|
||||
router = ContentRouter()
|
||||
calls: list[str] = []
|
||||
floor = OpenAIHandlerMixin.OPENAI_RESPONSES_ROUTER_MIN_BYTES
|
||||
outputs = [" ".join(f"unit{index}_{token}" for token in range(12)) for index in range(6)]
|
||||
assert sum(len(output.encode("utf-8")) for output in outputs[:4]) < floor
|
||||
assert sum(len(output.encode("utf-8")) for output in outputs) >= floor
|
||||
|
||||
def compress(self, content: str, **_kwargs):
|
||||
calls.append(content)
|
||||
compressed = content
|
||||
for output in outputs:
|
||||
compressed = compressed.replace(output, "x")
|
||||
return RouterCompressionResult(
|
||||
compressed=compressed,
|
||||
original=content,
|
||||
strategy_used=CompressionStrategy.KOMPRESS,
|
||||
)
|
||||
|
||||
router.compress = MethodType(compress, router)
|
||||
handler = _handler_with_router(router)
|
||||
handler.config = SimpleNamespace(mode="cache")
|
||||
|
||||
def turn(count: int) -> dict:
|
||||
return {
|
||||
"model": "gpt-5",
|
||||
"input": [
|
||||
{"type": "local_shell_call_output", "call_id": f"c{index}", "output": output}
|
||||
for index, output in enumerate(outputs[:count])
|
||||
],
|
||||
}
|
||||
|
||||
first, *_ = handler._compress_openai_responses_live_text_units_with_router(
|
||||
turn(4), model="gpt-5", request_id="req_turn_1"
|
||||
)
|
||||
second, modified, *_ = handler._compress_openai_responses_live_text_units_with_router(
|
||||
turn(6), model="gpt-5", request_id="req_turn_2"
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
assert modified is False
|
||||
assert second["input"][:4] == first["input"]
|
||||
|
||||
Reference in New Issue
Block a user