mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cache): preserve the forwarded OpenAI chat prefix across turns (#3757)
## Description In cache mode, `/v1/chat/completions` changes its forwarded prefix every turn, so the provider's prompt cache misses from last turn's tool result onward. Two causes, and each one alone busts the prefix: ### Tracker originals (all modes) - **Write:** tracker writes omit `original_messages` - **Stored:** compressed bytes, as if the client's - **Next turn:** `finalize_turn` finds no match - **Replay:** none ### Restore after replay (cache mode) - **Order:** restore runs after `finalize_turn` - **Effect:** replayed bytes revert to raw ### Fix - **Originals:** passed on every chat tracker write - **Order:** restore, then `finalize_turn` (cache) - **Token mode:** shared replay now sees originals Folds in #3758: the live run below showed neither half fixes the prefix alone. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - **Fix:** restore, then `finalize_turn` - **Fix:** originals on every chat tracker write - **Test:** 3-turn lifecycle, buffered and stream - **Test:** originals reach the tracker, per path - **Test:** replay survives restore - **Test:** partial replay, block append, divergence ## Testing - [x] Unit tests pass (`pytest`): no new failures; 8 fail on `main` too - [x] Linting passes (`ruff check .`), plus `ruff format --check .` - [x] Type checking passes (`mypy headroom`): `--ignore-missing-imports`, mypy 1.20.2 as in CI - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text test_openai_cache_mode_forwards_last_turns_bytes_unchanged[buffered|stream]: maind17ac6a12 failed restore fix only 2 failed originals fix only 2 failed both 2 passed E At index 2 diff: {'role': 'tool', 'tool_call_id': 'call_1', 'content': 'build 1 log line ... E - 'content': '[compressed call_1]', pytest, full suite: maind17ac6a18 failed, 12961 passed, 591 skipped, 6137 warnings branch 8 failed, 12972 passed, 591 skipped, 6137 warnings same 8 fail on both: tests/test_proxy_health.py (4 readyz Kompress tests) tests/test_read_maturation_handler_nobust.py (3) tests/test_upstream_guard.py::test_no_route_forwards_to_a_loopback_upstream ruff check .: All checks passed! ruff format --check .: 1634 files already formatted mypy 1.20.2 headroom --ignore-missing-imports: Success: no issues found in 539 source files (same on main) ``` ## Real Behavior Proof - Environment: Linux, Python 3.12.3, `gpt-6-sol`, Kompress on ONNX Runtime - Exact command / steps: see Commands below - Observed result: see table - Not tested: `/v1/responses`; token mode live (shares the originals change); backend paths live ### Setup 1. **Trees:** `main`, restore, originals, both 2. **Proxy:** one per tree, own workspace 3. **Client:** `repro-3757.py` (stdlib), below 4. **Head:** ~9 KB system prompt 5. **Salt:** unique per run: new session, cold cache ### Commands Per tree, from its checkout: ```bash HEADROOM_WORKSPACE_DIR=ws PYTHONPATH=. headroom proxy --mode cache --port 8787 --log-file ws/requests.jsonl --log-messages ``` Client, buffered then streaming: ```bash export OPENAI_API_KEY=... OPENAI_BASE_URL=... # upstream, sent as x-headroom-base-url python3 repro-3757.py --proxy http://127.0.0.1:8787 --model gpt-6-sol --log ws/requests.jsonl python3 repro-3757.py --proxy http://127.0.0.1:8787 --model gpt-6-sol --log ws/requests.jsonl --stream ``` ### `repro-3757.py` ```python """Send a 4-turn tool-calling chat through a headroom proxy; print usage per turn. Each turn adds a user ask, an assistant tool call, and a ~48 KB JSON log result behind a ~9 KB system prompt. --salt leads the system prompt, so each run is its own headroom session and a cold provider cache. With --log (the proxy's --log-file, run with --log-messages), each turn also compares the forwarded messages with last turn's. """ import argparse, json, os, random, time, urllib.request, uuid SERVICES = ["checkout", "billing", "search", "inventory", "auth", "shipping", "ledger", "notify"] TOOL = {"type": "function", "function": { "name": "read_log", "description": "Read one service's structured request log.", "parameters": {"type": "object", "properties": {"service": {"type": "string"}}, "required": ["service"]}}} def system_prompt(salt): rules = (f"Rule {i:03d}: when a {SERVICES[i % 8]} request exceeds its latency budget, check " "the upstream pool, the retry policy, and the last deploy before paging on-call." for i in range(60)) return f"Run {salt}. You are an SRE assistant reviewing service logs.\n" + "\n".join(rules) def tool_log(turn): rng, svc = random.Random(3757_3758 + turn), SERVICES[turn % 8] rows = [] for i in range(300): err = i in (41, 173, 260) rows.append({ "ts": f"2026-09-25T10:{(i // 60) % 60:02d}:{i % 60:02d}.{rng.randint(0, 999):03d}Z", "service": svc, "level": "ERROR" if err else "INFO", "route": f"/api/v1/{svc}/{rng.choice(['get', 'list', 'update'])}", "status": 502 if err else 200, "latency_ms": rng.randint(900, 4000) if err else rng.randint(8, 60), "msg": f"upstream reset by peer (pool={svc}-db)" if err else "request ok"}) return json.dumps(rows) def forwarded(log, n): for _ in range(40): # streaming logs just after [DONE] lines = open(log).read().splitlines() if os.path.exists(log) else [] if len(lines) >= n: entry = json.loads(lines[n - 1]) return entry.get("compressed_messages") or entry.get("request_messages"), entry time.sleep(0.25) return None, {} def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--proxy", required=True, help="headroom URL, e.g. http://127.0.0.1:8787") ap.add_argument("--model", required=True) ap.add_argument("--salt", default=uuid.uuid4().hex[:12], help="default: random") ap.add_argument("--stream", action="store_true", help="stream, with include_usage") ap.add_argument("--turns", type=int, default=4) ap.add_argument("--upstream", default=os.environ.get("OPENAI_BASE_URL"), help="sent as x-headroom-base-url (env: OPENAI_BASE_URL)") ap.add_argument("--api-key", default=os.environ.get("OPENAI_API_KEY"), help="env: OPENAI_API_KEY") ap.add_argument("--log", help="proxy --log-file path, for the prefix check") a = ap.parse_args() headers = {"authorization": f"Bearer {a.api_key}", "content-type": "application/json"} if a.upstream: headers["x-headroom-base-url"] = a.upstream msgs = [{"role": "system", "content": system_prompt(a.salt)}] base = len(open(a.log).read().splitlines()) if a.log and os.path.exists(a.log) else 0 prev = None for turn in range(1, a.turns + 1): svc, cid = SERVICES[turn % 8], f"call_{turn:02d}" msgs += [{"role": "user", "content": f"Turn {turn}: check the {svc} log. Reply in one short sentence."}, {"role": "assistant", "content": None, "tool_calls": [{"id": cid, "type": "function", "function": {"name": "read_log", "arguments": json.dumps({"service": svc})}}]}, {"role": "tool", "tool_call_id": cid, "content": tool_log(turn)}] body = {"model": a.model, "messages": msgs, "tools": [TOOL], "tool_choice": "none"} if a.stream: body |= {"stream": True, "stream_options": {"include_usage": True}} req = urllib.request.Request(f"{a.proxy.rstrip('/')}/v1/chat/completions", json.dumps(body).encode(), headers) with urllib.request.urlopen(req, timeout=300) as resp: text = resp.read().decode() if a.stream: chunks = [json.loads(l[5:]) for l in text.splitlines() if l.startswith("data:") and l[5:].strip() not in ("", "[DONE]")] reply = "".join((c.get("delta") or {}).get("content") or "" for ch in chunks for c in ch.get("choices") or []) usage = next((ch["usage"] for ch in reversed(chunks) if ch.get("usage")), {}) else: data = json.loads(text) reply, usage = data["choices"][0]["message"].get("content") or "", data.get("usage") or {} msgs.append({"role": "assistant", "content": reply}) cached = (usage.get("prompt_tokens_details") or {}).get("cached_tokens", 0) line = f"turn {turn} prompt {usage.get('prompt_tokens', 0):>7,} cached {cached:>7,}" if a.log: sent, entry = forwarded(a.log, base + turn) if not sent: state = "unavailable (no log entry)" elif prev is None: state = "first turn" else: diff = next((i for i, (x, y) in enumerate(zip(prev, sent)) if x != y), None if len(sent) >= len(prev) else len(sent)) state = f"changed at msg {diff}/{len(prev)}" if diff is not None else f"stable {len(prev)}/{len(prev)}" line += f" saved {entry.get('tokens_saved')} prefix {state}" prev = sent print(line, flush=True) if __name__ == "__main__": main() ``` ### Each turn 1. **Ask:** user message 2. **Tool:** call returns ~48 KB JSON log 3. **Cached:** `prompt_tokens_details.cached_tokens` 4. **Saved:** `tokens_saved`, proxy log 5. **Prefix:** forwarded vs last turn's ### Result, turn 4, buffered | | `main`d17ac6a1| restore only | originals only | both | | --- | --- | --- | --- | --- | | Prompt tokens | 69,848 | 69,862 | 69,856 | 42,944 | | Cached tokens | 40,448 (58%) | 40,448 (58%) | 40,448 (58%) | 32,768 (76%) | | Tokens saved | 9,214 | 9,214 | 9,214 | 37,226 | | Forwarded prefix | changed at msg 11/12 | changed at msg 11/12 | changed at msg 11/12 | stable 12/12 | Turn-4 streaming results: same cached tokens, saved tokens, and prefix result; prompt tokens within 18. Earlier turns differ between modes, e.g. `main` turn 2 cached 2,048 buffered vs 0 streaming. In this workload, unchanged history that the overlay validates stays byte-stable. Divergent history is not replayed, by design. ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Minimum rollout channel: Stable. - Stable/default behavior changed: Yes, OpenAI chat only. All modes: the prefix tracker stores client originals, so the shared replay can match them. Cache mode only: restore runs before replay. - Kill switch / disable path: None dedicated. - Unsafe override required: No. - Qualification impact: None. - Rollback path: Revert. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes (no new failures) - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) **main** https://github.com/user-attachments/assets/46dcd2a1-7246-4d0b-aa33-256bbe5109da **both fixes** (`headroom/` identical to this PR's head) https://github.com/user-attachments/assets/5c687b18-93cd-4c1a-a675-72deb6579cac ## Additional Notes - **Closes:** none; no issue tracks this bug - **Related:** #3391, other replay gaps; not fixed here - **Docs:** unchecked; no documented behavior - **Baseline:** 8 fail on `main` too; listed above --------- Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
@@ -4174,6 +4174,17 @@ class OpenAIHandlerMixin:
|
||||
f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
|
||||
"to preserve cache stability (openai)"
|
||||
)
|
||||
# The restore writes raw client originals, but the provider cached
|
||||
# what we forwarded last turn. Replay that over the restored result;
|
||||
# the overlay's own checks keep the originals wherever replay is not
|
||||
# provably safe. Token counts are recomputed from the final body below.
|
||||
optimized_messages = finalize_turn(
|
||||
optimized_messages,
|
||||
original_client_messages,
|
||||
openai_prefix_tracker.get_last_original_messages(),
|
||||
openai_prefix_tracker.get_last_forwarded_messages(),
|
||||
confirmed_frozen_count=_openai_confirmed_frozen,
|
||||
).messages
|
||||
|
||||
# Memory: inject context and tools for OpenAI requests.
|
||||
#
|
||||
@@ -4646,6 +4657,7 @@ class OpenAIHandlerMixin:
|
||||
waste_signals=waste_signals_dict,
|
||||
prefix_tracker=openai_prefix_tracker,
|
||||
optimized_messages=optimized_messages,
|
||||
original_messages=original_client_messages,
|
||||
backend=request_backend,
|
||||
)
|
||||
else:
|
||||
@@ -4885,6 +4897,7 @@ class OpenAIHandlerMixin:
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
messages=optimized_messages,
|
||||
original_messages=original_client_messages,
|
||||
)
|
||||
|
||||
await self._record_request_outcome(
|
||||
@@ -4983,6 +4996,7 @@ class OpenAIHandlerMixin:
|
||||
optimization_latency,
|
||||
pipeline_timing=pipeline_timing,
|
||||
prefix_tracker=openai_prefix_tracker,
|
||||
original_messages=original_client_messages,
|
||||
outcome_provider=openai_chat_outcome_provider,
|
||||
)
|
||||
else:
|
||||
@@ -5263,6 +5277,7 @@ class OpenAIHandlerMixin:
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
messages=optimized_messages,
|
||||
original_messages=original_client_messages,
|
||||
)
|
||||
|
||||
# OpenAI has no write penalty — uncached = total - cached
|
||||
|
||||
@@ -2126,6 +2126,7 @@ class StreamingMixin:
|
||||
waste_signals: dict[str, int] | None = None,
|
||||
prefix_tracker: Any | None = None,
|
||||
optimized_messages: list[dict] | None = None,
|
||||
original_messages: list[dict] | None = None,
|
||||
backend: Any | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Stream OpenAI chat completion response from backend.
|
||||
@@ -2148,6 +2149,11 @@ class StreamingMixin:
|
||||
the FINAL usage frame can update the tracker for the next turn
|
||||
— mirroring the direct streaming path
|
||||
(``_stream_response``/``_finalize_stream_response``).
|
||||
``original_messages`` is the immutable pre-transform client
|
||||
snapshot for the same update: the tracker must record what the
|
||||
client SENT as its ``last_original_messages``, not what we
|
||||
forwarded, or next turn's overlay never matches the client
|
||||
prefix and cannot replay the cached bytes.
|
||||
|
||||
NOTE: CCR request-level intercept on the streaming path is
|
||||
intentionally OUT OF SCOPE. Mirrors the Anthropic streaming
|
||||
@@ -2271,6 +2277,7 @@ class StreamingMixin:
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
messages=tracker_messages,
|
||||
original_messages=original_messages,
|
||||
)
|
||||
|
||||
# CCR Feedback: record headroom_retrieve tool calls so
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -12,6 +14,7 @@ pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from headroom.backends.base import BackendResponse
|
||||
from headroom.proxy.server import ProxyConfig, create_app
|
||||
|
||||
|
||||
@@ -25,6 +28,7 @@ class _FakePrefixTracker:
|
||||
self._frozen_count = frozen_count
|
||||
self._previous_original = previous_original or []
|
||||
self._previous_forwarded = previous_forwarded or []
|
||||
self.update_calls: list[dict] = []
|
||||
|
||||
def get_frozen_message_count(self) -> int:
|
||||
return self._frozen_count
|
||||
@@ -40,10 +44,11 @@ class _FakePrefixTracker:
|
||||
return copy.deepcopy(self._previous_forwarded)
|
||||
|
||||
def update_from_response(self, **kwargs): # noqa: ANN003
|
||||
self.update_calls.append(kwargs)
|
||||
return None
|
||||
|
||||
|
||||
def _make_proxy_client() -> TestClient:
|
||||
def _make_proxy_client(**config_overrides) -> TestClient: # noqa: ANN003
|
||||
config = ProxyConfig(
|
||||
optimize=False,
|
||||
cache_enabled=False,
|
||||
@@ -54,6 +59,7 @@ def _make_proxy_client() -> TestClient:
|
||||
ccr_handle_responses=False,
|
||||
ccr_context_tracking=False,
|
||||
image_optimize=False,
|
||||
**config_overrides,
|
||||
)
|
||||
app = create_app(config)
|
||||
return TestClient(app)
|
||||
@@ -564,3 +570,610 @@ def test_openai_handler_replays_the_provider_confirmed_prefix_even_when_it_infla
|
||||
assert response.status_code == 200
|
||||
assert captured["body"]["messages"][0] == previous_forwarded[0]
|
||||
assert captured["body"]["messages"][1] == {"role": "user", "content": "new suffix"}
|
||||
|
||||
|
||||
def _tool_history() -> list[dict]:
|
||||
"""A history whose big tool result the pipeline can compress."""
|
||||
return [
|
||||
{"role": "user", "content": "run the build and show me the log"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "bash", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "build log line 0001 module m0 status OK\n" * 200,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _compress_first_tool_result(**kwargs):
|
||||
"""Fake pipeline: compress the big tool result so forwarded ≠ client bytes."""
|
||||
mutated = [dict(message) for message in kwargs["messages"]]
|
||||
for i, message in enumerate(mutated):
|
||||
if message.get("role") == "tool":
|
||||
mutated[i] = {**message, "content": "[compressed build log]"}
|
||||
break
|
||||
return SimpleNamespace(
|
||||
messages=mutated,
|
||||
transforms_applied=["fake:mutated"],
|
||||
timing={},
|
||||
tokens_before=2000,
|
||||
tokens_after=30,
|
||||
waste_signals=None,
|
||||
)
|
||||
|
||||
|
||||
def _install_tracker(proxy, tracker) -> None:
|
||||
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
|
||||
"stable-session"
|
||||
)
|
||||
proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
|
||||
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
|
||||
|
||||
|
||||
def _chat_usage(prompt_tokens: int, cached_tokens: int) -> dict:
|
||||
return {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": 3,
|
||||
"total_tokens": prompt_tokens + 3,
|
||||
"prompt_tokens_details": {"cached_tokens": cached_tokens},
|
||||
}
|
||||
|
||||
|
||||
def test_openai_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
|
||||
"""Direct streaming chat must give the stream finalizer the client's originals.
|
||||
|
||||
Without them the prefix tracker stores the forwarded (compressed) messages
|
||||
as "original", so next turn's overlay never matches the client prefix and
|
||||
cannot replay the cached bytes.
|
||||
"""
|
||||
captured = {}
|
||||
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "cache"
|
||||
_install_tracker(proxy, _FakePrefixTracker(frozen_count=0))
|
||||
proxy.openai_pipeline.apply = _compress_first_tool_result
|
||||
|
||||
async def _fake_stream_response(*args, **kwargs): # noqa: ANN002, ANN003
|
||||
captured.update(kwargs)
|
||||
return httpx.Response(200, text="data: [DONE]\n\n")
|
||||
|
||||
proxy._stream_response = _fake_stream_response
|
||||
client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
|
||||
)
|
||||
|
||||
assert captured.get("original_messages") == client_messages
|
||||
|
||||
|
||||
def test_openai_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
|
||||
"""Direct buffered chat must record the client's originals, not its own output.
|
||||
|
||||
Token mode isolates the recording behavior: in cache mode the frozen-prefix
|
||||
restore reverts prefix bytes before they are forwarded, which would make the
|
||||
forwarded list coincidentally equal the originals. The originals plumbing
|
||||
under test here is mode-independent.
|
||||
"""
|
||||
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "token"
|
||||
tracker = _FakePrefixTracker(frozen_count=0)
|
||||
_install_tracker(proxy, tracker)
|
||||
proxy.openai_pipeline.apply = _compress_first_tool_result
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_originals_buffered",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": _chat_usage(30, 0),
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "messages": client_messages},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert len(tracker.update_calls) == 1
|
||||
call = tracker.update_calls[0]
|
||||
# The tracker must store what the client SENT as "original", and the
|
||||
# compressed form separately as what was forwarded.
|
||||
assert call["original_messages"] == client_messages
|
||||
assert call["messages"][2]["content"] == "[compressed build log]"
|
||||
|
||||
|
||||
def _mock_openai_backend_response() -> dict:
|
||||
return {
|
||||
"id": "chatcmpl-backend-originals",
|
||||
"object": "chat.completion",
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": _chat_usage(30, 0),
|
||||
}
|
||||
|
||||
|
||||
def _sse_chunks(prompt_tokens: int, cached_tokens: int) -> list[str]:
|
||||
return [
|
||||
'data: {"id":"c1","object":"chat.completion.chunk",'
|
||||
'"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"}}]}\n\n',
|
||||
'data: {"id":"c1","object":"chat.completion.chunk",'
|
||||
'"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
|
||||
'data: {"id":"c1","object":"chat.completion.chunk","choices":[],'
|
||||
f'"usage":{{"prompt_tokens":{prompt_tokens},"completion_tokens":3,'
|
||||
f'"total_tokens":{prompt_tokens + 3},'
|
||||
f'"prompt_tokens_details":{{"cached_tokens":{cached_tokens}}}}}}}\n\n',
|
||||
"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
|
||||
def _mock_streaming_backend(chunks: list[str]) -> MagicMock:
|
||||
async def fake_stream(body, headers): # noqa: ANN001
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
backend = MagicMock()
|
||||
backend.name = "anyllm-openai"
|
||||
backend.stream_openai_message = fake_stream
|
||||
return backend
|
||||
|
||||
|
||||
def test_openai_backend_buffered_chat_hands_client_originals_to_prefix_tracker() -> None:
|
||||
"""Backend-routed buffered chat must record the client's originals too."""
|
||||
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
|
||||
backend = MagicMock()
|
||||
backend.name = "anyllm-openai"
|
||||
backend.send_openai_message = AsyncMock(
|
||||
return_value=BackendResponse(
|
||||
body=_mock_openai_backend_response(),
|
||||
status_code=200,
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
)
|
||||
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
||||
with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "token"
|
||||
tracker = _FakePrefixTracker(frozen_count=0)
|
||||
_install_tracker(proxy, tracker)
|
||||
proxy.openai_pipeline.apply = _compress_first_tool_result
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "stream": False, "messages": client_messages},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert backend.send_openai_message.await_count == 1
|
||||
assert len(tracker.update_calls) == 1
|
||||
call = tracker.update_calls[0]
|
||||
assert call["original_messages"] == client_messages
|
||||
assert call["messages"][2]["content"] == "[compressed build log]"
|
||||
|
||||
|
||||
def test_openai_backend_streaming_chat_hands_client_originals_to_prefix_tracker() -> None:
|
||||
"""Backend-routed streaming chat must record the client's originals too."""
|
||||
client_messages = _tool_history() + [{"role": "user", "content": "current turn"}]
|
||||
backend = _mock_streaming_backend(_sse_chunks(prompt_tokens=30, cached_tokens=0))
|
||||
with patch("headroom.proxy.server.AnyLLMBackend", return_value=backend):
|
||||
with _make_proxy_client(backend="anyllm", anyllm_provider="openai") as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "token"
|
||||
tracker = _FakePrefixTracker(frozen_count=0)
|
||||
_install_tracker(proxy, tracker)
|
||||
proxy.openai_pipeline.apply = _compress_first_tool_result
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "stream": True, "messages": client_messages},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "[DONE]" in response.text
|
||||
assert len(tracker.update_calls) == 1
|
||||
call = tracker.update_calls[0]
|
||||
assert call["original_messages"] == client_messages
|
||||
assert call["messages"][2]["content"] == "[compressed build log]"
|
||||
|
||||
|
||||
def test_openai_chat_real_tracker_stores_client_originals_across_turns() -> None:
|
||||
"""Real PrefixCacheTracker, two buffered turns (token mode).
|
||||
|
||||
Turn 2's stored "original" history must be the client's own messages
|
||||
(including the new turn), not the compressed bytes that were forwarded —
|
||||
otherwise next turn's overlay can never match the client prefix and
|
||||
cannot replay the previously forwarded bytes.
|
||||
"""
|
||||
from headroom.cache.prefix_tracker import PrefixCacheTracker
|
||||
|
||||
turn1_messages = _tool_history()
|
||||
turn2_messages = turn1_messages + [
|
||||
{"role": "assistant", "content": "the build passed"},
|
||||
{"role": "user", "content": "and now?"},
|
||||
]
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "token"
|
||||
tracker = PrefixCacheTracker(provider="openai")
|
||||
_install_tracker(proxy, tracker)
|
||||
proxy.openai_pipeline.apply = _compress_first_tool_result
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_real_tracker",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": _chat_usage(prompt_tokens=5000, cached_tokens=0),
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
for messages in (turn1_messages, turn2_messages):
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "messages": messages},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# The forwarded history is the compressed tool result...
|
||||
forwarded = tracker.get_last_forwarded_messages()
|
||||
assert forwarded[2]["content"] == "[compressed build log]"
|
||||
# ...but the recorded "original" history must be exactly what the client
|
||||
# sent on turn 2, byte-for-byte, new turns included.
|
||||
assert tracker.get_last_original_messages() == turn2_messages
|
||||
|
||||
|
||||
def test_openai_cache_mode_keeps_replayed_prefix_over_frozen_restore() -> None:
|
||||
"""Cache mode: the frozen-prefix restore must not undo a replayed prefix.
|
||||
|
||||
The overlay replays last turn's forwarded bytes, which the provider cached.
|
||||
Restoring the frozen prefix to the raw client original afterwards
|
||||
re-forwards different bytes and busts the cache from message 0.
|
||||
"""
|
||||
captured = {}
|
||||
previous_original = [{"role": "user", "content": "original prefix"}]
|
||||
previous_forwarded = [{"role": "user", "content": "comp"}]
|
||||
fake_tracker = _FakePrefixTracker(1, previous_original, previous_forwarded)
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "cache"
|
||||
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
|
||||
"stable-session"
|
||||
)
|
||||
proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
|
||||
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: fake_tracker
|
||||
|
||||
def _fake_apply(**kwargs):
|
||||
return SimpleNamespace(
|
||||
messages=kwargs["messages"],
|
||||
transforms_applied=[],
|
||||
timing={},
|
||||
tokens_before=20,
|
||||
tokens_after=20,
|
||||
waste_signals=None,
|
||||
)
|
||||
|
||||
proxy.openai_pipeline.apply = _fake_apply
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
captured["body"] = body
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_replay_restore",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [
|
||||
{"role": "user", "content": "original prefix"},
|
||||
{"role": "user", "content": "new suffix"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert captured["body"]["messages"] == [
|
||||
previous_forwarded[0],
|
||||
{"role": "user", "content": "new suffix"},
|
||||
]
|
||||
|
||||
|
||||
def _forward_cache_mode_chat(
|
||||
tracker: _FakePrefixTracker,
|
||||
messages: list[dict],
|
||||
*,
|
||||
mutate_index: int | None = None,
|
||||
) -> list[dict]:
|
||||
"""Send one cache-mode chat turn and return the messages forwarded upstream.
|
||||
|
||||
``mutate_index`` makes the fake pipeline rewrite that message, standing in
|
||||
for a transform that touched a frozen position."""
|
||||
captured: dict = {}
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "cache"
|
||||
proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
|
||||
"stable-session"
|
||||
)
|
||||
proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
|
||||
proxy.session_tracker_store.resolve_tracker = lambda *args, **kwargs: tracker
|
||||
|
||||
def _fake_apply(**kwargs):
|
||||
out = list(kwargs["messages"])
|
||||
if mutate_index is not None:
|
||||
out[mutate_index] = {**out[mutate_index], "content": "MUTATED_BY_PIPELINE"}
|
||||
return SimpleNamespace(
|
||||
messages=out,
|
||||
transforms_applied=[],
|
||||
timing={},
|
||||
tokens_before=20,
|
||||
tokens_after=20,
|
||||
waste_signals=None,
|
||||
)
|
||||
|
||||
proxy.openai_pipeline.apply = _fake_apply
|
||||
|
||||
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
|
||||
captured["body"] = body
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_restore_replay",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23},
|
||||
},
|
||||
)
|
||||
|
||||
proxy._retry_request = _fake_retry
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "messages": messages},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
return captured["body"]["messages"]
|
||||
|
||||
|
||||
def test_openai_cache_mode_replays_only_up_to_the_first_changed_message() -> None:
|
||||
"""Replay stops where the client's history diverges. Past that point the
|
||||
frozen messages are restored to the client's bytes, so a pipeline rewrite
|
||||
of a frozen message never reaches the provider, even on a replayed turn."""
|
||||
previous_original = [
|
||||
{"role": "user", "content": "first original " * 20},
|
||||
{"role": "user", "content": "second original " * 20},
|
||||
]
|
||||
previous_forwarded = [
|
||||
{"role": "user", "content": "first comp"},
|
||||
{"role": "user", "content": "second comp"},
|
||||
]
|
||||
current = [
|
||||
previous_original[0],
|
||||
{"role": "user", "content": "second edited by the client"},
|
||||
{"role": "user", "content": "new suffix"},
|
||||
]
|
||||
|
||||
forwarded = _forward_cache_mode_chat(
|
||||
_FakePrefixTracker(2, previous_original, previous_forwarded),
|
||||
current,
|
||||
mutate_index=1,
|
||||
)
|
||||
|
||||
assert forwarded == [previous_forwarded[0], current[1], current[2]]
|
||||
|
||||
|
||||
def test_openai_cache_mode_keeps_restored_originals_when_history_diverges() -> None:
|
||||
"""A rewritten history head has no safe replay: the restored originals go out."""
|
||||
previous_original = [{"role": "user", "content": "original prefix"}]
|
||||
previous_forwarded = [{"role": "user", "content": "comp"}]
|
||||
current = [
|
||||
{"role": "user", "content": "client rewrote the head"},
|
||||
{"role": "user", "content": "new suffix"},
|
||||
]
|
||||
|
||||
forwarded = _forward_cache_mode_chat(
|
||||
_FakePrefixTracker(1, previous_original, previous_forwarded),
|
||||
current,
|
||||
mutate_index=0,
|
||||
)
|
||||
|
||||
assert forwarded == current
|
||||
|
||||
|
||||
def test_openai_cache_mode_block_append_replay_survives_the_restore() -> None:
|
||||
"""Blocks appended inside the last message: forwarded blocks, then new ones."""
|
||||
previous_original = [{"role": "user", "content": [{"type": "text", "text": "block one " * 30}]}]
|
||||
previous_forwarded = [{"role": "user", "content": [{"type": "text", "text": "[compacted]"}]}]
|
||||
current = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "block one " * 30},
|
||||
{"type": "text", "text": "block two"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
forwarded = _forward_cache_mode_chat(
|
||||
_FakePrefixTracker(1, previous_original, previous_forwarded), current
|
||||
)
|
||||
|
||||
assert forwarded == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "[compacted]"},
|
||||
{"type": "text", "text": "block two"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _compress_unfrozen_tool_results(**kwargs):
|
||||
"""Fake pipeline that, like the real one, only rewrites past the frozen count."""
|
||||
frozen = kwargs.get("frozen_message_count") or 0
|
||||
out = [
|
||||
{**message, "content": f"[compressed {message['tool_call_id']}]"}
|
||||
if i >= frozen and message.get("role") == "tool"
|
||||
else message
|
||||
for i, message in enumerate(kwargs["messages"])
|
||||
]
|
||||
return SimpleNamespace(
|
||||
messages=out,
|
||||
transforms_applied=["fake:compressed"],
|
||||
timing={},
|
||||
tokens_before=2000,
|
||||
tokens_after=30,
|
||||
waste_signals=None,
|
||||
)
|
||||
|
||||
|
||||
def _provider_reply(request: httpx.Request) -> httpx.Response:
|
||||
usage = _chat_usage(prompt_tokens=4000, cached_tokens=2048)
|
||||
if json.loads(request.content).get("stream"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
content="".join(_sse_chunks(4000, 2048)).encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_lifecycle",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": usage,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "stream"])
|
||||
def test_openai_cache_mode_forwards_last_turns_bytes_unchanged(stream: bool) -> None:
|
||||
"""Cache mode, real tracker, three turns: every turn re-forwards the previous
|
||||
turn's forwarded messages byte-for-byte, so the provider's prefix cache holds.
|
||||
|
||||
Needs both halves: the tracker must record the client's originals (else the
|
||||
overlay cannot match the client prefix), and the frozen-prefix restore must
|
||||
not undo the replay (else the compressed tool results revert to raw bytes).
|
||||
"""
|
||||
from headroom.cache.prefix_tracker import PrefixCacheTracker
|
||||
|
||||
def tool_turn(n: int) -> list[dict]:
|
||||
call_id = f"call_{n}"
|
||||
return [
|
||||
{"role": "user", "content": f"turn {n}: run the build"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": "bash", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
"content": f"build {n} log line module m0 status OK\n" * 200,
|
||||
},
|
||||
]
|
||||
|
||||
sent: list[list[dict]] = []
|
||||
|
||||
def provider(request: httpx.Request) -> httpx.Response:
|
||||
sent.append(json.loads(request.content)["messages"])
|
||||
return _provider_reply(request)
|
||||
|
||||
history: list[dict] = []
|
||||
with _make_proxy_client() as client:
|
||||
proxy = client.app.state.proxy
|
||||
proxy.config.optimize = True
|
||||
proxy.config.mode = "cache"
|
||||
_install_tracker(proxy, PrefixCacheTracker(provider="openai"))
|
||||
proxy.openai_pipeline.apply = _compress_unfrozen_tool_results
|
||||
proxy.http_client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
|
||||
for n in (1, 2, 3):
|
||||
history += tool_turn(n)
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key"},
|
||||
json={"model": "gpt-4o-mini", "stream": stream, "messages": history},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
response.read()
|
||||
history.append({"role": "assistant", "content": "ok"})
|
||||
|
||||
assert len(sent) == 3
|
||||
assert sent[0][2]["content"] == "[compressed call_1]"
|
||||
assert sent[1][: len(sent[0])] == sent[0]
|
||||
assert sent[2][: len(sent[1])] == sent[1]
|
||||
|
||||
Reference in New Issue
Block a user