fix(proxy): bound Codex WS pending savings rows and book every token

The WS session's pending attribution list grew by each rewritten frame's
rows, while from_tags reads only MAX_SOURCES (32) rows, so rows past the
cap were cleared unbooked (33 x 100-token rows booked 3,200 of 3,300).

savings_attribution.accumulate sums rows of one (source, realized,
estimated) key and folds new sources past the cap into that pair's
"other" row, so the list stays within MAX_SOURCES and totals are kept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-16 08:28:50 -07:00
co-authored by Claude Opus 5.5
parent 97ca2318b2
commit aecae31b6c
3 changed files with 172 additions and 2 deletions
+6 -2
View File
@@ -44,6 +44,7 @@ from headroom.proxy.provider_usage import (
)
from headroom.proxy.rate_limit_identity import rate_limit_identity
from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG
from headroom.proxy.savings_attribution import accumulate as accumulate_savings
from headroom.proxy.savings_attribution import from_tags as savings_from_tags
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
@@ -7028,6 +7029,7 @@ class OpenAIHandlerMixin:
# that were rewritten and sent, waiting for the next turn's outcome. Each
# frame compresses into its own dict: a timed-out compression thread
# keeps running and must not book rows for a frame sent uncompressed.
# Bounded by ``accumulate_savings`` so ``from_tags`` reads every row.
ws_turn_savings: list[dict[str, Any]] = []
# Extract subprotocol from client — this is an application-level negotiation
@@ -8104,7 +8106,9 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_rewrite_ms)
tokens_saved += int(_ws_saved)
ws_turn_savings.extend(savings_from_tags(_frame_savings))
accumulate_savings(
ws_turn_savings, savings_from_tags(_frame_savings)
)
ws_conversation_tokens_saved = int(_ws_saved)
ws_conversation_key = savings_conversation_key(
_send_body, session_id=f"ws:{session_id}"
@@ -8601,7 +8605,7 @@ class OpenAIHandlerMixin:
)
_record_ws_compression_overhead(_rewrite_ms)
tokens_saved += int(frame_saved)
ws_turn_savings.extend(savings_from_tags(frame_savings))
accumulate_savings(ws_turn_savings, savings_from_tags(frame_savings))
ws_conversation_tokens_saved = int(frame_saved)
ws_conversation_key = savings_conversation_key(
new_inner, session_id=f"ws:{session_id}"
+41
View File
@@ -203,6 +203,47 @@ def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]:
return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)]
# One "other" row per (realized, estimated) pair is kept free, so overflow is
# summed rather than dropped.
_OVERFLOW_SLOTS = 4
def accumulate(pending: list[dict[str, Any]], items: list[dict[str, Any]]) -> None:
"""Add ``items`` to ``pending`` in at most ``MAX_SOURCES`` rows, conserving totals.
For a ledger that outlives one request: a WS session holds rows until a
usage-carrying turn claims them, and :func:`from_tags` reads only the first
``MAX_SOURCES``. Rows with the same (source, realized, estimated) sum into
one; a new source past the cap sums into that pair's ``other`` row.
``details`` stay only while every summed row agrees on them.
"""
for item in items:
key = _key(item)
target = _row_for(pending, key)
if target is None and len(pending) >= MAX_SOURCES - _OVERFLOW_SLOTS:
key = ("other", key[1], key[2])
target = _row_for(pending, key)
item = {k: v for k, v in item.items() if k != "details"}
if target is None:
pending.append({**item, "source": key[0]})
continue
target["tokens"] = int(target.get("tokens", 0)) + int(item.get("tokens", 0) or 0)
target["usd"] = round(
float(target.get("usd", 0.0)) + float(item.get("usd", 0.0) or 0.0), 12
)
if target.get("details") != item.get("details"):
target.pop("details", None)
def _key(row: dict[str, Any]) -> tuple[str, bool, bool]:
# ``realized`` defaults to True, as in every consumer of these rows.
return (str(row.get("source")), bool(row.get("realized", True)), bool(row.get("estimated")))
def _row_for(rows: list[dict[str, Any]], key: tuple[str, bool, bool]) -> dict[str, Any] | None:
return next((row for row in rows if _key(row) == key), None)
_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG})
+125
View File
@@ -2407,3 +2407,128 @@ async def test_ws_turn_hook_savings_reach_the_turn_outcome(usage, expected):
rows = [row for outcome in outcomes for row in from_tags(outcome.tags)]
assert [(row["source"], row["realized"]) for row in rows] == expected
assert all(row["tokens"] > 0 for row in rows)
class _UpstreamAfterSends(_FakeUpstream):
"""Streams its events only once ``sends`` frames reached it, so every frame's
savings are pending before the one usage-carrying outcome claims them."""
def __init__(self, events: list[str], *, sends: int) -> None:
super().__init__(events)
self._sends = sends
async def _iter(self):
while len(self.sent) < self._sends:
await asyncio.sleep(0)
for ev in self._events:
yield ev
@pytest.mark.asyncio
@pytest.mark.parametrize(
("sources_per_frame", "expected_rows"),
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 29)],
ids=["repeated_source", "distinct_sources"],
)
async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
sources_per_frame, expected_rows
):
"""33 frames each booking 100 tokens must hand 3,300 tokens to the outcome.
The session-wide pending list grew by each frame's rows (here a ``ws_fold``
row and a 100-token hook row) and the outcome funnel reads at most
``MAX_SOURCES`` rows, so 66 rows were cut to 32 and the rest cleared
unbooked (1,600 of 3,300). Pending rows now coalesce per (source, realized,
estimated) and distinct sources past the cap fold into ``other``.
"""
from headroom.proxy.prometheus_metrics import PrometheusMetrics
from headroom.proxy.savings_attribution import (
MAX_SOURCES,
SAVINGS_ATTRIBUTION_TAG,
from_tags,
)
from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
frames_n = MAX_SOURCES + 1
seen = {"frame": 0}
class _DeferTools:
name = "defer_tools"
savings_source = "ws_fold"
stream_safe = True
def on_request(self, ctx: SimpleNamespace) -> None:
del ctx.messages[:-1]
ctx.record_savings(sources_per_frame(seen["frame"]), tokens=100)
seen["frame"] += 1
def _frame(i: int) -> str:
return json.dumps(
{
"type": "response.create",
"response": {
"model": "gpt-5.4",
"input": [
{"type": "message", "role": "user", "content": f"old {i} " * 50},
{"type": "message", "role": "user", "content": f"new {i}"},
],
},
}
)
upstream = _UpstreamAfterSends(
[
json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
json.dumps(
{
"type": "response.completed",
"response": {
"id": "r_1",
"usage": {"input_tokens": 10, "output_tokens": 1},
},
}
),
],
sends=frames_n,
)
client_ws = _FakeWebSocket(frames=[_frame(i) for i in range(frames_n)])
handler = _DummyOpenAIHandler()
handler.config.optimize = True
outcomes = []
async def _record_request_outcome(outcome):
outcomes.append(outcome)
handler._record_request_outcome = _record_request_outcome
clear_turn_hooks()
register_turn_hook(_DeferTools())
try:
with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):
await handler.handle_openai_responses_ws(client_ws)
finally:
clear_turn_hooks()
assert len(upstream.sent) == frames_n
assert seen["frame"] == frames_n
metrics = PrometheusMetrics()
stored = []
for outcome in outcomes:
rows = from_tags(outcome.tags)
stored.append(len(outcome.tags.get(SAVINGS_ATTRIBUTION_TAG, [])))
await metrics.record_request(
provider="openai",
model="gpt-5.4",
input_tokens=10,
output_tokens=1,
tokens_saved=0,
latency_ms=1.0,
savings_attribution=rows,
)
booked = sum(
int(row["tokens"])
for row in metrics.savings_by_source.values()
if row["source"] != "ws_fold"
)
assert booked == 100 * frames_n
assert stored == [expected_rows]