mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): bound Codex WS pending savings rows and book every token
The WS session's pending attribution list grew by each rewritten frame's rows, while from_tags reads only MAX_SOURCES (32) rows, so rows past the cap were cleared unbooked (33 x 100-token rows booked 3,200 of 3,300). savings_attribution.accumulate sums rows of one (source, realized, estimated) key and folds new sources past the cap into that pair's "other" row, so the list stays within MAX_SOURCES and totals are kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
97ca2318b2
commit
aecae31b6c
@@ -44,6 +44,7 @@ from headroom.proxy.provider_usage import (
|
||||
)
|
||||
from headroom.proxy.rate_limit_identity import rate_limit_identity
|
||||
from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG
|
||||
from headroom.proxy.savings_attribution import accumulate as accumulate_savings
|
||||
from headroom.proxy.savings_attribution import from_tags as savings_from_tags
|
||||
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
|
||||
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
|
||||
@@ -7028,6 +7029,7 @@ class OpenAIHandlerMixin:
|
||||
# that were rewritten and sent, waiting for the next turn's outcome. Each
|
||||
# frame compresses into its own dict: a timed-out compression thread
|
||||
# keeps running and must not book rows for a frame sent uncompressed.
|
||||
# Bounded by ``accumulate_savings`` so ``from_tags`` reads every row.
|
||||
ws_turn_savings: list[dict[str, Any]] = []
|
||||
|
||||
# Extract subprotocol from client — this is an application-level negotiation
|
||||
@@ -8104,7 +8106,9 @@ class OpenAIHandlerMixin:
|
||||
)
|
||||
_record_ws_compression_overhead(_rewrite_ms)
|
||||
tokens_saved += int(_ws_saved)
|
||||
ws_turn_savings.extend(savings_from_tags(_frame_savings))
|
||||
accumulate_savings(
|
||||
ws_turn_savings, savings_from_tags(_frame_savings)
|
||||
)
|
||||
ws_conversation_tokens_saved = int(_ws_saved)
|
||||
ws_conversation_key = savings_conversation_key(
|
||||
_send_body, session_id=f"ws:{session_id}"
|
||||
@@ -8601,7 +8605,7 @@ class OpenAIHandlerMixin:
|
||||
)
|
||||
_record_ws_compression_overhead(_rewrite_ms)
|
||||
tokens_saved += int(frame_saved)
|
||||
ws_turn_savings.extend(savings_from_tags(frame_savings))
|
||||
accumulate_savings(ws_turn_savings, savings_from_tags(frame_savings))
|
||||
ws_conversation_tokens_saved = int(frame_saved)
|
||||
ws_conversation_key = savings_conversation_key(
|
||||
new_inner, session_id=f"ws:{session_id}"
|
||||
|
||||
@@ -203,6 +203,47 @@ def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]:
|
||||
return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)]
|
||||
|
||||
|
||||
# One "other" row per (realized, estimated) pair is kept free, so overflow is
|
||||
# summed rather than dropped.
|
||||
_OVERFLOW_SLOTS = 4
|
||||
|
||||
|
||||
def accumulate(pending: list[dict[str, Any]], items: list[dict[str, Any]]) -> None:
|
||||
"""Add ``items`` to ``pending`` in at most ``MAX_SOURCES`` rows, conserving totals.
|
||||
|
||||
For a ledger that outlives one request: a WS session holds rows until a
|
||||
usage-carrying turn claims them, and :func:`from_tags` reads only the first
|
||||
``MAX_SOURCES``. Rows with the same (source, realized, estimated) sum into
|
||||
one; a new source past the cap sums into that pair's ``other`` row.
|
||||
``details`` stay only while every summed row agrees on them.
|
||||
"""
|
||||
for item in items:
|
||||
key = _key(item)
|
||||
target = _row_for(pending, key)
|
||||
if target is None and len(pending) >= MAX_SOURCES - _OVERFLOW_SLOTS:
|
||||
key = ("other", key[1], key[2])
|
||||
target = _row_for(pending, key)
|
||||
item = {k: v for k, v in item.items() if k != "details"}
|
||||
if target is None:
|
||||
pending.append({**item, "source": key[0]})
|
||||
continue
|
||||
target["tokens"] = int(target.get("tokens", 0)) + int(item.get("tokens", 0) or 0)
|
||||
target["usd"] = round(
|
||||
float(target.get("usd", 0.0)) + float(item.get("usd", 0.0) or 0.0), 12
|
||||
)
|
||||
if target.get("details") != item.get("details"):
|
||||
target.pop("details", None)
|
||||
|
||||
|
||||
def _key(row: dict[str, Any]) -> tuple[str, bool, bool]:
|
||||
# ``realized`` defaults to True, as in every consumer of these rows.
|
||||
return (str(row.get("source")), bool(row.get("realized", True)), bool(row.get("estimated")))
|
||||
|
||||
|
||||
def _row_for(rows: list[dict[str, Any]], key: tuple[str, bool, bool]) -> dict[str, Any] | None:
|
||||
return next((row for row in rows if _key(row) == key), None)
|
||||
|
||||
|
||||
_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG})
|
||||
|
||||
|
||||
|
||||
@@ -2407,3 +2407,128 @@ async def test_ws_turn_hook_savings_reach_the_turn_outcome(usage, expected):
|
||||
rows = [row for outcome in outcomes for row in from_tags(outcome.tags)]
|
||||
assert [(row["source"], row["realized"]) for row in rows] == expected
|
||||
assert all(row["tokens"] > 0 for row in rows)
|
||||
|
||||
|
||||
class _UpstreamAfterSends(_FakeUpstream):
|
||||
"""Streams its events only once ``sends`` frames reached it, so every frame's
|
||||
savings are pending before the one usage-carrying outcome claims them."""
|
||||
|
||||
def __init__(self, events: list[str], *, sends: int) -> None:
|
||||
super().__init__(events)
|
||||
self._sends = sends
|
||||
|
||||
async def _iter(self):
|
||||
while len(self.sent) < self._sends:
|
||||
await asyncio.sleep(0)
|
||||
for ev in self._events:
|
||||
yield ev
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("sources_per_frame", "expected_rows"),
|
||||
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 29)],
|
||||
ids=["repeated_source", "distinct_sources"],
|
||||
)
|
||||
async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
|
||||
sources_per_frame, expected_rows
|
||||
):
|
||||
"""33 frames each booking 100 tokens must hand 3,300 tokens to the outcome.
|
||||
|
||||
The session-wide pending list grew by each frame's rows (here a ``ws_fold``
|
||||
row and a 100-token hook row) and the outcome funnel reads at most
|
||||
``MAX_SOURCES`` rows, so 66 rows were cut to 32 and the rest cleared
|
||||
unbooked (1,600 of 3,300). Pending rows now coalesce per (source, realized,
|
||||
estimated) and distinct sources past the cap fold into ``other``.
|
||||
"""
|
||||
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
from headroom.proxy.savings_attribution import (
|
||||
MAX_SOURCES,
|
||||
SAVINGS_ATTRIBUTION_TAG,
|
||||
from_tags,
|
||||
)
|
||||
from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
|
||||
|
||||
frames_n = MAX_SOURCES + 1
|
||||
seen = {"frame": 0}
|
||||
|
||||
class _DeferTools:
|
||||
name = "defer_tools"
|
||||
savings_source = "ws_fold"
|
||||
stream_safe = True
|
||||
|
||||
def on_request(self, ctx: SimpleNamespace) -> None:
|
||||
del ctx.messages[:-1]
|
||||
ctx.record_savings(sources_per_frame(seen["frame"]), tokens=100)
|
||||
seen["frame"] += 1
|
||||
|
||||
def _frame(i: int) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "response.create",
|
||||
"response": {
|
||||
"model": "gpt-5.4",
|
||||
"input": [
|
||||
{"type": "message", "role": "user", "content": f"old {i} " * 50},
|
||||
{"type": "message", "role": "user", "content": f"new {i}"},
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
upstream = _UpstreamAfterSends(
|
||||
[
|
||||
json.dumps({"type": "response.created", "response": {"id": "r_1"}}),
|
||||
json.dumps(
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": "r_1",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
}
|
||||
),
|
||||
],
|
||||
sends=frames_n,
|
||||
)
|
||||
client_ws = _FakeWebSocket(frames=[_frame(i) for i in range(frames_n)])
|
||||
handler = _DummyOpenAIHandler()
|
||||
handler.config.optimize = True
|
||||
outcomes = []
|
||||
|
||||
async def _record_request_outcome(outcome):
|
||||
outcomes.append(outcome)
|
||||
|
||||
handler._record_request_outcome = _record_request_outcome
|
||||
|
||||
clear_turn_hooks()
|
||||
register_turn_hook(_DeferTools())
|
||||
try:
|
||||
with patch.dict(sys.modules, {"websockets": _make_fake_websockets_module(upstream)}):
|
||||
await handler.handle_openai_responses_ws(client_ws)
|
||||
finally:
|
||||
clear_turn_hooks()
|
||||
|
||||
assert len(upstream.sent) == frames_n
|
||||
assert seen["frame"] == frames_n
|
||||
metrics = PrometheusMetrics()
|
||||
stored = []
|
||||
for outcome in outcomes:
|
||||
rows = from_tags(outcome.tags)
|
||||
stored.append(len(outcome.tags.get(SAVINGS_ATTRIBUTION_TAG, [])))
|
||||
await metrics.record_request(
|
||||
provider="openai",
|
||||
model="gpt-5.4",
|
||||
input_tokens=10,
|
||||
output_tokens=1,
|
||||
tokens_saved=0,
|
||||
latency_ms=1.0,
|
||||
savings_attribution=rows,
|
||||
)
|
||||
booked = sum(
|
||||
int(row["tokens"])
|
||||
for row in metrics.savings_by_source.values()
|
||||
if row["source"] != "ws_fold"
|
||||
)
|
||||
assert booked == 100 * frames_n
|
||||
assert stored == [expected_rows]
|
||||
|
||||
Reference in New Issue
Block a user