fix(proxy): drop the 32-row savings cap; coalesce rows and count events

record_savings dropped a request's 33rd attribution row on every handler,
and the WS pending list folded sources past 28 rows into "other". Every
writer now merges rows by (source, realized, estimated) through one
accumulate helper, so rows are bounded by distinct sources and no token
is dropped. Each row carries an events count (1 when fresh, summed on
merge); Prometheus, OTel and the perf analyzer add it, so 33 coalesced
frames are 33 events, not 1. Rows without the field count as 1.

MAX_SOURCES no longer caps anything and is kept behind a deprecation
warning for importers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-16 10:16:19 -07:00
co-authored by Claude Opus 5.5
parent f4098d72ef
commit 90d5e13288
11 changed files with 178 additions and 58 deletions
+2 -1
View File
@@ -14,6 +14,7 @@ from opentelemetry.metrics import CallbackOptions, Observation
from headroom._version import get_version
from headroom.offline import guard_egress
from headroom.proxy.savings_attribution import event_count
logger = logging.getLogger(__name__)
@@ -558,7 +559,7 @@ class HeadroomOtelMetrics:
realized=bool(item.get("realized", True)),
estimated=bool(item.get("estimated", False)),
)
self._savings_attribution_events.add(1, attrs)
self._savings_attribution_events.add(event_count(item), attrs)
saved = max(0, int(item.get("tokens", 0) or 0))
if saved:
self._savings_attributed_tokens.add(saved, attrs)
+6 -2
View File
@@ -813,13 +813,15 @@ def format_report(report: PerfReport) -> str:
# exactly zero tokens; every token-savings channel in the proxy would record
# it as nothing. Showing `$` beside a `0 tokens` row is the honest rendering
# of that, and collapsing them into one number would be an invented saving.
from headroom.proxy.savings_attribution import event_count
by_source: dict[tuple[str, bool], dict[str, float]] = {}
for record in report.perf_records:
for item in getattr(record, "savings_breakdown", ()) or ():
source = str(item.get("source") or "other")
realized = bool(item.get("realized", True))
row = by_source.setdefault((source, realized), {"events": 0, "tokens": 0, "usd": 0.0})
row["events"] += 1
row["events"] += event_count(item)
row["tokens"] += max(0, _as_number(item.get("tokens"), int))
row["usd"] += _as_number(item.get("usd"), float)
if by_source:
@@ -1356,6 +1358,8 @@ def build_perf_summary(report: PerfReport) -> dict:
}
)
from headroom.proxy.savings_attribution import event_count
by_source_groups: dict[tuple[str, bool], dict[str, int | float | str | bool]] = {}
for record in records:
for item in record.savings_breakdown:
@@ -1372,7 +1376,7 @@ def build_perf_summary(report: PerfReport) -> dict:
"usd": 0.0,
},
)
row["events"] = int(row["events"]) + 1
row["events"] = int(row["events"]) + event_count(item)
raw_tokens = item.get("tokens", 0)
raw_usd = item.get("usd", 0.0)
tokens = int(raw_tokens) if isinstance(raw_tokens, (str, int, float)) else 0
+1 -1
View File
@@ -7029,7 +7029,7 @@ class OpenAIHandlerMixin:
# that were rewritten and sent, waiting for the next turn's outcome. Each
# frame compresses into its own dict: a timed-out compression thread
# keeps running and must not book rows for a frame sent uncompressed.
# Bounded by ``accumulate_savings`` so ``from_tags`` reads every row.
# Coalesced per source by ``accumulate_savings``, so it stays small.
ws_turn_savings: list[dict[str, Any]] = []
# Extract subprotocol from client — this is an application-level negotiation
+2 -1
View File
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
from headroom import savings_ledger
from headroom.observability import get_otel_metrics
from headroom.proxy.savings_attribution import event_count
from headroom.proxy.savings_tracker import SavingsTracker, estimate_request_savings_usd
logger = logging.getLogger("headroom.proxy")
@@ -988,7 +989,7 @@ class PrometheusMetrics:
"usd": 0.0,
},
)
row["events"] = int(row["events"]) + 1
row["events"] = int(row["events"]) + event_count(item)
row["tokens"] = int(row["tokens"]) + max(0, int(item.get("tokens", 0) or 0))
row["usd"] = round(
float(row["usd"]) + float(item.get("usd", 0.0) or 0.0),
+48 -28
View File
@@ -6,12 +6,12 @@ import base64
import json
import math
import re
from collections.abc import MutableMapping
import warnings
from collections.abc import Iterable, MutableMapping
from typing import Any
SAVINGS_ATTRIBUTION_TAG = "_headroom_savings_attribution"
_NAME_RE = re.compile(r"[^a-z0-9_.-]+")
MAX_SOURCES = 32
_SCOPE_KEY = "headroom_savings_attribution"
# Per-request stage timings contributed from outside the handler, merged into
@@ -164,9 +164,6 @@ def record_savings(
details: dict[str, Any] | None = None,
) -> None:
"""Attribute savings to a source; this never changes headline totals."""
ledger = _ledger(tags)
if len(ledger) >= MAX_SOURCES:
return
# Same hazard as MAX_STAGE_MS, on the amounts rather than the durations:
# ``usd=inf`` reaches ``/stats`` and raises out of the JSON encoder, and
# ``int(inf)`` raises OverflowError right here, inside the handler, on a
@@ -193,55 +190,64 @@ def record_savings(
for key, value in list(details.items())[:12]
if isinstance(value, (str, int, float, bool)) or value is None
}
ledger.append(item)
accumulate(_ledger(tags), [item])
def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]:
raw = (tags or {}).get(SAVINGS_ATTRIBUTION_TAG)
if not isinstance(raw, list):
return []
return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)]
return [dict(item) for item in raw if isinstance(item, dict)]
# One "other" row per (realized, estimated) pair is kept free, so overflow is
# summed rather than dropped.
_OVERFLOW_SLOTS = 4
def event_count(row: dict[str, Any]) -> int:
"""How many recordings a row stands for; a row without ``events`` is one."""
try:
return max(1, int(row.get("events", 1)))
except (TypeError, ValueError, OverflowError):
return 1
def accumulate(pending: list[dict[str, Any]], items: list[dict[str, Any]]) -> None:
"""Add ``items`` to ``pending`` in at most ``MAX_SOURCES`` rows, conserving totals.
def accumulate(rows: list[dict[str, Any]], items: Iterable[dict[str, Any]]) -> None:
"""Add ``items`` to ``rows``, one row per (source, realized, estimated).
For a ledger that outlives one request: a WS session holds rows until a
usage-carrying turn claims them, and :func:`from_tags` reads only the first
``MAX_SOURCES``. Rows with the same (source, realized, estimated) sum into
one; a new source past the cap sums into that pair's ``other`` row.
``details`` stay only while every summed row agrees on them.
Every writer goes through here -- the per-request ledger and a WS session's
pending rows alike -- so the row count is bounded by the distinct sources
that booked, and no row is ever dropped. Same-key rows sum ``tokens``,
``usd`` and ``events``; ``details`` stay only while every summed row agrees.
"""
for item in items:
key = _key(item)
target = _row_for(pending, key)
if target is None and len(pending) >= MAX_SOURCES - _OVERFLOW_SLOTS:
key = ("other", key[1], key[2])
target = _row_for(pending, key)
item = {k: v for k, v in item.items() if k != "details"}
target = _row_for(rows, key)
if target is None:
pending.append({**item, "source": key[0]})
rows.append({**item, "source": key[0], "events": event_count(item)})
continue
target["tokens"] = int(target.get("tokens", 0)) + int(item.get("tokens", 0) or 0)
target["usd"] = round(
float(target.get("usd", 0.0)) + float(item.get("usd", 0.0) or 0.0), 12
)
target["tokens"] = _num(target.get("tokens"), int) + _num(item.get("tokens"), int)
target["usd"] = round(_num(target.get("usd"), float) + _num(item.get("usd"), float), 12)
target["events"] = event_count(target) + event_count(item)
if target.get("details") != item.get("details"):
target.pop("details", None)
def _num(value: Any, cast: type[int] | type[float]) -> Any:
# A row edited in place by a reader (deferral reconcile, plugins) may hold
# junk; summing it must not raise inside the handler.
try:
number = cast(value or 0)
except (TypeError, ValueError, OverflowError):
return cast(0)
return number if math.isfinite(number) else cast(0)
def _key(row: dict[str, Any]) -> tuple[str, bool, bool]:
# ``realized`` defaults to True, as in every consumer of these rows.
return (str(row.get("source")), bool(row.get("realized", True)), bool(row.get("estimated")))
def _row_for(rows: list[dict[str, Any]], key: tuple[str, bool, bool]) -> dict[str, Any] | None:
return next((row for row in rows if _key(row) == key), None)
# The ledger is a plain list reachable through ``tags``; skip what this
# module did not write rather than raise inside the handler.
return next((row for row in rows if isinstance(row, dict) and _key(row) == key), None)
_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG})
@@ -287,3 +293,17 @@ def decode(value: str) -> list[dict[str, Any]]:
if isinstance(decoded, list)
else []
)
def __getattr__(name: str) -> Any:
# MAX_SOURCES used to cap a request's ledger at 32 rows and silently drop
# the rest; rows now coalesce per source instead. Kept for importers.
if name == "MAX_SOURCES":
warnings.warn(
"headroom.proxy.savings_attribution.MAX_SOURCES no longer caps anything "
"and will be removed; savings rows coalesce per source instead.",
DeprecationWarning,
stacklevel=2,
)
return 32
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+4 -3
View File
@@ -84,9 +84,10 @@ def deferred_booking(tags: object, tools: object) -> DeferredBooking | None:
``None`` when nothing is booked. The schemas are copied as they were when
booked, so a later in-place edit cannot change what the credit stands for.
The ledger entry is the latest ``tool_search`` one marked ``estimated``:
both the core deferral and the native deferral hook book that way, and a
hook's own measured saving never is, so it is never the one debited.
The ledger entry is the ``tool_search`` row marked ``estimated``: both the
core deferral and the native deferral hook book that way (rows coalesce, so
they share it), and a hook's own measured saving never is, so it is never
the one debited.
"""
if not isinstance(tags, dict) or "tool_search_deferred_tokens" not in tags:
return None
+20
View File
@@ -578,3 +578,23 @@ def test_token_source_and_dollar_source_coexist(tmp_path, monkeypatch):
def test_no_section_when_nothing_attributed(tmp_path, monkeypatch):
out = _report_for([_savings_perf_line("none")], tmp_path, monkeypatch)
assert "Savings by Source" not in out
def test_coalesced_savings_rows_count_every_event_in_text_and_summary():
"""A PERF row summed from 3 recordings is 3 events; a row without the field is 1."""
report = PerfReport(
perf_records=[
PerfRecord(
timestamp="2026-06-05 10:00:00,000",
request_id="hr_1",
savings_breakdown=[
{"source": "tool_search", "tokens": 300, "events": 3},
{"source": "router", "tokens": 5},
],
)
]
)
by_source = {row["source"]: row["events"] for row in build_perf_summary(report)["by_source"]}
assert by_source == {"tool_search": 3, "router": 1}
assert "tool_search: 3 events" in analyzer.format_report(report)
+66
View File
@@ -396,3 +396,69 @@ async def test_extension_savings_reach_the_metrics_call() -> None:
attribution = h.metrics.record_request.await_args.kwargs["savings_attribution"]
assert [(row["source"], row["usd"]) for row in attribution] == [("routemegood", 0.42)]
# --- no row cap: every recording is booked -------------------------------------
@pytest.mark.asyncio
async def test_a_request_with_40_sources_books_every_token() -> None:
"""The ledger used to stop at 32 rows and drop the 33rd recording on."""
from headroom.proxy.prometheus_metrics import PrometheusMetrics
from headroom.proxy.savings_attribution import record_savings
tags: dict = {}
for i in range(40):
record_savings(tags, f"plugin{i}", tokens=100, usd=0.01)
record_savings(tags, "plugin0", tokens=100, usd=0.01)
rows = from_tags(tags)
metrics = PrometheusMetrics()
await metrics.record_request(
provider="openai",
model="gpt-5.4",
input_tokens=10,
output_tokens=1,
tokens_saved=0,
latency_ms=1.0,
savings_attribution=rows,
)
assert len(rows) == 40
booked = metrics.savings_by_source.values()
assert sum(int(row["tokens"]) for row in booked) == 4_100
assert sum(int(row["events"]) for row in booked) == 41
assert metrics.savings_by_source["plugin0:1"]["events"] == 2
def test_rows_coalesce_per_source_realized_and_estimated() -> None:
from headroom.proxy.savings_attribution import record_savings
tags: dict = {}
for _ in range(3):
record_savings(tags, "router", tokens=1, realized=True)
record_savings(tags, "router", tokens=1, realized=False)
record_savings(tags, "router", tokens=1, estimated=True)
rows = from_tags(tags)
assert [(r["realized"], r["estimated"], r["tokens"], r["events"]) for r in rows] == [
(True, False, 3, 3),
(False, False, 3, 3),
(True, True, 3, 3),
]
def test_merging_into_a_junk_row_does_not_raise_inside_the_handler() -> None:
from headroom.proxy.savings_attribution import record_savings
tags: dict = {SAVINGS_ATTRIBUTION_TAG: [{"source": "x", "tokens": None, "usd": "junk"}]}
record_savings(tags, "x", tokens=5, usd=0.5)
assert [(r["tokens"], r["usd"], r["events"]) for r in from_tags(tags)] == [(5, 0.5, 2)]
def test_max_sources_is_deprecated_not_removed() -> None:
from headroom.proxy import savings_attribution
with pytest.warns(DeprecationWarning, match="MAX_SOURCES"):
assert savings_attribution.MAX_SOURCES == 32
+17
View File
@@ -262,6 +262,23 @@ def test_request_attribute_provider_enriches_core_and_savings_metrics() -> None:
unregister_otel_metric_attribute_provider(identity_attributes)
def test_coalesced_savings_rows_count_every_event() -> None:
"""A row summed from 33 recordings is 33 events, not 1; a bare row is 1."""
reader = InMemoryMetricReader()
otel_metrics = HeadroomOtelMetrics(meter_provider=MeterProvider(metric_readers=[reader]))
otel_metrics.record_savings_attribution(
[
{"source": "tool_search", "tokens": 3_300, "events": 33},
{"source": "router", "tokens": 5},
]
)
events = _collect_metrics(reader)["headroom.savings.attribution.events"]
assert _find_point(events, source="tool_search").value == 33
assert _find_point(events, source="router").value == 1
def test_failing_request_attribute_provider_is_fail_open() -> None:
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader])
+11 -22
View File
@@ -2427,29 +2427,20 @@ class _UpstreamAfterSends(_FakeUpstream):
@pytest.mark.asyncio
@pytest.mark.parametrize(
("sources_per_frame", "expected_rows"),
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 29)],
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 34)],
ids=["repeated_source", "distinct_sources"],
)
async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
sources_per_frame, expected_rows
):
"""33 frames each booking 100 tokens must hand 3,300 tokens to the outcome.
async def test_ws_pending_savings_beyond_32_rows_are_all_booked(sources_per_frame, expected_rows):
"""33 pending frames hand every token and every event to the billed turn.
The session-wide pending list grew by each frame's rows (here a ``ws_fold``
row and a 100-token hook row) and the outcome funnel reads at most
``MAX_SOURCES`` rows, so 66 rows were cut to 32 and the rest cleared
unbooked (1,600 of 3,300). Pending rows now coalesce per (source, realized,
estimated) and distinct sources past the cap fold into ``other``.
Rows coalesce to one per (source, realized, estimated) key, with no cap and
no ``other`` bucket.
"""
from headroom.proxy.prometheus_metrics import PrometheusMetrics
from headroom.proxy.savings_attribution import (
MAX_SOURCES,
SAVINGS_ATTRIBUTION_TAG,
from_tags,
)
from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG, from_tags
from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
frames_n = MAX_SOURCES + 1
frames_n = 33
seen = {"frame": 0}
class _DeferTools:
@@ -2525,10 +2516,8 @@ async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
latency_ms=1.0,
savings_attribution=rows,
)
booked = sum(
int(row["tokens"])
for row in metrics.savings_by_source.values()
if row["source"] != "ws_fold"
)
assert booked == 100 * frames_n
hook_rows = [row for row in metrics.savings_by_source.values() if row["source"] != "ws_fold"]
assert sum(int(row["tokens"]) for row in hook_rows) == 100 * frames_n
assert sum(int(row["events"]) for row in hook_rows) == frames_n
assert "other" not in {row["source"] for row in hook_rows}
assert stored == [expected_rows]
+1
View File
@@ -133,6 +133,7 @@ def test_request_runner_attributes_savings_with_handler_counters():
"tokens": 15,
"usd": 0.0,
"details": {"message_tokens_saved": 14, "tool_tokens_saved": 1},
"events": 1,
}
]