mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): drop the 32-row savings cap; coalesce rows and count events
record_savings dropped a request's 33rd attribution row on every handler, and the WS pending list folded sources past 28 rows into "other". Every writer now merges rows by (source, realized, estimated) through one accumulate helper, so rows are bounded by distinct sources and no token is dropped. Each row carries an events count (1 when fresh, summed on merge); Prometheus, OTel and the perf analyzer add it, so 33 coalesced frames are 33 events, not 1. Rows without the field count as 1. MAX_SOURCES no longer caps anything and is kept behind a deprecation warning for importers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f4098d72ef
commit
90d5e13288
@@ -14,6 +14,7 @@ from opentelemetry.metrics import CallbackOptions, Observation
|
||||
|
||||
from headroom._version import get_version
|
||||
from headroom.offline import guard_egress
|
||||
from headroom.proxy.savings_attribution import event_count
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -558,7 +559,7 @@ class HeadroomOtelMetrics:
|
||||
realized=bool(item.get("realized", True)),
|
||||
estimated=bool(item.get("estimated", False)),
|
||||
)
|
||||
self._savings_attribution_events.add(1, attrs)
|
||||
self._savings_attribution_events.add(event_count(item), attrs)
|
||||
saved = max(0, int(item.get("tokens", 0) or 0))
|
||||
if saved:
|
||||
self._savings_attributed_tokens.add(saved, attrs)
|
||||
|
||||
@@ -813,13 +813,15 @@ def format_report(report: PerfReport) -> str:
|
||||
# exactly zero tokens; every token-savings channel in the proxy would record
|
||||
# it as nothing. Showing `$` beside a `0 tokens` row is the honest rendering
|
||||
# of that, and collapsing them into one number would be an invented saving.
|
||||
from headroom.proxy.savings_attribution import event_count
|
||||
|
||||
by_source: dict[tuple[str, bool], dict[str, float]] = {}
|
||||
for record in report.perf_records:
|
||||
for item in getattr(record, "savings_breakdown", ()) or ():
|
||||
source = str(item.get("source") or "other")
|
||||
realized = bool(item.get("realized", True))
|
||||
row = by_source.setdefault((source, realized), {"events": 0, "tokens": 0, "usd": 0.0})
|
||||
row["events"] += 1
|
||||
row["events"] += event_count(item)
|
||||
row["tokens"] += max(0, _as_number(item.get("tokens"), int))
|
||||
row["usd"] += _as_number(item.get("usd"), float)
|
||||
if by_source:
|
||||
@@ -1356,6 +1358,8 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
}
|
||||
)
|
||||
|
||||
from headroom.proxy.savings_attribution import event_count
|
||||
|
||||
by_source_groups: dict[tuple[str, bool], dict[str, int | float | str | bool]] = {}
|
||||
for record in records:
|
||||
for item in record.savings_breakdown:
|
||||
@@ -1372,7 +1376,7 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
"usd": 0.0,
|
||||
},
|
||||
)
|
||||
row["events"] = int(row["events"]) + 1
|
||||
row["events"] = int(row["events"]) + event_count(item)
|
||||
raw_tokens = item.get("tokens", 0)
|
||||
raw_usd = item.get("usd", 0.0)
|
||||
tokens = int(raw_tokens) if isinstance(raw_tokens, (str, int, float)) else 0
|
||||
|
||||
@@ -7029,7 +7029,7 @@ class OpenAIHandlerMixin:
|
||||
# that were rewritten and sent, waiting for the next turn's outcome. Each
|
||||
# frame compresses into its own dict: a timed-out compression thread
|
||||
# keeps running and must not book rows for a frame sent uncompressed.
|
||||
# Bounded by ``accumulate_savings`` so ``from_tags`` reads every row.
|
||||
# Coalesced per source by ``accumulate_savings``, so it stays small.
|
||||
ws_turn_savings: list[dict[str, Any]] = []
|
||||
|
||||
# Extract subprotocol from client — this is an application-level negotiation
|
||||
|
||||
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from headroom import savings_ledger
|
||||
from headroom.observability import get_otel_metrics
|
||||
from headroom.proxy.savings_attribution import event_count
|
||||
from headroom.proxy.savings_tracker import SavingsTracker, estimate_request_savings_usd
|
||||
|
||||
logger = logging.getLogger("headroom.proxy")
|
||||
@@ -988,7 +989,7 @@ class PrometheusMetrics:
|
||||
"usd": 0.0,
|
||||
},
|
||||
)
|
||||
row["events"] = int(row["events"]) + 1
|
||||
row["events"] = int(row["events"]) + event_count(item)
|
||||
row["tokens"] = int(row["tokens"]) + max(0, int(item.get("tokens", 0) or 0))
|
||||
row["usd"] = round(
|
||||
float(row["usd"]) + float(item.get("usd", 0.0) or 0.0),
|
||||
|
||||
@@ -6,12 +6,12 @@ import base64
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections.abc import MutableMapping
|
||||
import warnings
|
||||
from collections.abc import Iterable, MutableMapping
|
||||
from typing import Any
|
||||
|
||||
SAVINGS_ATTRIBUTION_TAG = "_headroom_savings_attribution"
|
||||
_NAME_RE = re.compile(r"[^a-z0-9_.-]+")
|
||||
MAX_SOURCES = 32
|
||||
_SCOPE_KEY = "headroom_savings_attribution"
|
||||
|
||||
# Per-request stage timings contributed from outside the handler, merged into
|
||||
@@ -164,9 +164,6 @@ def record_savings(
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Attribute savings to a source; this never changes headline totals."""
|
||||
ledger = _ledger(tags)
|
||||
if len(ledger) >= MAX_SOURCES:
|
||||
return
|
||||
# Same hazard as MAX_STAGE_MS, on the amounts rather than the durations:
|
||||
# ``usd=inf`` reaches ``/stats`` and raises out of the JSON encoder, and
|
||||
# ``int(inf)`` raises OverflowError right here, inside the handler, on a
|
||||
@@ -193,55 +190,64 @@ def record_savings(
|
||||
for key, value in list(details.items())[:12]
|
||||
if isinstance(value, (str, int, float, bool)) or value is None
|
||||
}
|
||||
ledger.append(item)
|
||||
accumulate(_ledger(tags), [item])
|
||||
|
||||
|
||||
def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]:
|
||||
raw = (tags or {}).get(SAVINGS_ATTRIBUTION_TAG)
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)]
|
||||
return [dict(item) for item in raw if isinstance(item, dict)]
|
||||
|
||||
|
||||
# One "other" row per (realized, estimated) pair is kept free, so overflow is
|
||||
# summed rather than dropped.
|
||||
_OVERFLOW_SLOTS = 4
|
||||
def event_count(row: dict[str, Any]) -> int:
|
||||
"""How many recordings a row stands for; a row without ``events`` is one."""
|
||||
try:
|
||||
return max(1, int(row.get("events", 1)))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 1
|
||||
|
||||
|
||||
def accumulate(pending: list[dict[str, Any]], items: list[dict[str, Any]]) -> None:
|
||||
"""Add ``items`` to ``pending`` in at most ``MAX_SOURCES`` rows, conserving totals.
|
||||
def accumulate(rows: list[dict[str, Any]], items: Iterable[dict[str, Any]]) -> None:
|
||||
"""Add ``items`` to ``rows``, one row per (source, realized, estimated).
|
||||
|
||||
For a ledger that outlives one request: a WS session holds rows until a
|
||||
usage-carrying turn claims them, and :func:`from_tags` reads only the first
|
||||
``MAX_SOURCES``. Rows with the same (source, realized, estimated) sum into
|
||||
one; a new source past the cap sums into that pair's ``other`` row.
|
||||
``details`` stay only while every summed row agrees on them.
|
||||
Every writer goes through here -- the per-request ledger and a WS session's
|
||||
pending rows alike -- so the row count is bounded by the distinct sources
|
||||
that booked, and no row is ever dropped. Same-key rows sum ``tokens``,
|
||||
``usd`` and ``events``; ``details`` stay only while every summed row agrees.
|
||||
"""
|
||||
for item in items:
|
||||
key = _key(item)
|
||||
target = _row_for(pending, key)
|
||||
if target is None and len(pending) >= MAX_SOURCES - _OVERFLOW_SLOTS:
|
||||
key = ("other", key[1], key[2])
|
||||
target = _row_for(pending, key)
|
||||
item = {k: v for k, v in item.items() if k != "details"}
|
||||
target = _row_for(rows, key)
|
||||
if target is None:
|
||||
pending.append({**item, "source": key[0]})
|
||||
rows.append({**item, "source": key[0], "events": event_count(item)})
|
||||
continue
|
||||
target["tokens"] = int(target.get("tokens", 0)) + int(item.get("tokens", 0) or 0)
|
||||
target["usd"] = round(
|
||||
float(target.get("usd", 0.0)) + float(item.get("usd", 0.0) or 0.0), 12
|
||||
)
|
||||
target["tokens"] = _num(target.get("tokens"), int) + _num(item.get("tokens"), int)
|
||||
target["usd"] = round(_num(target.get("usd"), float) + _num(item.get("usd"), float), 12)
|
||||
target["events"] = event_count(target) + event_count(item)
|
||||
if target.get("details") != item.get("details"):
|
||||
target.pop("details", None)
|
||||
|
||||
|
||||
def _num(value: Any, cast: type[int] | type[float]) -> Any:
|
||||
# A row edited in place by a reader (deferral reconcile, plugins) may hold
|
||||
# junk; summing it must not raise inside the handler.
|
||||
try:
|
||||
number = cast(value or 0)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return cast(0)
|
||||
return number if math.isfinite(number) else cast(0)
|
||||
|
||||
|
||||
def _key(row: dict[str, Any]) -> tuple[str, bool, bool]:
|
||||
# ``realized`` defaults to True, as in every consumer of these rows.
|
||||
return (str(row.get("source")), bool(row.get("realized", True)), bool(row.get("estimated")))
|
||||
|
||||
|
||||
def _row_for(rows: list[dict[str, Any]], key: tuple[str, bool, bool]) -> dict[str, Any] | None:
|
||||
return next((row for row in rows if _key(row) == key), None)
|
||||
# The ledger is a plain list reachable through ``tags``; skip what this
|
||||
# module did not write rather than raise inside the handler.
|
||||
return next((row for row in rows if isinstance(row, dict) and _key(row) == key), None)
|
||||
|
||||
|
||||
_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG})
|
||||
@@ -287,3 +293,17 @@ def decode(value: str) -> list[dict[str, Any]]:
|
||||
if isinstance(decoded, list)
|
||||
else []
|
||||
)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
# MAX_SOURCES used to cap a request's ledger at 32 rows and silently drop
|
||||
# the rest; rows now coalesce per source instead. Kept for importers.
|
||||
if name == "MAX_SOURCES":
|
||||
warnings.warn(
|
||||
"headroom.proxy.savings_attribution.MAX_SOURCES no longer caps anything "
|
||||
"and will be removed; savings rows coalesce per source instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return 32
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
@@ -84,9 +84,10 @@ def deferred_booking(tags: object, tools: object) -> DeferredBooking | None:
|
||||
|
||||
``None`` when nothing is booked. The schemas are copied as they were when
|
||||
booked, so a later in-place edit cannot change what the credit stands for.
|
||||
The ledger entry is the latest ``tool_search`` one marked ``estimated``:
|
||||
both the core deferral and the native deferral hook book that way, and a
|
||||
hook's own measured saving never is, so it is never the one debited.
|
||||
The ledger entry is the ``tool_search`` row marked ``estimated``: both the
|
||||
core deferral and the native deferral hook book that way (rows coalesce, so
|
||||
they share it), and a hook's own measured saving never is, so it is never
|
||||
the one debited.
|
||||
"""
|
||||
if not isinstance(tags, dict) or "tool_search_deferred_tokens" not in tags:
|
||||
return None
|
||||
|
||||
@@ -578,3 +578,23 @@ def test_token_source_and_dollar_source_coexist(tmp_path, monkeypatch):
|
||||
def test_no_section_when_nothing_attributed(tmp_path, monkeypatch):
|
||||
out = _report_for([_savings_perf_line("none")], tmp_path, monkeypatch)
|
||||
assert "Savings by Source" not in out
|
||||
|
||||
|
||||
def test_coalesced_savings_rows_count_every_event_in_text_and_summary():
|
||||
"""A PERF row summed from 3 recordings is 3 events; a row without the field is 1."""
|
||||
report = PerfReport(
|
||||
perf_records=[
|
||||
PerfRecord(
|
||||
timestamp="2026-06-05 10:00:00,000",
|
||||
request_id="hr_1",
|
||||
savings_breakdown=[
|
||||
{"source": "tool_search", "tokens": 300, "events": 3},
|
||||
{"source": "router", "tokens": 5},
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
by_source = {row["source"]: row["events"] for row in build_perf_summary(report)["by_source"]}
|
||||
assert by_source == {"tool_search": 3, "router": 1}
|
||||
assert "tool_search: 3 events" in analyzer.format_report(report)
|
||||
|
||||
@@ -396,3 +396,69 @@ async def test_extension_savings_reach_the_metrics_call() -> None:
|
||||
|
||||
attribution = h.metrics.record_request.await_args.kwargs["savings_attribution"]
|
||||
assert [(row["source"], row["usd"]) for row in attribution] == [("routemegood", 0.42)]
|
||||
|
||||
|
||||
# --- no row cap: every recording is booked -------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_with_40_sources_books_every_token() -> None:
|
||||
"""The ledger used to stop at 32 rows and drop the 33rd recording on."""
|
||||
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
from headroom.proxy.savings_attribution import record_savings
|
||||
|
||||
tags: dict = {}
|
||||
for i in range(40):
|
||||
record_savings(tags, f"plugin{i}", tokens=100, usd=0.01)
|
||||
record_savings(tags, "plugin0", tokens=100, usd=0.01)
|
||||
|
||||
rows = from_tags(tags)
|
||||
metrics = PrometheusMetrics()
|
||||
await metrics.record_request(
|
||||
provider="openai",
|
||||
model="gpt-5.4",
|
||||
input_tokens=10,
|
||||
output_tokens=1,
|
||||
tokens_saved=0,
|
||||
latency_ms=1.0,
|
||||
savings_attribution=rows,
|
||||
)
|
||||
|
||||
assert len(rows) == 40
|
||||
booked = metrics.savings_by_source.values()
|
||||
assert sum(int(row["tokens"]) for row in booked) == 4_100
|
||||
assert sum(int(row["events"]) for row in booked) == 41
|
||||
assert metrics.savings_by_source["plugin0:1"]["events"] == 2
|
||||
|
||||
|
||||
def test_rows_coalesce_per_source_realized_and_estimated() -> None:
|
||||
from headroom.proxy.savings_attribution import record_savings
|
||||
|
||||
tags: dict = {}
|
||||
for _ in range(3):
|
||||
record_savings(tags, "router", tokens=1, realized=True)
|
||||
record_savings(tags, "router", tokens=1, realized=False)
|
||||
record_savings(tags, "router", tokens=1, estimated=True)
|
||||
|
||||
rows = from_tags(tags)
|
||||
assert [(r["realized"], r["estimated"], r["tokens"], r["events"]) for r in rows] == [
|
||||
(True, False, 3, 3),
|
||||
(False, False, 3, 3),
|
||||
(True, True, 3, 3),
|
||||
]
|
||||
|
||||
|
||||
def test_merging_into_a_junk_row_does_not_raise_inside_the_handler() -> None:
|
||||
from headroom.proxy.savings_attribution import record_savings
|
||||
|
||||
tags: dict = {SAVINGS_ATTRIBUTION_TAG: [{"source": "x", "tokens": None, "usd": "junk"}]}
|
||||
record_savings(tags, "x", tokens=5, usd=0.5)
|
||||
|
||||
assert [(r["tokens"], r["usd"], r["events"]) for r in from_tags(tags)] == [(5, 0.5, 2)]
|
||||
|
||||
|
||||
def test_max_sources_is_deprecated_not_removed() -> None:
|
||||
from headroom.proxy import savings_attribution
|
||||
|
||||
with pytest.warns(DeprecationWarning, match="MAX_SOURCES"):
|
||||
assert savings_attribution.MAX_SOURCES == 32
|
||||
|
||||
@@ -262,6 +262,23 @@ def test_request_attribute_provider_enriches_core_and_savings_metrics() -> None:
|
||||
unregister_otel_metric_attribute_provider(identity_attributes)
|
||||
|
||||
|
||||
def test_coalesced_savings_rows_count_every_event() -> None:
|
||||
"""A row summed from 33 recordings is 33 events, not 1; a bare row is 1."""
|
||||
reader = InMemoryMetricReader()
|
||||
otel_metrics = HeadroomOtelMetrics(meter_provider=MeterProvider(metric_readers=[reader]))
|
||||
|
||||
otel_metrics.record_savings_attribution(
|
||||
[
|
||||
{"source": "tool_search", "tokens": 3_300, "events": 33},
|
||||
{"source": "router", "tokens": 5},
|
||||
]
|
||||
)
|
||||
|
||||
events = _collect_metrics(reader)["headroom.savings.attribution.events"]
|
||||
assert _find_point(events, source="tool_search").value == 33
|
||||
assert _find_point(events, source="router").value == 1
|
||||
|
||||
|
||||
def test_failing_request_attribute_provider_is_fail_open() -> None:
|
||||
reader = InMemoryMetricReader()
|
||||
provider = MeterProvider(metric_readers=[reader])
|
||||
|
||||
@@ -2427,29 +2427,20 @@ class _UpstreamAfterSends(_FakeUpstream):
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("sources_per_frame", "expected_rows"),
|
||||
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 29)],
|
||||
[(lambda i: "tool_search", 2), (lambda i: f"src{i}", 34)],
|
||||
ids=["repeated_source", "distinct_sources"],
|
||||
)
|
||||
async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
|
||||
sources_per_frame, expected_rows
|
||||
):
|
||||
"""33 frames each booking 100 tokens must hand 3,300 tokens to the outcome.
|
||||
async def test_ws_pending_savings_beyond_32_rows_are_all_booked(sources_per_frame, expected_rows):
|
||||
"""33 pending frames hand every token and every event to the billed turn.
|
||||
|
||||
The session-wide pending list grew by each frame's rows (here a ``ws_fold``
|
||||
row and a 100-token hook row) and the outcome funnel reads at most
|
||||
``MAX_SOURCES`` rows, so 66 rows were cut to 32 and the rest cleared
|
||||
unbooked (1,600 of 3,300). Pending rows now coalesce per (source, realized,
|
||||
estimated) and distinct sources past the cap fold into ``other``.
|
||||
Rows coalesce to one per (source, realized, estimated) key, with no cap and
|
||||
no ``other`` bucket.
|
||||
"""
|
||||
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
from headroom.proxy.savings_attribution import (
|
||||
MAX_SOURCES,
|
||||
SAVINGS_ATTRIBUTION_TAG,
|
||||
from_tags,
|
||||
)
|
||||
from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG, from_tags
|
||||
from headroom.proxy.turn_hooks import clear_turn_hooks, register_turn_hook
|
||||
|
||||
frames_n = MAX_SOURCES + 1
|
||||
frames_n = 33
|
||||
seen = {"frame": 0}
|
||||
|
||||
class _DeferTools:
|
||||
@@ -2525,10 +2516,8 @@ async def test_ws_pending_savings_beyond_max_sources_are_all_booked(
|
||||
latency_ms=1.0,
|
||||
savings_attribution=rows,
|
||||
)
|
||||
booked = sum(
|
||||
int(row["tokens"])
|
||||
for row in metrics.savings_by_source.values()
|
||||
if row["source"] != "ws_fold"
|
||||
)
|
||||
assert booked == 100 * frames_n
|
||||
hook_rows = [row for row in metrics.savings_by_source.values() if row["source"] != "ws_fold"]
|
||||
assert sum(int(row["tokens"]) for row in hook_rows) == 100 * frames_n
|
||||
assert sum(int(row["events"]) for row in hook_rows) == frames_n
|
||||
assert "other" not in {row["source"] for row in hook_rows}
|
||||
assert stored == [expected_rows]
|
||||
|
||||
@@ -133,6 +133,7 @@ def test_request_runner_attributes_savings_with_handler_counters():
|
||||
"tokens": 15,
|
||||
"usd": 0.0,
|
||||
"details": {"message_tokens_saved": 14, "tool_tokens_saved": 1},
|
||||
"events": 1,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user