fix(metrics): keep tool_search's savings row named past the source cap

After 256 rows, a tool_search saving seen for the first time folded into
"other", so the documented headline query (source="tool_search") lost
it. savings_by_source now exempts the same core sources the
per-request ledger never folds (UNFOLDED_SOURCES, made public so both
read one set). Covered for realized and projected rows first seen after
the cap.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-11-07 13:56:50 +05:30
co-authored by Claude Opus 5.5
parent fa88cf4bca
commit 7196e28d6f
3 changed files with 48 additions and 11 deletions
+4 -3
View File
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
from headroom import savings_ledger
from headroom.observability import get_otel_metrics
from headroom.proxy.savings_attribution import event_count
from headroom.proxy.savings_attribution import UNFOLDED_SOURCES, event_count
from headroom.proxy.savings_tracker import SavingsTracker, estimate_request_savings_usd
logger = logging.getLogger("headroom.proxy")
@@ -44,8 +44,8 @@ _OTHER_PATH = "other"
# from extensions and turn hooks, so one that mints a name per request would grow
# savings_by_source, and its three exported series, without bound. Past this many
# rows, new sources are summed into ``_OTHER_SOURCE``: only the name is lost,
# never a token or a dollar. Unlike the per-request ledger's bound, ``tool_search``
# may fold here: nothing reads this table by source name, every reader sums it.
# never a token or a dollar. As in the per-request ledger, ``UNFOLDED_SOURCES``
# never fold: the documented headline query selects ``source="tool_search"``.
MAX_DISTINCT_SAVINGS_ROWS = 256
_OTHER_SOURCE = "other"
@@ -997,6 +997,7 @@ class PrometheusMetrics:
key = f"{source}:{int(realized)}"
if (
key not in self.savings_by_source
and source not in UNFOLDED_SOURCES
and len(self.savings_by_source) >= MAX_DISTINCT_SAVINGS_ROWS
):
source, key = _OTHER_SOURCE, f"{_OTHER_SOURCE}:{int(realized)}"
+5 -8
View File
@@ -58,8 +58,9 @@ MAX_STAGE_MS = 3_600_000.0
MAX_NAMED_ROWS = 256
OVERFLOW_SOURCE = "other"
# Core-booked sources never fold: ``tool_schema_savings_policy`` finds the
# deferral's row by name to lower it when hooks un-defer tools.
_UNFOLDED_SOURCES = frozenset({"tool_search"})
# deferral's row by name to lower it when hooks un-defer tools, and the
# metrics headline query selects ``source="tool_search"``.
UNFOLDED_SOURCES = frozenset({"tool_search"})
# A row's sums saturate here instead of overflowing to ``inf`` (see
# MAX_STAGE_MS for what a non-finite value does downstream). Both are far
@@ -232,16 +233,12 @@ def accumulate(rows: list[dict[str, Any]], items: Iterable[dict[str, Any]]) -> N
``tokens``, ``usd`` and ``events``; ``details`` stay only while every
summed row agrees. Once ``rows`` holds ``MAX_NAMED_ROWS`` rows, a new
source folds into ``OVERFLOW_SOURCE`` (which also takes unnamed savings),
so the list grows by at most the overflow and ``_UNFOLDED_SOURCES`` keys.
so the list grows by at most the overflow and ``UNFOLDED_SOURCES`` keys.
"""
for item in items:
key = _key(item)
target = _row_for(rows, key)
if (
target is None
and key[0] not in _UNFOLDED_SOURCES
and _row_count(rows) >= MAX_NAMED_ROWS
):
if target is None and key[0] not in UNFOLDED_SOURCES and _row_count(rows) >= MAX_NAMED_ROWS:
if key[0] != OVERFLOW_SOURCE:
# Details describe one source; the overflow row mixes several.
item = {k: v for k, v in item.items() if k != "details"}
+39
View File
@@ -670,6 +670,45 @@ async def test_prometheus_metrics_caps_savings_source_cardinality() -> None:
assert metrics.savings_by_source == {}
@pytest.mark.asyncio
@pytest.mark.parametrize("realized", [True, False])
async def test_tool_search_first_seen_after_the_savings_cap_keeps_its_row(realized) -> None:
"""The headline query selects source="tool_search", so a tool-search saving
that first arrives after the cap must keep its named row, not fold."""
metrics = PrometheusMetrics(stateless=True)
for i in range(MAX_DISTINCT_SAVINGS_ROWS):
await metrics.record_request(
provider="openai",
model="gpt-4o",
input_tokens=10,
output_tokens=1,
tokens_saved=0,
latency_ms=1.0,
savings_attribution=[{"source": f"plugin_{i}", "realized": realized, "tokens": 1}],
)
await metrics.record_request(
provider="openai",
model="gpt-4o",
input_tokens=10,
output_tokens=1,
tokens_saved=0,
latency_ms=1.0,
savings_attribution=[
{"source": "tool_search", "realized": realized, "tokens": 100},
{"source": "late_plugin", "realized": realized, "tokens": 7},
],
)
rows = metrics.savings_by_source
assert rows[f"tool_search:{int(realized)}"]["tokens"] == 100
assert rows[f"other:{int(realized)}"]["tokens"] == 7
exported = await metrics.export()
assert (
f'headroom_savings_attributed_tokens_total{{realized="{str(realized).lower()}",'
f'source="tool_search"}} 100' in exported
)
@pytest.mark.asyncio
async def test_prometheus_metrics_export_bounds_model_series() -> None:
"""export() emits at most MAX_DISTINCT_MODELS model series plus the 'other' bucket."""