mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(metrics): keep tool_search's savings row named past the source cap
After 256 rows, a tool_search saving seen for the first time folded into "other", so the documented headline query (source="tool_search") lost it. savings_by_source now exempts the same core sources the per-request ledger never folds (UNFOLDED_SOURCES, made public so both read one set). Covered for realized and projected rows first seen after the cap. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fa88cf4bca
commit
7196e28d6f
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from headroom import savings_ledger
|
||||
from headroom.observability import get_otel_metrics
|
||||
from headroom.proxy.savings_attribution import event_count
|
||||
from headroom.proxy.savings_attribution import UNFOLDED_SOURCES, event_count
|
||||
from headroom.proxy.savings_tracker import SavingsTracker, estimate_request_savings_usd
|
||||
|
||||
logger = logging.getLogger("headroom.proxy")
|
||||
@@ -44,8 +44,8 @@ _OTHER_PATH = "other"
|
||||
# from extensions and turn hooks, so one that mints a name per request would grow
|
||||
# savings_by_source, and its three exported series, without bound. Past this many
|
||||
# rows, new sources are summed into ``_OTHER_SOURCE``: only the name is lost,
|
||||
# never a token or a dollar. Unlike the per-request ledger's bound, ``tool_search``
|
||||
# may fold here: nothing reads this table by source name, every reader sums it.
|
||||
# never a token or a dollar. As in the per-request ledger, ``UNFOLDED_SOURCES``
|
||||
# never fold: the documented headline query selects ``source="tool_search"``.
|
||||
MAX_DISTINCT_SAVINGS_ROWS = 256
|
||||
_OTHER_SOURCE = "other"
|
||||
|
||||
@@ -997,6 +997,7 @@ class PrometheusMetrics:
|
||||
key = f"{source}:{int(realized)}"
|
||||
if (
|
||||
key not in self.savings_by_source
|
||||
and source not in UNFOLDED_SOURCES
|
||||
and len(self.savings_by_source) >= MAX_DISTINCT_SAVINGS_ROWS
|
||||
):
|
||||
source, key = _OTHER_SOURCE, f"{_OTHER_SOURCE}:{int(realized)}"
|
||||
|
||||
@@ -58,8 +58,9 @@ MAX_STAGE_MS = 3_600_000.0
|
||||
MAX_NAMED_ROWS = 256
|
||||
OVERFLOW_SOURCE = "other"
|
||||
# Core-booked sources never fold: ``tool_schema_savings_policy`` finds the
|
||||
# deferral's row by name to lower it when hooks un-defer tools.
|
||||
_UNFOLDED_SOURCES = frozenset({"tool_search"})
|
||||
# deferral's row by name to lower it when hooks un-defer tools, and the
|
||||
# metrics headline query selects ``source="tool_search"``.
|
||||
UNFOLDED_SOURCES = frozenset({"tool_search"})
|
||||
|
||||
# A row's sums saturate here instead of overflowing to ``inf`` (see
|
||||
# MAX_STAGE_MS for what a non-finite value does downstream). Both are far
|
||||
@@ -232,16 +233,12 @@ def accumulate(rows: list[dict[str, Any]], items: Iterable[dict[str, Any]]) -> N
|
||||
``tokens``, ``usd`` and ``events``; ``details`` stay only while every
|
||||
summed row agrees. Once ``rows`` holds ``MAX_NAMED_ROWS`` rows, a new
|
||||
source folds into ``OVERFLOW_SOURCE`` (which also takes unnamed savings),
|
||||
so the list grows by at most the overflow and ``_UNFOLDED_SOURCES`` keys.
|
||||
so the list grows by at most the overflow and ``UNFOLDED_SOURCES`` keys.
|
||||
"""
|
||||
for item in items:
|
||||
key = _key(item)
|
||||
target = _row_for(rows, key)
|
||||
if (
|
||||
target is None
|
||||
and key[0] not in _UNFOLDED_SOURCES
|
||||
and _row_count(rows) >= MAX_NAMED_ROWS
|
||||
):
|
||||
if target is None and key[0] not in UNFOLDED_SOURCES and _row_count(rows) >= MAX_NAMED_ROWS:
|
||||
if key[0] != OVERFLOW_SOURCE:
|
||||
# Details describe one source; the overflow row mixes several.
|
||||
item = {k: v for k, v in item.items() if k != "details"}
|
||||
|
||||
@@ -670,6 +670,45 @@ async def test_prometheus_metrics_caps_savings_source_cardinality() -> None:
|
||||
assert metrics.savings_by_source == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("realized", [True, False])
|
||||
async def test_tool_search_first_seen_after_the_savings_cap_keeps_its_row(realized) -> None:
|
||||
"""The headline query selects source="tool_search", so a tool-search saving
|
||||
that first arrives after the cap must keep its named row, not fold."""
|
||||
metrics = PrometheusMetrics(stateless=True)
|
||||
for i in range(MAX_DISTINCT_SAVINGS_ROWS):
|
||||
await metrics.record_request(
|
||||
provider="openai",
|
||||
model="gpt-4o",
|
||||
input_tokens=10,
|
||||
output_tokens=1,
|
||||
tokens_saved=0,
|
||||
latency_ms=1.0,
|
||||
savings_attribution=[{"source": f"plugin_{i}", "realized": realized, "tokens": 1}],
|
||||
)
|
||||
await metrics.record_request(
|
||||
provider="openai",
|
||||
model="gpt-4o",
|
||||
input_tokens=10,
|
||||
output_tokens=1,
|
||||
tokens_saved=0,
|
||||
latency_ms=1.0,
|
||||
savings_attribution=[
|
||||
{"source": "tool_search", "realized": realized, "tokens": 100},
|
||||
{"source": "late_plugin", "realized": realized, "tokens": 7},
|
||||
],
|
||||
)
|
||||
|
||||
rows = metrics.savings_by_source
|
||||
assert rows[f"tool_search:{int(realized)}"]["tokens"] == 100
|
||||
assert rows[f"other:{int(realized)}"]["tokens"] == 7
|
||||
exported = await metrics.export()
|
||||
assert (
|
||||
f'headroom_savings_attributed_tokens_total{{realized="{str(realized).lower()}",'
|
||||
f'source="tool_search"}} 100' in exported
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prometheus_metrics_export_bounds_model_series() -> None:
|
||||
"""export() emits at most MAX_DISTINCT_MODELS model series plus the 'other' bucket."""
|
||||
|
||||
Reference in New Issue
Block a user