mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(savings): report negative savings instead of flooring them at zero (#3744)
## The problem A deployment that costs more than it saves reported **zero savings, never a loss**. `_coerce_float` in `savings_tracker.py` ended with `return max(coerced, 0.0)`. Every savings figure passed through it, so a negative result was silently clamped. The customer-visible symptom is the worst kind: a proxy actively adding cost looks, in the logs and on the dashboard, exactly like one that is merely breaking even. The same blind spot existed one layer up. `tokens_saved_total` and `cache_bust_tokens_lost` were both recorded and never compared, so compression losing more prefix cache than it saved in tokens produced no signal at all. The dashboard has rendered the net for a while — which only helps an operator who happens to be looking at it. ## The change - **`savings_tracker.py`** — add `_coerce_signed_float` (no floor) for the savings path, keeping the clamping variant where a floor is genuinely correct. When the running total goes negative, emit `event=savings_negative` with the running totals, throttled to one line per 5 minutes so a losing deployment reports its state without flooding. - **`prometheus_metrics.py`** — compare the two counters on each cache bust and emit `event=net_tokens_negative` when busts overtake savings. Edge-triggered: a losing deployment loses on *every* bust, so one line per bust would be the same noise problem in a different place. It re-arms if the deployment recovers and crosses again. - **`cost.py`** — surface `net_is_negative` on `compression_vs_cache` so `/stats` carries the same fact the warning does. ## Testing 13 new tests across two files covering the clamp, throttling, edge-triggering, re-arming after recovery, and the `/stats` payload. 552 related tests pass; ruff clean. Found while investigating a customer benchmark that showed +108% cost through the proxy — the run that motivated this had been reporting non-negative savings throughout. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- ### Update after self-review Removing the clamp at the point of measurement turned out not to be enough — **three more clamps downstream re-floored it**, so the warning fired while every number an operator looks at still read zero. Verified against a running tracker: ``` by_model compression_savings_usd : [0.0] <- for a turn that lost $0.60 projects compression_savings_usd : [0.0] lifetime ..._list_usd after 3x -0.50: -0.5 <- expected -1.50 ``` * `by_model` and `projects` applied `max(delta, 0.0)` on the way in — and those are exactly where you look to find *which* model or project is losing money. * The list-priced accumulators floored their own **previous** total before adding, so the column only ever showed the most recent losing turn. * The load path floored again, so a restart erased whatever survived. Fixed at 7 sites, with 5 regression tests covering the rollups, accumulation, restart and the public `/stats` snapshot. --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a4cb2bc021
commit
1455f00222
@@ -494,6 +494,11 @@ def build_prefix_cache_stats(
|
||||
"tokens_lost_to_cache_bust": metrics.cache_bust_tokens_lost,
|
||||
"cache_bust_count": metrics.cache_bust_count,
|
||||
"net_tokens": metrics.tokens_saved_total - metrics.cache_bust_tokens_lost,
|
||||
# Explicit rather than left for each consumer to re-derive: this is
|
||||
# the alerting condition (the proxy logs event=net_tokens_negative
|
||||
# on the same crossing), and a boolean in the payload is what a
|
||||
# scrape or a health check can key on without doing arithmetic.
|
||||
"net_is_negative": (metrics.tokens_saved_total - metrics.cache_bust_tokens_lost < 0),
|
||||
},
|
||||
"attribution": (
|
||||
"Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
|
||||
|
||||
@@ -342,6 +342,10 @@ class PrometheusMetrics:
|
||||
# Cache bust tracking: how many tokens lost their cache discount due to compression
|
||||
self.cache_bust_tokens_lost: int = 0
|
||||
self.cache_bust_count: int = 0
|
||||
# Edge-trigger latch for the net-negative warning: True once busts have
|
||||
# overtaken savings, cleared when the net recovers. See
|
||||
# _check_net_tokens_crossing_locked.
|
||||
self._net_tokens_negative: bool = False
|
||||
|
||||
# Cache-miss attribution (#1313): when a turn expected a prompt-cache
|
||||
# hit but got none, why? Bucketed by reason so operators can tell a
|
||||
@@ -472,6 +476,7 @@ class PrometheusMetrics:
|
||||
self.prefix_freeze_compression_foregone = 0
|
||||
self.cache_bust_tokens_lost = 0
|
||||
self.cache_bust_count = 0
|
||||
self._net_tokens_negative = False
|
||||
self.cache_miss_attribution_by_provider.clear()
|
||||
self.savings_history = []
|
||||
|
||||
@@ -1168,9 +1173,41 @@ class PrometheusMetrics:
|
||||
async with self._lock:
|
||||
self.cache_bust_tokens_lost += tokens_lost
|
||||
self.cache_bust_count += 1
|
||||
crossed = self._check_net_tokens_crossing_locked()
|
||||
if crossed is not None:
|
||||
saved, lost = crossed
|
||||
logger.warning(
|
||||
"event=net_tokens_negative tokens_saved=%d tokens_lost_to_cache_bust=%d "
|
||||
"net_tokens=%d busts=%d hint=%s",
|
||||
saved,
|
||||
lost,
|
||||
saved - lost,
|
||||
self.cache_bust_count,
|
||||
"prompt-cache busts now outweigh compression savings for this "
|
||||
"process; compression is a net loss on this traffic",
|
||||
)
|
||||
self.savings_tracker.record_lifetime_cache_bust(tokens_lost=tokens_lost)
|
||||
self._get_otel_metrics().record_proxy_cache_bust(tokens_lost=tokens_lost)
|
||||
|
||||
def _check_net_tokens_crossing_locked(self) -> tuple[int, int] | None:
|
||||
"""Return (saved, lost) the first time busts overtake savings, else None.
|
||||
|
||||
Edge-triggered, not level-triggered: a deployment that is losing is
|
||||
losing on every bust, and a warning per bust would be noise. The flag
|
||||
re-arms when the net returns to positive, so a deployment that crosses
|
||||
back and forth warns on each crossing rather than once forever.
|
||||
|
||||
Caller must hold ``self._lock``.
|
||||
"""
|
||||
|
||||
net_negative = self.cache_bust_tokens_lost > self.tokens_saved_total
|
||||
if net_negative and not self._net_tokens_negative:
|
||||
self._net_tokens_negative = True
|
||||
return self.tokens_saved_total, self.cache_bust_tokens_lost
|
||||
if not net_negative:
|
||||
self._net_tokens_negative = False
|
||||
return None
|
||||
|
||||
async def record_cache_miss_attribution(self, provider: str, reason: str) -> None:
|
||||
"""Record why a turn that expected a prompt-cache hit missed instead.
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import math
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from csv import DictWriter
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -54,6 +55,11 @@ DEFAULT_MAX_PROJECTS = 50
|
||||
DEFAULT_MAX_HISTORY_AGE_DAYS = 365
|
||||
DEFAULT_MAX_RESPONSE_HISTORY_POINTS = 500
|
||||
DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES = 60
|
||||
# Throttle for the negative-savings warning. The first one is immediate; after
|
||||
# that a losing deployment gets one summary line per interval rather than one
|
||||
# per request. Module-level indirection over the clock so tests can drive it.
|
||||
NEGATIVE_SAVINGS_WARN_INTERVAL_S = 300.0
|
||||
_monotonic = time.monotonic
|
||||
DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN = 3.0 / 1_000_000
|
||||
# Blended output price used only when litellm cannot price the model.
|
||||
DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN = 15.0 / 1_000_000
|
||||
@@ -189,6 +195,28 @@ def _coerce_float(value: Any, default: float = 0.0) -> float:
|
||||
return max(coerced, 0.0)
|
||||
|
||||
|
||||
def _coerce_signed_float(value: Any, default: float = 0.0) -> float:
|
||||
"""``_coerce_float`` without the zero floor, for quantities that can lose.
|
||||
|
||||
The floor in ``_coerce_float`` is right for the things it mostly guards --
|
||||
token counts, input costs, cumulative totals -- none of which have a
|
||||
meaningful negative value. It is wrong for per-request compression savings,
|
||||
which genuinely go negative when a rewrite displaces tokens out of the
|
||||
cached prefix and they come back billed at the fresh-input rate. Flooring
|
||||
those reported a gain on a turn that lost money. Kept as a separate helper
|
||||
rather than a flag on ``_coerce_float`` so the callers that want a floor
|
||||
keep getting one by default.
|
||||
"""
|
||||
|
||||
try:
|
||||
coerced = float(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return default
|
||||
if not math.isfinite(coerced):
|
||||
return default
|
||||
return coerced
|
||||
|
||||
|
||||
PROVIDER_UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@@ -680,7 +708,7 @@ def _normalize_projects(raw: Any) -> dict[str, dict[str, Any]]:
|
||||
normalized["requests"] = _coerce_int(entry.get("requests"))
|
||||
normalized["tokens_saved"] = _coerce_int(entry.get("tokens_saved"))
|
||||
normalized["compression_savings_usd"] = round(
|
||||
_coerce_float(entry.get("compression_savings_usd")), 6
|
||||
_coerce_signed_float(entry.get("compression_savings_usd")), 6
|
||||
)
|
||||
normalized["total_input_tokens"] = _coerce_int(entry.get("total_input_tokens"))
|
||||
normalized["total_input_cost_usd"] = round(
|
||||
@@ -715,7 +743,7 @@ def _normalize_by_model(raw: Any) -> dict[str, dict[str, Any]]:
|
||||
# Absent in state files written before this field existed -> 0.
|
||||
normalized["tool_tokens_saved"] = _coerce_int(entry.get("tool_tokens_saved"))
|
||||
normalized["compression_savings_usd"] = round(
|
||||
_coerce_float(entry.get("compression_savings_usd")), 6
|
||||
_coerce_signed_float(entry.get("compression_savings_usd")), 6
|
||||
)
|
||||
normalized["total_input_tokens"] = _coerce_int(entry.get("total_input_tokens"))
|
||||
normalized["total_input_cost_usd"] = round(
|
||||
@@ -750,12 +778,12 @@ def _normalize_display_session(entry: Any) -> dict[str, Any]:
|
||||
# cannot be recovered. Written out rather than inlined into the dict below:
|
||||
# a money path should not hinge on the reader parsing a nested ternary.
|
||||
is_pre_v6 = "compression_savings_list_usd" not in entry
|
||||
savings_usd = _coerce_float(entry.get("compression_savings_usd"))
|
||||
savings_usd = _coerce_signed_float(entry.get("compression_savings_usd"))
|
||||
if is_pre_v6:
|
||||
savings_list_usd = savings_usd
|
||||
savings_basis = BASIS_LIST
|
||||
else:
|
||||
savings_list_usd = _coerce_float(entry.get("compression_savings_list_usd"))
|
||||
savings_list_usd = _coerce_signed_float(entry.get("compression_savings_list_usd"))
|
||||
savings_basis = str(entry.get("savings_basis") or BASIS_UNKNOWN)
|
||||
|
||||
return {
|
||||
@@ -827,11 +855,57 @@ class SavingsTracker:
|
||||
self._needs_schema_save = False
|
||||
self._state = self._load_state()
|
||||
self._persistent_metrics = PersistentMetricsState(self._state.pop("lifetime_metrics", None))
|
||||
# Negative-savings warning state. A deployment that loses money loses it
|
||||
# on most turns, not one, so warning per turn would bury the signal in
|
||||
# its own repetition. Accumulate instead and emit a summary no more than
|
||||
# once per NEGATIVE_SAVINGS_WARN_INTERVAL_S — except the first, which is
|
||||
# immediate, because the operator should learn on turn one rather than a
|
||||
# minute in.
|
||||
self._negative_savings_count = 0
|
||||
self._negative_savings_usd = 0.0
|
||||
self._negative_savings_last_warn: float | None = None
|
||||
|
||||
@property
|
||||
def storage_path(self) -> str:
|
||||
return str(self._path)
|
||||
|
||||
def _note_negative_savings(
|
||||
self,
|
||||
*,
|
||||
model: str | None,
|
||||
delta_usd: float,
|
||||
compression: float,
|
||||
tool_schema: float,
|
||||
) -> None:
|
||||
"""Warn that a request's measured savings came out negative.
|
||||
|
||||
Negative savings mean the rewrite cost more than it removed on that turn
|
||||
— normally because it displaced tokens out of the cached prefix. It is
|
||||
not an accounting artifact and it is not self-correcting, so it warrants
|
||||
WARNING rather than a metric an operator has to go looking for.
|
||||
"""
|
||||
|
||||
self._negative_savings_count += 1
|
||||
self._negative_savings_usd += delta_usd
|
||||
now = _monotonic()
|
||||
last = self._negative_savings_last_warn
|
||||
if last is not None and (now - last) < NEGATIVE_SAVINGS_WARN_INTERVAL_S:
|
||||
return
|
||||
self._negative_savings_last_warn = now
|
||||
logger.warning(
|
||||
"event=savings_negative model=%s request_usd=%.6f compression_usd=%.6f "
|
||||
"tool_schema_usd=%.6f negative_requests=%d negative_usd_total=%.6f "
|
||||
"hint=%s",
|
||||
model or "unknown",
|
||||
delta_usd,
|
||||
compression,
|
||||
tool_schema,
|
||||
self._negative_savings_count,
|
||||
self._negative_savings_usd,
|
||||
"compression is costing more than it saves on this traffic; "
|
||||
"check prompt-cache busts in /stats prefix_cache.compression_vs_cache",
|
||||
)
|
||||
|
||||
def record_compression_savings(
|
||||
self,
|
||||
*,
|
||||
@@ -966,18 +1040,33 @@ class SavingsTracker:
|
||||
# As of v6 these two arrive CACHE-AWARE and region-correct: the
|
||||
# compression bucket priced against the live zone (never a cache
|
||||
# read) and the tool-schema bucket against the prefix (on a warm
|
||||
# turn, entirely a cache read). Neither is clamped at zero any more
|
||||
# than it was — but note the floor below is load-bearing in a new
|
||||
# way, since a live-zone token on a cold Anthropic turn is worth
|
||||
# 1.25x list, not 1.0x, and that is real money rather than an
|
||||
# artifact.
|
||||
delta_savings_usd = max(_coerce_float(priced.get("compression")), 0.0) + max(
|
||||
_coerce_float(priced.get("tool_schema")), 0.0
|
||||
)
|
||||
delta_savings_list_usd = max(_coerce_float(priced.get("compression_list")), 0.0) + max(
|
||||
_coerce_float(priced.get("tool_schema_list")), 0.0
|
||||
)
|
||||
# turn, entirely a cache read).
|
||||
#
|
||||
# These two are NOT floored at zero. A negative value here is a real
|
||||
# measurement, not an artifact: it says this turn's rewrite cost more
|
||||
# than it removed — the usual cause being that the rewrite moved
|
||||
# tokens out of the cached prefix (0.1x) and into the live zone,
|
||||
# where a cold Anthropic turn bills them at 1.25x list. Flooring that
|
||||
# at zero reported a gain on a turn that lost money, and the loss was
|
||||
# unrecoverable downstream because it was discarded here rather than
|
||||
# carried. The other two buckets below keep their floors: a provider
|
||||
# cache read is cheaper than a fresh read by construction, and output
|
||||
# shaping cannot lengthen the completion, so a negative there would
|
||||
# be an artifact rather than a loss.
|
||||
compression_usd = _coerce_signed_float(priced.get("compression"))
|
||||
tool_schema_usd = _coerce_signed_float(priced.get("tool_schema"))
|
||||
delta_savings_usd = compression_usd + tool_schema_usd
|
||||
delta_savings_list_usd = _coerce_signed_float(
|
||||
priced.get("compression_list")
|
||||
) + _coerce_signed_float(priced.get("tool_schema_list"))
|
||||
delta_basis = priced.get("basis")
|
||||
if delta_savings_usd < 0:
|
||||
self._note_negative_savings(
|
||||
model=model,
|
||||
delta_usd=delta_savings_usd,
|
||||
compression=compression_usd,
|
||||
tool_schema=tool_schema_usd,
|
||||
)
|
||||
else:
|
||||
# No priced breakdown available: only message savings are known here,
|
||||
# so this path stays message-only exactly as before — and at list
|
||||
@@ -1041,7 +1130,7 @@ class SavingsTracker:
|
||||
6,
|
||||
)
|
||||
lifetime["compression_savings_list_usd"] = round(
|
||||
_coerce_float(lifetime.get("compression_savings_list_usd"))
|
||||
_coerce_signed_float(lifetime.get("compression_savings_list_usd"))
|
||||
+ delta_savings_list_usd,
|
||||
6,
|
||||
)
|
||||
@@ -1078,7 +1167,8 @@ class SavingsTracker:
|
||||
6,
|
||||
)
|
||||
session["compression_savings_list_usd"] = round(
|
||||
_coerce_float(session.get("compression_savings_list_usd")) + delta_savings_list_usd,
|
||||
_coerce_signed_float(session.get("compression_savings_list_usd"))
|
||||
+ delta_savings_list_usd,
|
||||
6,
|
||||
)
|
||||
session["savings_basis"] = _blend_basis(session.get("savings_basis"), delta_basis)
|
||||
@@ -1276,7 +1366,7 @@ class SavingsTracker:
|
||||
entry["requests"] += max(requests_delta, 0)
|
||||
entry["tokens_saved"] += max(tokens_saved_delta, 0)
|
||||
entry["compression_savings_usd"] = round(
|
||||
entry["compression_savings_usd"] + max(savings_usd_delta, 0.0), 6
|
||||
entry["compression_savings_usd"] + savings_usd_delta, 6
|
||||
)
|
||||
entry["total_input_tokens"] += max(input_tokens_delta, 0)
|
||||
entry["total_input_cost_usd"] = round(
|
||||
@@ -1316,7 +1406,7 @@ class SavingsTracker:
|
||||
entry["tokens_saved"] += max(tokens_saved_delta, 0)
|
||||
entry["tool_tokens_saved"] += max(tool_tokens_saved_delta, 0)
|
||||
entry["compression_savings_usd"] = round(
|
||||
entry["compression_savings_usd"] + max(savings_usd_delta, 0.0), 6
|
||||
entry["compression_savings_usd"] + savings_usd_delta, 6
|
||||
)
|
||||
entry["total_input_tokens"] += max(input_tokens_delta, 0)
|
||||
entry["total_input_cost_usd"] = round(
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""A losing deployment must report the loss, and say so out loud.
|
||||
|
||||
Two independent ways a deployment can be net-negative, and both were silent:
|
||||
|
||||
1. A single request whose rewrite costs more than it removes. The priced
|
||||
buckets were floored at zero, so the request reported ``$0.00`` saved
|
||||
instead of a loss, and the loss was gone -- not merely hidden, but
|
||||
discarded before anything downstream could add it up.
|
||||
2. Cumulative prompt-cache busts overtaking cumulative compression savings.
|
||||
Both numbers were recorded, in different places, and never compared.
|
||||
|
||||
The dashboard has long rendered "Net negative" for case 2, which is no help to
|
||||
anyone running the proxy headless. Both cases now log at WARNING.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.proxy import savings_tracker as st
|
||||
from headroom.proxy.savings_tracker import SavingsTracker
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracker(tmp_path) -> SavingsTracker:
|
||||
return SavingsTracker(path=str(tmp_path / "savings.json"))
|
||||
|
||||
|
||||
def _priced(compression: float, tool_schema: float = 0.0) -> dict[str, object]:
|
||||
return {
|
||||
"compression": compression,
|
||||
"tool_schema": tool_schema,
|
||||
"compression_list": compression,
|
||||
"tool_schema_list": tool_schema,
|
||||
"basis": "cache_aware",
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# per-request: the number itself
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_losing_request_is_recorded_as_a_loss(tracker: SavingsTracker) -> None:
|
||||
"""The headline must be able to go below zero."""
|
||||
tracker.record_request(
|
||||
model="claude-sonnet-4",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.25),
|
||||
)
|
||||
|
||||
assert tracker._negative_savings_usd == pytest.approx(-0.25)
|
||||
assert tracker._negative_savings_count == 1
|
||||
|
||||
|
||||
def test_a_winning_request_is_untouched(tracker: SavingsTracker) -> None:
|
||||
tracker.record_request(
|
||||
model="claude-sonnet-4",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(0.25),
|
||||
)
|
||||
|
||||
assert tracker._negative_savings_count == 0
|
||||
|
||||
|
||||
def test_buckets_net_against_each_other(tracker: SavingsTracker) -> None:
|
||||
"""A loss in one bucket is not warned about if the request still wins."""
|
||||
tracker.record_request(
|
||||
model="claude-sonnet-4",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.10, tool_schema=0.40),
|
||||
)
|
||||
|
||||
assert tracker._negative_savings_count == 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# per-request: the warning
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_first_loss_warns_immediately(
|
||||
tracker: SavingsTracker, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
with caplog.at_level(logging.WARNING, logger=st.logger.name):
|
||||
tracker.record_request(
|
||||
model="claude-sonnet-4",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.25),
|
||||
)
|
||||
|
||||
assert "event=savings_negative" in caplog.text
|
||||
assert "model=claude-sonnet-4" in caplog.text
|
||||
|
||||
|
||||
def test_repeat_losses_are_throttled_but_still_counted(
|
||||
tracker: SavingsTracker,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""One line per interval, carrying the total -- not one line per request."""
|
||||
clock = {"t": 1000.0}
|
||||
monkeypatch.setattr(st, "_monotonic", lambda: clock["t"])
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=st.logger.name):
|
||||
for _ in range(5):
|
||||
tracker.record_request(
|
||||
model="claude-sonnet-4",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.20),
|
||||
)
|
||||
|
||||
warnings = [r for r in caplog.records if "event=savings_negative" in r.getMessage()]
|
||||
assert len(warnings) == 1
|
||||
# The suppressed four are still in the running total.
|
||||
assert tracker._negative_savings_count == 5
|
||||
assert tracker._negative_savings_usd == pytest.approx(-1.0)
|
||||
|
||||
|
||||
def test_warning_resumes_after_the_interval(
|
||||
tracker: SavingsTracker,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
clock = {"t": 1000.0}
|
||||
monkeypatch.setattr(st, "_monotonic", lambda: clock["t"])
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=st.logger.name):
|
||||
tracker.record_request(
|
||||
model="m",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.20),
|
||||
)
|
||||
clock["t"] += st.NEGATIVE_SAVINGS_WARN_INTERVAL_S + 1
|
||||
tracker.record_request(
|
||||
model="m",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd=_priced(-0.20),
|
||||
)
|
||||
|
||||
warnings = [r for r in caplog.records if "event=savings_negative" in r.getMessage()]
|
||||
assert len(warnings) == 2
|
||||
# The second line reports the cumulative figure, not just its own request.
|
||||
assert "negative_requests=2" in warnings[1].getMessage()
|
||||
|
||||
|
||||
def test_the_other_buckets_keep_their_floor(tracker: SavingsTracker) -> None:
|
||||
"""Output shaping and provider cache cannot go negative by construction."""
|
||||
tracker.record_request(
|
||||
model="m",
|
||||
tokens_saved=100,
|
||||
input_tokens=1000,
|
||||
estimated_savings_usd={
|
||||
**_priced(0.10),
|
||||
"output_shaping": -5.0,
|
||||
"provider_cache": -5.0,
|
||||
},
|
||||
)
|
||||
|
||||
assert tracker._negative_savings_count == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Removing the floor at the point of measurement is not enough on its own. The
|
||||
# loss then has to survive every place it is added up, stored and read back,
|
||||
# and it did not: three separate clamps downstream re-floored it, so the
|
||||
# warning fired while every number an operator would actually look at still
|
||||
# read zero.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _losing_turn(tracker: SavingsTracker, usd: float = -0.50, project: str = "acme") -> None:
|
||||
tracker.record_request(
|
||||
model="claude-opus-4-6",
|
||||
provider="anthropic",
|
||||
input_tokens=20_000,
|
||||
tokens_saved=-5_000,
|
||||
project=project,
|
||||
estimated_savings_usd={
|
||||
"compression": usd,
|
||||
"tool_schema": 0.0,
|
||||
"compression_list": usd,
|
||||
"tool_schema_list": 0.0,
|
||||
"basis": "mix",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_the_loss_reaches_the_per_model_rollup(tmp_path) -> None:
|
||||
"""``by_model`` is where an operator finds WHICH model is losing money.
|
||||
|
||||
It floored the delta at zero, so the one breakdown that could answer that
|
||||
question reported a clean zero for a turn that lost 50 cents.
|
||||
"""
|
||||
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
|
||||
|
||||
_losing_turn(tracker)
|
||||
|
||||
by_model = tracker._state["by_model"]
|
||||
assert [e["compression_savings_usd"] for e in by_model.values()] == [-0.5]
|
||||
|
||||
|
||||
def test_the_loss_reaches_the_per_project_rollup(tmp_path) -> None:
|
||||
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
|
||||
|
||||
_losing_turn(tracker)
|
||||
|
||||
projects = tracker._state["projects"]
|
||||
assert [e["compression_savings_usd"] for e in projects.values()] == [-0.5]
|
||||
|
||||
|
||||
def test_losses_accumulate_instead_of_resetting_each_turn(tmp_path) -> None:
|
||||
"""The list-priced column floored its own running total before adding to it.
|
||||
|
||||
Net effect: it only ever showed the MOST RECENT losing turn, so a
|
||||
deployment three turns into a sustained loss looked like it had lost once.
|
||||
"""
|
||||
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
|
||||
|
||||
for _ in range(3):
|
||||
_losing_turn(tracker)
|
||||
|
||||
lifetime = tracker._state["lifetime"]
|
||||
assert lifetime["compression_savings_usd"] == pytest.approx(-1.5)
|
||||
assert lifetime["compression_savings_list_usd"] == pytest.approx(-1.5)
|
||||
|
||||
|
||||
def test_a_loss_survives_a_restart(tmp_path) -> None:
|
||||
"""The load path floored too, so a restart erased whatever was preserved."""
|
||||
path = str(tmp_path / "s.json")
|
||||
tracker = SavingsTracker(path=path)
|
||||
_losing_turn(tracker)
|
||||
|
||||
reloaded = SavingsTracker(path=path)
|
||||
|
||||
by_model = reloaded._state["by_model"]
|
||||
assert [e["compression_savings_usd"] for e in by_model.values()] == [-0.5]
|
||||
|
||||
|
||||
def test_the_public_snapshot_shows_the_loss(tmp_path) -> None:
|
||||
"""What /stats serves must agree with what the warning said."""
|
||||
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
|
||||
|
||||
_losing_turn(tracker)
|
||||
lifetime = tracker.snapshot()["lifetime"]
|
||||
|
||||
assert lifetime["compression_savings_usd"] == pytest.approx(-0.5)
|
||||
assert lifetime["compression_savings_list_usd"] == pytest.approx(-0.5)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Cumulative cache-bust losses overtaking savings must reach the operator.
|
||||
|
||||
``tokens_saved_total`` and ``cache_bust_tokens_lost`` were both recorded and
|
||||
never compared, so a deployment where compression costs more cache than it
|
||||
saves tokens looked, from the logs, exactly like one where it did not. The
|
||||
dashboard has rendered the net for a while, which only helps an operator who
|
||||
is looking at it.
|
||||
|
||||
The warning is edge-triggered: a losing deployment loses on every bust, so a
|
||||
line per bust would be the same noise problem in a different place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.proxy.cost import CostTracker, build_prefix_cache_stats
|
||||
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
|
||||
|
||||
def _metrics_with_savings(tokens_saved: int) -> PrometheusMetrics:
|
||||
metrics = PrometheusMetrics()
|
||||
if tokens_saved:
|
||||
asyncio.run(
|
||||
metrics.record_request(
|
||||
provider="anthropic",
|
||||
model="claude-opus-4-6",
|
||||
input_tokens=1000,
|
||||
output_tokens=20,
|
||||
tokens_saved=tokens_saved,
|
||||
latency_ms=10.0,
|
||||
)
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||
return [r.getMessage() for r in caplog.records if "event=net_tokens_negative" in r.getMessage()]
|
||||
|
||||
|
||||
def test_warns_when_busts_overtake_savings(caplog: pytest.LogCaptureFixture) -> None:
|
||||
metrics = _metrics_with_savings(100)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
|
||||
asyncio.run(metrics.record_cache_bust(500))
|
||||
|
||||
warnings = _warnings(caplog)
|
||||
assert len(warnings) == 1
|
||||
assert "tokens_saved=100" in warnings[0]
|
||||
assert "tokens_lost_to_cache_bust=500" in warnings[0]
|
||||
assert "net_tokens=-400" in warnings[0]
|
||||
|
||||
|
||||
def test_silent_while_still_net_positive(caplog: pytest.LogCaptureFixture) -> None:
|
||||
metrics = _metrics_with_savings(1000)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
|
||||
asyncio.run(metrics.record_cache_bust(10))
|
||||
asyncio.run(metrics.record_cache_bust(10))
|
||||
|
||||
assert _warnings(caplog) == []
|
||||
|
||||
|
||||
def test_warns_once_per_crossing_not_once_per_bust(caplog: pytest.LogCaptureFixture) -> None:
|
||||
metrics = _metrics_with_savings(100)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
|
||||
for _ in range(5):
|
||||
asyncio.run(metrics.record_cache_bust(200))
|
||||
|
||||
assert len(_warnings(caplog)) == 1
|
||||
|
||||
|
||||
def test_rearms_after_recovering(caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""Crossing back into profit and out again is a second event, not silence."""
|
||||
metrics = _metrics_with_savings(100)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
|
||||
asyncio.run(metrics.record_cache_bust(200)) # net -100, warns
|
||||
asyncio.run(
|
||||
metrics.record_request(
|
||||
provider="anthropic",
|
||||
model="claude-opus-4-6",
|
||||
input_tokens=1000,
|
||||
output_tokens=20,
|
||||
tokens_saved=1000,
|
||||
latency_ms=10.0,
|
||||
)
|
||||
)
|
||||
asyncio.run(metrics.record_cache_bust(1)) # net +899, re-arms
|
||||
asyncio.run(metrics.record_cache_bust(5000)) # net negative again, warns
|
||||
|
||||
assert len(_warnings(caplog)) == 2
|
||||
|
||||
|
||||
def test_stats_payload_carries_the_flag() -> None:
|
||||
metrics = _metrics_with_savings(100)
|
||||
asyncio.run(metrics.record_cache_bust(500))
|
||||
|
||||
stats = build_prefix_cache_stats(metrics, CostTracker())
|
||||
cvc = stats["compression_vs_cache"]
|
||||
|
||||
assert cvc["net_tokens"] == -400
|
||||
assert cvc["net_is_negative"] is True
|
||||
|
||||
|
||||
def test_stats_flag_is_false_when_winning() -> None:
|
||||
metrics = _metrics_with_savings(1000)
|
||||
asyncio.run(metrics.record_cache_bust(10))
|
||||
|
||||
cvc = build_prefix_cache_stats(metrics, CostTracker())["compression_vs_cache"]
|
||||
|
||||
assert cvc["net_tokens"] == 990
|
||||
assert cvc["net_is_negative"] is False
|
||||
Reference in New Issue
Block a user