fix(savings): report negative savings instead of flooring them at zero (#3744)

## The problem

A deployment that costs more than it saves reported **zero savings,
never a loss**.

`_coerce_float` in `savings_tracker.py` ended with `return max(coerced,
0.0)`. Every savings figure passed through it, so a negative result was
silently clamped. The customer-visible symptom is the worst kind: a
proxy actively adding cost looks, in the logs and on the dashboard,
exactly like one that is merely breaking even.

The same blind spot existed one layer up. `tokens_saved_total` and
`cache_bust_tokens_lost` were both recorded and never compared, so
compression losing more prefix cache than it saved in tokens produced no
signal at all. The dashboard has rendered the net for a while — which
only helps an operator who happens to be looking at it.

## The change

- **`savings_tracker.py`** — add `_coerce_signed_float` (no floor) for
the savings path, keeping the clamping variant where a floor is
genuinely correct. When the running total goes negative, emit
`event=savings_negative` with the running totals, throttled to one line
per 5 minutes so a losing deployment reports its state without flooding.
- **`prometheus_metrics.py`** — compare the two counters on each cache
bust and emit `event=net_tokens_negative` when busts overtake savings.
Edge-triggered: a losing deployment loses on *every* bust, so one line
per bust would be the same noise problem in a different place. It
re-arms if the deployment recovers and crosses again.
- **`cost.py`** — surface `net_is_negative` on `compression_vs_cache` so
`/stats` carries the same fact the warning does.

## Testing

13 new tests across two files covering the clamp, throttling,
edge-triggering, re-arming after recovery, and the `/stats` payload. 552
related tests pass; ruff clean.

Found while investigating a customer benchmark that showed +108% cost
through the proxy — the run that motivated this had been reporting
non-negative savings throughout.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---

### Update after self-review

Removing the clamp at the point of measurement turned out not to be
enough — **three more clamps downstream re-floored it**, so the warning
fired while every number an operator looks at still read zero. Verified
against a running tracker:

```
by_model compression_savings_usd : [0.0]   <- for a turn that lost $0.60
projects compression_savings_usd : [0.0]
lifetime ..._list_usd after 3x -0.50: -0.5 <- expected -1.50
```

* `by_model` and `projects` applied `max(delta, 0.0)` on the way in —
and those are exactly where you look to find *which* model or project is
losing money.
* The list-priced accumulators floored their own **previous** total
before adding, so the column only ever showed the most recent losing
turn.
* The load path floored again, so a restart erased whatever survived.

Fixed at 7 sites, with 5 regression tests covering the rollups,
accumulation, restart and the public `/stats` snapshot.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-09-23 22:39:02 +02:00
committed by GitHub
co-authored by Claude Opus 5
parent a4cb2bc021
commit 1455f00222
5 changed files with 525 additions and 19 deletions
+5
View File
@@ -494,6 +494,11 @@ def build_prefix_cache_stats(
"tokens_lost_to_cache_bust": metrics.cache_bust_tokens_lost,
"cache_bust_count": metrics.cache_bust_count,
"net_tokens": metrics.tokens_saved_total - metrics.cache_bust_tokens_lost,
# Explicit rather than left for each consumer to re-derive: this is
# the alerting condition (the proxy logs event=net_tokens_negative
# on the same crossing), and a boolean in the payload is what a
# scrape or a health check can key on without doing arithmetic.
"net_is_negative": (metrics.tokens_saved_total - metrics.cache_bust_tokens_lost < 0),
},
"attribution": (
"Prefix caching is performed by the LLM provider (Anthropic, OpenAI). "
+37
View File
@@ -342,6 +342,10 @@ class PrometheusMetrics:
# Cache bust tracking: how many tokens lost their cache discount due to compression
self.cache_bust_tokens_lost: int = 0
self.cache_bust_count: int = 0
# Edge-trigger latch for the net-negative warning: True once busts have
# overtaken savings, cleared when the net recovers. See
# _check_net_tokens_crossing_locked.
self._net_tokens_negative: bool = False
# Cache-miss attribution (#1313): when a turn expected a prompt-cache
# hit but got none, why? Bucketed by reason so operators can tell a
@@ -472,6 +476,7 @@ class PrometheusMetrics:
self.prefix_freeze_compression_foregone = 0
self.cache_bust_tokens_lost = 0
self.cache_bust_count = 0
self._net_tokens_negative = False
self.cache_miss_attribution_by_provider.clear()
self.savings_history = []
@@ -1168,9 +1173,41 @@ class PrometheusMetrics:
async with self._lock:
self.cache_bust_tokens_lost += tokens_lost
self.cache_bust_count += 1
crossed = self._check_net_tokens_crossing_locked()
if crossed is not None:
saved, lost = crossed
logger.warning(
"event=net_tokens_negative tokens_saved=%d tokens_lost_to_cache_bust=%d "
"net_tokens=%d busts=%d hint=%s",
saved,
lost,
saved - lost,
self.cache_bust_count,
"prompt-cache busts now outweigh compression savings for this "
"process; compression is a net loss on this traffic",
)
self.savings_tracker.record_lifetime_cache_bust(tokens_lost=tokens_lost)
self._get_otel_metrics().record_proxy_cache_bust(tokens_lost=tokens_lost)
def _check_net_tokens_crossing_locked(self) -> tuple[int, int] | None:
"""Return (saved, lost) the first time busts overtake savings, else None.
Edge-triggered, not level-triggered: a deployment that is losing is
losing on every bust, and a warning per bust would be noise. The flag
re-arms when the net returns to positive, so a deployment that crosses
back and forth warns on each crossing rather than once forever.
Caller must hold ``self._lock``.
"""
net_negative = self.cache_bust_tokens_lost > self.tokens_saved_total
if net_negative and not self._net_tokens_negative:
self._net_tokens_negative = True
return self.tokens_saved_total, self.cache_bust_tokens_lost
if not net_negative:
self._net_tokens_negative = False
return None
async def record_cache_miss_attribution(self, provider: str, reason: str) -> None:
"""Record why a turn that expected a prompt-cache hit missed instead.
+109 -19
View File
@@ -14,6 +14,7 @@ import math
import os
import tempfile
import threading
import time
from collections.abc import Mapping
from csv import DictWriter
from datetime import datetime, timedelta, timezone
@@ -54,6 +55,11 @@ DEFAULT_MAX_PROJECTS = 50
DEFAULT_MAX_HISTORY_AGE_DAYS = 365
DEFAULT_MAX_RESPONSE_HISTORY_POINTS = 500
DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES = 60
# Throttle for the negative-savings warning. The first one is immediate; after
# that a losing deployment gets one summary line per interval rather than one
# per request. Module-level indirection over the clock so tests can drive it.
NEGATIVE_SAVINGS_WARN_INTERVAL_S = 300.0
_monotonic = time.monotonic
DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN = 3.0 / 1_000_000
# Blended output price used only when litellm cannot price the model.
DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN = 15.0 / 1_000_000
@@ -189,6 +195,28 @@ def _coerce_float(value: Any, default: float = 0.0) -> float:
return max(coerced, 0.0)
def _coerce_signed_float(value: Any, default: float = 0.0) -> float:
"""``_coerce_float`` without the zero floor, for quantities that can lose.
The floor in ``_coerce_float`` is right for the things it mostly guards --
token counts, input costs, cumulative totals -- none of which have a
meaningful negative value. It is wrong for per-request compression savings,
which genuinely go negative when a rewrite displaces tokens out of the
cached prefix and they come back billed at the fresh-input rate. Flooring
those reported a gain on a turn that lost money. Kept as a separate helper
rather than a flag on ``_coerce_float`` so the callers that want a floor
keep getting one by default.
"""
try:
coerced = float(value)
except (TypeError, ValueError, OverflowError):
return default
if not math.isfinite(coerced):
return default
return coerced
PROVIDER_UNKNOWN = "unknown"
@@ -680,7 +708,7 @@ def _normalize_projects(raw: Any) -> dict[str, dict[str, Any]]:
normalized["requests"] = _coerce_int(entry.get("requests"))
normalized["tokens_saved"] = _coerce_int(entry.get("tokens_saved"))
normalized["compression_savings_usd"] = round(
_coerce_float(entry.get("compression_savings_usd")), 6
_coerce_signed_float(entry.get("compression_savings_usd")), 6
)
normalized["total_input_tokens"] = _coerce_int(entry.get("total_input_tokens"))
normalized["total_input_cost_usd"] = round(
@@ -715,7 +743,7 @@ def _normalize_by_model(raw: Any) -> dict[str, dict[str, Any]]:
# Absent in state files written before this field existed -> 0.
normalized["tool_tokens_saved"] = _coerce_int(entry.get("tool_tokens_saved"))
normalized["compression_savings_usd"] = round(
_coerce_float(entry.get("compression_savings_usd")), 6
_coerce_signed_float(entry.get("compression_savings_usd")), 6
)
normalized["total_input_tokens"] = _coerce_int(entry.get("total_input_tokens"))
normalized["total_input_cost_usd"] = round(
@@ -750,12 +778,12 @@ def _normalize_display_session(entry: Any) -> dict[str, Any]:
# cannot be recovered. Written out rather than inlined into the dict below:
# a money path should not hinge on the reader parsing a nested ternary.
is_pre_v6 = "compression_savings_list_usd" not in entry
savings_usd = _coerce_float(entry.get("compression_savings_usd"))
savings_usd = _coerce_signed_float(entry.get("compression_savings_usd"))
if is_pre_v6:
savings_list_usd = savings_usd
savings_basis = BASIS_LIST
else:
savings_list_usd = _coerce_float(entry.get("compression_savings_list_usd"))
savings_list_usd = _coerce_signed_float(entry.get("compression_savings_list_usd"))
savings_basis = str(entry.get("savings_basis") or BASIS_UNKNOWN)
return {
@@ -827,11 +855,57 @@ class SavingsTracker:
self._needs_schema_save = False
self._state = self._load_state()
self._persistent_metrics = PersistentMetricsState(self._state.pop("lifetime_metrics", None))
# Negative-savings warning state. A deployment that loses money loses it
# on most turns, not one, so warning per turn would bury the signal in
# its own repetition. Accumulate instead and emit a summary no more than
# once per NEGATIVE_SAVINGS_WARN_INTERVAL_S — except the first, which is
# immediate, because the operator should learn on turn one rather than a
# minute in.
self._negative_savings_count = 0
self._negative_savings_usd = 0.0
self._negative_savings_last_warn: float | None = None
@property
def storage_path(self) -> str:
return str(self._path)
def _note_negative_savings(
self,
*,
model: str | None,
delta_usd: float,
compression: float,
tool_schema: float,
) -> None:
"""Warn that a request's measured savings came out negative.
Negative savings mean the rewrite cost more than it removed on that turn
— normally because it displaced tokens out of the cached prefix. It is
not an accounting artifact and it is not self-correcting, so it warrants
WARNING rather than a metric an operator has to go looking for.
"""
self._negative_savings_count += 1
self._negative_savings_usd += delta_usd
now = _monotonic()
last = self._negative_savings_last_warn
if last is not None and (now - last) < NEGATIVE_SAVINGS_WARN_INTERVAL_S:
return
self._negative_savings_last_warn = now
logger.warning(
"event=savings_negative model=%s request_usd=%.6f compression_usd=%.6f "
"tool_schema_usd=%.6f negative_requests=%d negative_usd_total=%.6f "
"hint=%s",
model or "unknown",
delta_usd,
compression,
tool_schema,
self._negative_savings_count,
self._negative_savings_usd,
"compression is costing more than it saves on this traffic; "
"check prompt-cache busts in /stats prefix_cache.compression_vs_cache",
)
def record_compression_savings(
self,
*,
@@ -966,18 +1040,33 @@ class SavingsTracker:
# As of v6 these two arrive CACHE-AWARE and region-correct: the
# compression bucket priced against the live zone (never a cache
# read) and the tool-schema bucket against the prefix (on a warm
# turn, entirely a cache read). Neither is clamped at zero any more
# than it was — but note the floor below is load-bearing in a new
# way, since a live-zone token on a cold Anthropic turn is worth
# 1.25x list, not 1.0x, and that is real money rather than an
# artifact.
delta_savings_usd = max(_coerce_float(priced.get("compression")), 0.0) + max(
_coerce_float(priced.get("tool_schema")), 0.0
)
delta_savings_list_usd = max(_coerce_float(priced.get("compression_list")), 0.0) + max(
_coerce_float(priced.get("tool_schema_list")), 0.0
)
# turn, entirely a cache read).
#
# These two are NOT floored at zero. A negative value here is a real
# measurement, not an artifact: it says this turn's rewrite cost more
# than it removed — the usual cause being that the rewrite moved
# tokens out of the cached prefix (0.1x) and into the live zone,
# where a cold Anthropic turn bills them at 1.25x list. Flooring that
# at zero reported a gain on a turn that lost money, and the loss was
# unrecoverable downstream because it was discarded here rather than
# carried. The other two buckets below keep their floors: a provider
# cache read is cheaper than a fresh read by construction, and output
# shaping cannot lengthen the completion, so a negative there would
# be an artifact rather than a loss.
compression_usd = _coerce_signed_float(priced.get("compression"))
tool_schema_usd = _coerce_signed_float(priced.get("tool_schema"))
delta_savings_usd = compression_usd + tool_schema_usd
delta_savings_list_usd = _coerce_signed_float(
priced.get("compression_list")
) + _coerce_signed_float(priced.get("tool_schema_list"))
delta_basis = priced.get("basis")
if delta_savings_usd < 0:
self._note_negative_savings(
model=model,
delta_usd=delta_savings_usd,
compression=compression_usd,
tool_schema=tool_schema_usd,
)
else:
# No priced breakdown available: only message savings are known here,
# so this path stays message-only exactly as before — and at list
@@ -1041,7 +1130,7 @@ class SavingsTracker:
6,
)
lifetime["compression_savings_list_usd"] = round(
_coerce_float(lifetime.get("compression_savings_list_usd"))
_coerce_signed_float(lifetime.get("compression_savings_list_usd"))
+ delta_savings_list_usd,
6,
)
@@ -1078,7 +1167,8 @@ class SavingsTracker:
6,
)
session["compression_savings_list_usd"] = round(
_coerce_float(session.get("compression_savings_list_usd")) + delta_savings_list_usd,
_coerce_signed_float(session.get("compression_savings_list_usd"))
+ delta_savings_list_usd,
6,
)
session["savings_basis"] = _blend_basis(session.get("savings_basis"), delta_basis)
@@ -1276,7 +1366,7 @@ class SavingsTracker:
entry["requests"] += max(requests_delta, 0)
entry["tokens_saved"] += max(tokens_saved_delta, 0)
entry["compression_savings_usd"] = round(
entry["compression_savings_usd"] + max(savings_usd_delta, 0.0), 6
entry["compression_savings_usd"] + savings_usd_delta, 6
)
entry["total_input_tokens"] += max(input_tokens_delta, 0)
entry["total_input_cost_usd"] = round(
@@ -1316,7 +1406,7 @@ class SavingsTracker:
entry["tokens_saved"] += max(tokens_saved_delta, 0)
entry["tool_tokens_saved"] += max(tool_tokens_saved_delta, 0)
entry["compression_savings_usd"] = round(
entry["compression_savings_usd"] + max(savings_usd_delta, 0.0), 6
entry["compression_savings_usd"] + savings_usd_delta, 6
)
entry["total_input_tokens"] += max(input_tokens_delta, 0)
entry["total_input_cost_usd"] = round(
+257
View File
@@ -0,0 +1,257 @@
"""A losing deployment must report the loss, and say so out loud.
Two independent ways a deployment can be net-negative, and both were silent:
1. A single request whose rewrite costs more than it removes. The priced
buckets were floored at zero, so the request reported ``$0.00`` saved
instead of a loss, and the loss was gone -- not merely hidden, but
discarded before anything downstream could add it up.
2. Cumulative prompt-cache busts overtaking cumulative compression savings.
Both numbers were recorded, in different places, and never compared.
The dashboard has long rendered "Net negative" for case 2, which is no help to
anyone running the proxy headless. Both cases now log at WARNING.
"""
from __future__ import annotations
import logging
import pytest
from headroom.proxy import savings_tracker as st
from headroom.proxy.savings_tracker import SavingsTracker
@pytest.fixture
def tracker(tmp_path) -> SavingsTracker:
return SavingsTracker(path=str(tmp_path / "savings.json"))
def _priced(compression: float, tool_schema: float = 0.0) -> dict[str, object]:
return {
"compression": compression,
"tool_schema": tool_schema,
"compression_list": compression,
"tool_schema_list": tool_schema,
"basis": "cache_aware",
}
# --------------------------------------------------------------------------
# per-request: the number itself
# --------------------------------------------------------------------------
def test_a_losing_request_is_recorded_as_a_loss(tracker: SavingsTracker) -> None:
"""The headline must be able to go below zero."""
tracker.record_request(
model="claude-sonnet-4",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.25),
)
assert tracker._negative_savings_usd == pytest.approx(-0.25)
assert tracker._negative_savings_count == 1
def test_a_winning_request_is_untouched(tracker: SavingsTracker) -> None:
tracker.record_request(
model="claude-sonnet-4",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(0.25),
)
assert tracker._negative_savings_count == 0
def test_buckets_net_against_each_other(tracker: SavingsTracker) -> None:
"""A loss in one bucket is not warned about if the request still wins."""
tracker.record_request(
model="claude-sonnet-4",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.10, tool_schema=0.40),
)
assert tracker._negative_savings_count == 0
# --------------------------------------------------------------------------
# per-request: the warning
# --------------------------------------------------------------------------
def test_first_loss_warns_immediately(
tracker: SavingsTracker, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(logging.WARNING, logger=st.logger.name):
tracker.record_request(
model="claude-sonnet-4",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.25),
)
assert "event=savings_negative" in caplog.text
assert "model=claude-sonnet-4" in caplog.text
def test_repeat_losses_are_throttled_but_still_counted(
tracker: SavingsTracker,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""One line per interval, carrying the total -- not one line per request."""
clock = {"t": 1000.0}
monkeypatch.setattr(st, "_monotonic", lambda: clock["t"])
with caplog.at_level(logging.WARNING, logger=st.logger.name):
for _ in range(5):
tracker.record_request(
model="claude-sonnet-4",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.20),
)
warnings = [r for r in caplog.records if "event=savings_negative" in r.getMessage()]
assert len(warnings) == 1
# The suppressed four are still in the running total.
assert tracker._negative_savings_count == 5
assert tracker._negative_savings_usd == pytest.approx(-1.0)
def test_warning_resumes_after_the_interval(
tracker: SavingsTracker,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
clock = {"t": 1000.0}
monkeypatch.setattr(st, "_monotonic", lambda: clock["t"])
with caplog.at_level(logging.WARNING, logger=st.logger.name):
tracker.record_request(
model="m",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.20),
)
clock["t"] += st.NEGATIVE_SAVINGS_WARN_INTERVAL_S + 1
tracker.record_request(
model="m",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd=_priced(-0.20),
)
warnings = [r for r in caplog.records if "event=savings_negative" in r.getMessage()]
assert len(warnings) == 2
# The second line reports the cumulative figure, not just its own request.
assert "negative_requests=2" in warnings[1].getMessage()
def test_the_other_buckets_keep_their_floor(tracker: SavingsTracker) -> None:
"""Output shaping and provider cache cannot go negative by construction."""
tracker.record_request(
model="m",
tokens_saved=100,
input_tokens=1000,
estimated_savings_usd={
**_priced(0.10),
"output_shaping": -5.0,
"provider_cache": -5.0,
},
)
assert tracker._negative_savings_count == 0
# ---------------------------------------------------------------------------
# Removing the floor at the point of measurement is not enough on its own. The
# loss then has to survive every place it is added up, stored and read back,
# and it did not: three separate clamps downstream re-floored it, so the
# warning fired while every number an operator would actually look at still
# read zero.
# ---------------------------------------------------------------------------
def _losing_turn(tracker: SavingsTracker, usd: float = -0.50, project: str = "acme") -> None:
tracker.record_request(
model="claude-opus-4-6",
provider="anthropic",
input_tokens=20_000,
tokens_saved=-5_000,
project=project,
estimated_savings_usd={
"compression": usd,
"tool_schema": 0.0,
"compression_list": usd,
"tool_schema_list": 0.0,
"basis": "mix",
},
)
def test_the_loss_reaches_the_per_model_rollup(tmp_path) -> None:
"""``by_model`` is where an operator finds WHICH model is losing money.
It floored the delta at zero, so the one breakdown that could answer that
question reported a clean zero for a turn that lost 50 cents.
"""
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
_losing_turn(tracker)
by_model = tracker._state["by_model"]
assert [e["compression_savings_usd"] for e in by_model.values()] == [-0.5]
def test_the_loss_reaches_the_per_project_rollup(tmp_path) -> None:
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
_losing_turn(tracker)
projects = tracker._state["projects"]
assert [e["compression_savings_usd"] for e in projects.values()] == [-0.5]
def test_losses_accumulate_instead_of_resetting_each_turn(tmp_path) -> None:
"""The list-priced column floored its own running total before adding to it.
Net effect: it only ever showed the MOST RECENT losing turn, so a
deployment three turns into a sustained loss looked like it had lost once.
"""
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
for _ in range(3):
_losing_turn(tracker)
lifetime = tracker._state["lifetime"]
assert lifetime["compression_savings_usd"] == pytest.approx(-1.5)
assert lifetime["compression_savings_list_usd"] == pytest.approx(-1.5)
def test_a_loss_survives_a_restart(tmp_path) -> None:
"""The load path floored too, so a restart erased whatever was preserved."""
path = str(tmp_path / "s.json")
tracker = SavingsTracker(path=path)
_losing_turn(tracker)
reloaded = SavingsTracker(path=path)
by_model = reloaded._state["by_model"]
assert [e["compression_savings_usd"] for e in by_model.values()] == [-0.5]
def test_the_public_snapshot_shows_the_loss(tmp_path) -> None:
"""What /stats serves must agree with what the warning said."""
tracker = SavingsTracker(path=str(tmp_path / "s.json"))
_losing_turn(tracker)
lifetime = tracker.snapshot()["lifetime"]
assert lifetime["compression_savings_usd"] == pytest.approx(-0.5)
assert lifetime["compression_savings_list_usd"] == pytest.approx(-0.5)
+117
View File
@@ -0,0 +1,117 @@
"""Cumulative cache-bust losses overtaking savings must reach the operator.
``tokens_saved_total`` and ``cache_bust_tokens_lost`` were both recorded and
never compared, so a deployment where compression costs more cache than it
saves tokens looked, from the logs, exactly like one where it did not. The
dashboard has rendered the net for a while, which only helps an operator who
is looking at it.
The warning is edge-triggered: a losing deployment loses on every bust, so a
line per bust would be the same noise problem in a different place.
"""
from __future__ import annotations
import asyncio
import logging
import pytest
from headroom.proxy.cost import CostTracker, build_prefix_cache_stats
from headroom.proxy.prometheus_metrics import PrometheusMetrics
def _metrics_with_savings(tokens_saved: int) -> PrometheusMetrics:
metrics = PrometheusMetrics()
if tokens_saved:
asyncio.run(
metrics.record_request(
provider="anthropic",
model="claude-opus-4-6",
input_tokens=1000,
output_tokens=20,
tokens_saved=tokens_saved,
latency_ms=10.0,
)
)
return metrics
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
return [r.getMessage() for r in caplog.records if "event=net_tokens_negative" in r.getMessage()]
def test_warns_when_busts_overtake_savings(caplog: pytest.LogCaptureFixture) -> None:
metrics = _metrics_with_savings(100)
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
asyncio.run(metrics.record_cache_bust(500))
warnings = _warnings(caplog)
assert len(warnings) == 1
assert "tokens_saved=100" in warnings[0]
assert "tokens_lost_to_cache_bust=500" in warnings[0]
assert "net_tokens=-400" in warnings[0]
def test_silent_while_still_net_positive(caplog: pytest.LogCaptureFixture) -> None:
metrics = _metrics_with_savings(1000)
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
asyncio.run(metrics.record_cache_bust(10))
asyncio.run(metrics.record_cache_bust(10))
assert _warnings(caplog) == []
def test_warns_once_per_crossing_not_once_per_bust(caplog: pytest.LogCaptureFixture) -> None:
metrics = _metrics_with_savings(100)
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
for _ in range(5):
asyncio.run(metrics.record_cache_bust(200))
assert len(_warnings(caplog)) == 1
def test_rearms_after_recovering(caplog: pytest.LogCaptureFixture) -> None:
"""Crossing back into profit and out again is a second event, not silence."""
metrics = _metrics_with_savings(100)
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
asyncio.run(metrics.record_cache_bust(200)) # net -100, warns
asyncio.run(
metrics.record_request(
provider="anthropic",
model="claude-opus-4-6",
input_tokens=1000,
output_tokens=20,
tokens_saved=1000,
latency_ms=10.0,
)
)
asyncio.run(metrics.record_cache_bust(1)) # net +899, re-arms
asyncio.run(metrics.record_cache_bust(5000)) # net negative again, warns
assert len(_warnings(caplog)) == 2
def test_stats_payload_carries_the_flag() -> None:
metrics = _metrics_with_savings(100)
asyncio.run(metrics.record_cache_bust(500))
stats = build_prefix_cache_stats(metrics, CostTracker())
cvc = stats["compression_vs_cache"]
assert cvc["net_tokens"] == -400
assert cvc["net_is_negative"] is True
def test_stats_flag_is_false_when_winning() -> None:
metrics = _metrics_with_savings(1000)
asyncio.run(metrics.record_cache_bust(10))
cvc = build_prefix_cache_stats(metrics, CostTracker())["compression_vs_cache"]
assert cvc["net_tokens"] == 990
assert cvc["net_is_negative"] is False