Files
headroom/tests/test_savings_ledger.py
aaab338b64 feat(savings): show the proxy's model-routing savings in headroom savings (#4304)
## Problem
`headroom savings` reads only the compression ledger, so the money a
model router saves by serving a cheaper model never shows. In a workshop
sandbox test, routing saved $0.057 against compression's $0.015, and the
command showed only the $0.015.

## Spec
- **User story:** as an operator running Headroom with a model router, I
run `headroom savings` and see what routing saved beside what
compression saved, without opening the dashboard.
- **Source:** the routing-stats provider contract
(`headroom.proxy.routing_stats`), as served by the running proxy's
`/stats` → `routing`.
- **Moved:** `enforced` per pair, the requests whose model was
rewritten. Holdouts and shadow-mode decisions don't count.
  - **Saved:** the sum of the pairs' signed `savings_usd`.
- **Scope:** the provider's `window`, or "all decisions on record" when
it gives none.
- **Failure and recovery:** with no proxy, no router, or an unreachable
`/stats` (2 s timeout), the section is omitted and the ledger output is
unchanged.
- A pair with no measured usage is shown as "not priced" rather than
counted. Example: Codex over WebSocket, where the router never sees the
response usage.
- **Security:** a GET of `http://127.0.0.1:<port>/stats` only, the same
probe `headroom doctor` makes. No credentials and no new endpoint.
- **Follow-up, not in this PR:** routing history while the proxy is
stopped. It lives in the plugin's own store, which core has no seam to
read.

## Change
```
Model routing (lifetime): 5 of 20 requests moved · $0.0256 saved
  claude-sonnet-5-5 → claude-haiku-5-5         4 moved  $0.0256
  gpt-5.6-sol → gpt-5.6-terra                  1 moved  not priced (no usage reported)
  Saved = each moved request's usage priced on the model asked for, minus its cost on the model served.
```
- **Port:** found as `headroom doctor` finds it: `HEADROOM_PORT`, then a
managed deployment's port, then the default.
- **JSON:** `--json` adds `"routing"`, the provider snapshot as served.

## Real behavior proof
- **Environment:**
  - a Daytona workshop sandbox, Debian 12, Python 3.12;
- headroom-ai 0.41.0+main.fbf7bc32 with this `savings.py`, plus
headroom-enterprise 0.4.3 and routemegood 0.5.3 as the routing provider;
  - real Claude Code 2.1 and Codex CLI 0.163 sessions through the proxy.
- **Commands:**
  1. `headroom login`, `headroom route apply`, `headroom start`;
2. a Claude Code session on Opus 5.5, one on Sonnet 5.5 (routed to Haiku
5.5), and a Codex session on gpt-5.6-sol (routed to Terra);
  3. then `headroom savings`. The output above is that run.
- **Cross-checked:**
- the section's totals and pairs against `curl localhost:8787/stats | jq
.routing` on the same proxy;
- a pair of 7 holdout requests, Haiku to Haiku from the Explore
subagent, is correctly left out of the moved count.
- **Not tested live:**
- a provider reporting `window: session`, or none (unit tests cover
both);
  - a managed deployment on a non-default port;
- upgrade pairs with negative savings (handled by the signed sum, no
live data).

## Tests
- 4 tests in `tests/test_savings_ledger.py`:
  - routing beside compression, including a holdout pair;
  - routing with an empty ledger;
  - each `window` label;
  - no router or no proxy, giving no section.
- The routing tests fail on the first version of this PR, before
8128b56e.
- `test_proxy_record_request_appends_ledger_event` fails the same way on
main in my environment (`CancelledError`); it's unrelated.
- ruff, ruff format and mypy are clean on the changed file.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
2026-11-03 23:43:01 -06:00

774 lines
29 KiB
Python

"""Tests for the durable savings event ledger and the `headroom savings` CLI."""
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta, timezone
import pytest
from headroom import savings_ledger as L
from headroom.log_safety import WarnOnce
from tests._pricing_models import anthropic_pricing_model
MODEL = anthropic_pricing_model()
UTC = timezone.utc
def _events_env(monkeypatch, tmp_path):
path = tmp_path / "savings_events.jsonl"
monkeypatch.setenv("HEADROOM_SAVINGS_EVENTS_PATH", str(path))
return path
# --------------------------------------------------------------------------- #
# core ledger
# --------------------------------------------------------------------------- #
def test_unknown_model_uses_blended_fallback(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
assert L.record_savings_event(tokens_before=1000, tokens_after=400, model=None, client="c")
report = L.aggregate_savings()
assert report.lifetime["tokens_saved"] == 600
expected = round(600 * L.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN, 6)
assert report.lifetime["cost_usd"] == pytest.approx(expected)
assert any(row["model"] == "unknown" for row in report.by_model)
def test_estimate_cost_unknown_short_circuits_to_fallback():
assert L.estimate_cost_usd("unknown", 1000, fallback_rate=1e-6) == pytest.approx(0.001)
assert L.estimate_cost_usd(L.UNKNOWN, 0) == 0.0
def test_free_model_is_not_billed_at_fallback(monkeypatch):
"""A known but 0-priced (free) model must cost $0, not the $3/M fallback.
_estimate_compression_savings_usd returns a legitimate 0.0 for free models;
the ledger must trust that rather than re-billing it at the blended rate.
"""
monkeypatch.setattr(L, "_estimate_compression_savings_usd", lambda model, tokens: 0.0)
assert L.estimate_cost_usd("free-local-model", 1_000_000) == 0.0
def test_priced_model_uses_litellm_estimate(monkeypatch):
monkeypatch.setattr(L, "_estimate_compression_savings_usd", lambda model, tokens: tokens * 2e-6)
assert L.estimate_cost_usd("some-model", 1_000_000) == pytest.approx(2.0)
def test_explicit_cost_is_honored(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=100, tokens_after=10, model="x", client="c", cost_usd=1.25)
assert L.aggregate_savings().lifetime["cost_usd"] == pytest.approx(1.25)
def test_zero_or_negative_savings_not_recorded(monkeypatch, tmp_path):
path = _events_env(monkeypatch, tmp_path)
assert L.record_savings_event(tokens_before=100, tokens_after=100) is False
assert L.record_savings_event(tokens_before=50, tokens_after=80) is False
assert not path.exists() or path.read_text().strip() == ""
assert L.aggregate_savings().lifetime["calls"] == 0
def test_stateless_mode_does_not_create_ledger(monkeypatch, tmp_path):
path = _events_env(monkeypatch, tmp_path)
monkeypatch.setenv("HEADROOM_STATELESS", "true")
assert L.record_savings_event(tokens_before=100, tokens_after=50) is False
assert not path.exists()
def test_breakdowns_aggregate_by_dimension(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=300, model=None, client="claude-code")
L.record_savings_event(tokens_before=500, tokens_after=200, model=None, client="claude-code")
L.record_savings_event(
tokens_before=2000, tokens_after=600, model="gpt", client="proxy", cost_usd=0.5
)
report = L.aggregate_savings()
clients = {row["client"]: row for row in report.by_client}
assert clients["claude-code"]["calls"] == 2
assert clients["proxy"]["tokens_saved"] == 1400
def test_windows_today_week_last30(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
now = datetime(2026, 6, 17, 12, 0, tzinfo=UTC)
L.record_savings_event(
tokens_before=1000, tokens_after=500, model=None, client="c", timestamp=now
)
L.record_savings_event(
tokens_before=1000,
tokens_after=600,
model=None,
client="c",
timestamp=now - timedelta(days=3),
)
L.record_savings_event(
tokens_before=1000,
tokens_after=700,
model=None,
client="c",
timestamp=now - timedelta(days=20),
)
report = L.aggregate_savings(now=now)
assert report.windows["today"]["tokens_saved"] == 500
assert report.windows["last_7_days"]["tokens_saved"] == 500 + 400
assert report.windows["last_30_days"]["tokens_saved"] == 500 + 400 + 300
assert report.windows["last_30_days"]["calls"] == 3
# 500 saved out of 1000 before today
assert report.windows["today"]["savings_percent"] == pytest.approx(50.0)
def test_new_input_basis_pairs_compression_only_with_new_input(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
# 400 saved, of which 100 is tool-schema deferral; 900 tokens newly entered
# context. Compression-only 300 / (900 + 300) = 25%, same as /stats.
L.record_savings_event(
tokens_before=10_000,
tokens_after=9_600,
model=None,
client="claude-code",
new_input_tokens=900,
deferred_tokens=100,
)
# An event without a cache breakdown (MCP tool, Bedrock) keeps writing the
# old line and must not lend its savings to the new-input ratio.
L.record_savings_event(tokens_before=1000, tokens_after=500, model=None, client="mcp")
report = L.aggregate_savings()
window = report.windows["today"]
assert window["tokens_saved"] == 900
assert window["new_input_tokens"] == 900
assert window["new_input_savings_percent"] == pytest.approx(25.0)
clients = {row["client"]: row for row in report.by_client}
assert clients["mcp"]["new_input_tokens"] == 0
assert clients["mcp"]["new_input_savings_percent"] == 0.0
# The whole-wire ratio is untouched by the new field.
assert window["savings_percent"] == pytest.approx(900 / 11_000 * 100, abs=0.1)
def test_savings_cli_prints_new_input_line_only_with_cache_data(monkeypatch, tmp_path):
from click.testing import CliRunner
from headroom.cli.savings import savings
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=500, model=None, client="mcp")
out = CliRunner().invoke(savings, []).output
assert "of new input" not in out
L.record_savings_event(
tokens_before=10_000, tokens_after=9_600, model=None, client="c", new_input_tokens=900
)
out = CliRunner().invoke(savings, []).output
assert "of new input 30.8%" in out
assert "newly entered context" in out
def test_retention_hard_capped_at_30_days(monkeypatch, tmp_path):
_events_env(monkeypatch, tmp_path)
now = datetime(2026, 6, 17, 12, 0, tzinfo=UTC)
L.record_savings_event(
tokens_before=1000, tokens_after=500, model=None, client="c", timestamp=now
)
# 60 days old: within the requested 365-day window but past the 30-day cap.
L.record_savings_event(
tokens_before=1000,
tokens_after=500,
model=None,
client="c",
timestamp=now - timedelta(days=60),
)
# Caller asks for 365 days, but retention is hard-capped at 30.
report = L.aggregate_savings(now=now, retention_days=365)
assert report.lifetime["calls"] == 1
assert report.windows["last_30_days"]["calls"] == 1
def test_appends_do_not_clobber_and_survive_restart(monkeypatch, tmp_path):
path = _events_env(monkeypatch, tmp_path)
for _ in range(5):
L.record_savings_event(tokens_before=100, tokens_after=10, model=None, client="c")
lines = [ln for ln in path.read_text().splitlines() if ln.strip()]
assert len(lines) == 5
# aggregate_savings holds no in-memory state — it reads purely from disk,
# so this also proves durability across a process restart.
assert L.aggregate_savings().lifetime["calls"] == 5
assert L.aggregate_savings().lifetime["tokens_saved"] == 5 * 90
def test_corrupt_lines_are_skipped(monkeypatch, tmp_path):
path = _events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=400, model=None, client="c")
with open(path, "a", encoding="utf-8") as handle:
handle.write("not json\n\n")
assert L.aggregate_savings().lifetime["calls"] == 1
# --------------------------------------------------------------------------- #
# CLI
# --------------------------------------------------------------------------- #
def test_cli_reset_deletes_ledger(monkeypatch, tmp_path):
pytest.importorskip("click")
from click.testing import CliRunner
from headroom.cli.savings import savings
path = _events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=300, model=None, client="claude-code")
assert path.exists()
result = CliRunner().invoke(savings, ["--reset"])
assert result.exit_code == 0
assert "reset" in result.output.lower()
assert not path.exists()
# second reset on missing file is a no-op
result2 = CliRunner().invoke(savings, ["--reset"])
assert result2.exit_code == 0
assert "Nothing to reset" in result2.output
def test_cli_empty_state(monkeypatch, tmp_path):
pytest.importorskip("click")
from click.testing import CliRunner
from headroom.cli.savings import savings
_events_env(monkeypatch, tmp_path)
result = CliRunner().invoke(savings, [])
assert result.exit_code == 0
assert "No savings recorded yet." in result.output
def test_cli_renders_sections_and_json(monkeypatch, tmp_path):
pytest.importorskip("click")
from click.testing import CliRunner
from headroom.cli.savings import savings
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=300, model=None, client="claude-code")
runner = CliRunner()
result = runner.invoke(savings, [])
assert result.exit_code == 0
# No redundant top-line headline; the windows lead the output.
assert "cost avoided" not in result.output
assert "Today" in result.output and "Last 30 days" in result.output
assert "Savings by client" in result.output and "claude-code" in result.output
assert "Per-repo totals" not in result.output
result_json = runner.invoke(savings, ["--json"])
assert result_json.exit_code == 0
payload = json.loads(result_json.output)
assert payload["lifetime"]["tokens_saved"] == 700
assert payload["windows"]["last_30_days"]["calls"] == 1
assert "by_repo" not in payload
# --------------------------------------------------------------------------- #
# MCP tool path
# --------------------------------------------------------------------------- #
def test_mcp_compress_records_durable_event(monkeypatch, tmp_path):
pytest.importorskip("mcp", reason="MCP SDK required")
from headroom.ccr import mcp_server
_events_env(monkeypatch, tmp_path)
monkeypatch.setenv("HEADROOM_MCP_CLIENT", "claude-code")
server = mcp_server.HeadroomMCPServer(check_proxy=False)
server._record_savings({"original_tokens": 1000, "compressed_tokens": 250})
report = L.aggregate_savings()
assert report.lifetime["tokens_saved"] == 750
assert {row["client"] for row in report.by_client} == {"claude-code"}
def test_mcp_record_savings_ignores_noop(monkeypatch, tmp_path):
pytest.importorskip("mcp", reason="MCP SDK required")
from headroom.ccr import mcp_server
_events_env(monkeypatch, tmp_path)
server = mcp_server.HeadroomMCPServer(check_proxy=False)
server._record_savings({"original_tokens": 500, "compressed_tokens": 500})
assert L.aggregate_savings().lifetime["calls"] == 0
# --------------------------------------------------------------------------- #
# proxy path
# --------------------------------------------------------------------------- #
def test_proxy_record_request_appends_ledger_event(tmp_path, monkeypatch):
pytest.importorskip("fastapi")
import asyncio
from fastapi.testclient import TestClient
from headroom.proxy.server import ProxyConfig, create_app
monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(tmp_path / "proxy_savings.json"))
monkeypatch.setenv("HEADROOM_SAVINGS_EVENTS_PATH", str(tmp_path / "savings_events.jsonl"))
monkeypatch.setattr(
"headroom.proxy.server.CostTracker._get_cache_prices",
# (cache_read, cache_write_5m, cache_write_1h, uncached). **kwargs so
# the stub keeps standing in as the real signature grows -- it takes a
# keyword-only `long_context` tier selector.
lambda self, model, **kwargs: (0.001, 0.0015, 0.0024, 0.002),
)
config = ProxyConfig(cache_enabled=False, rate_limit_enabled=False, log_requests=False)
with TestClient(create_app(config)) as client:
proxy = client.app.state.proxy
# identified harness -> recorded as that client
asyncio.run(
proxy.metrics.record_request(
provider="openai",
model="gpt-4o",
input_tokens=120,
output_tokens=24,
tokens_saved=40,
latency_ms=15.0,
client="claude-code",
)
)
# unidentified harness -> falls back to "proxy"
asyncio.run(
proxy.metrics.record_request(
provider="openai",
model="gpt-4o",
input_tokens=80,
output_tokens=10,
tokens_saved=20,
latency_ms=15.0,
)
)
report = L.aggregate_savings()
assert report.lifetime["tokens_saved"] == 60
clients = {row["client"] for row in report.by_client}
assert "claude-code" in clients
assert "proxy" in clients
assert any(row["model"] == "gpt-4o" for row in report.by_model)
# --------------------------------------------------------------------------- #
# retention cap + stale-schema reset
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("bad_days", ["31", "60", "365"])
def test_cli_days_flag_capped_at_30(monkeypatch, tmp_path, bad_days):
pytest.importorskip("click")
from click.testing import CliRunner
from headroom.cli.savings import savings
_events_env(monkeypatch, tmp_path)
result = CliRunner().invoke(savings, ["--days", bad_days])
assert result.exit_code != 0
assert "30" in result.output # IntRange error mentions the allowed max
def test_compaction_backs_off_when_nothing_can_be_dropped(tmp_path, monkeypatch):
"""A busy install's whole retention window can exceed the size threshold.
Compaction only removes events older than retention, so such a ledger has
NOTHING to drop — and without a back-off every subsequent append re-read and
rewrote the entire growing file under an exclusive lock, once per compressed
request. The back-off is per-process and purely a work-saver: retention is
enforced on read regardless.
"""
import headroom.savings_ledger as sl
path = tmp_path / "events.jsonl"
# Tiny thresholds so a handful of in-retention events trips compaction.
monkeypatch.setattr(sl, "_COMPACT_SIZE_BYTES", 200)
monkeypatch.setattr(sl, "_COMPACT_GROWTH_BYTES", 10_000)
monkeypatch.setattr(sl, "_compact_min_size", 200)
for _ in range(5):
sl.record_savings_event(
tokens_before=1_000, tokens_after=600, model="unknown", source="mcp", path=path
)
assert path.stat().st_size > 200
# Every event is fresh, so none aged out and the floor ratcheted above the
# current size rather than staying at the threshold.
assert sl._compact_min_size > 200
# All five events survive: skipping the rewrite must never lose data.
assert sl.aggregate_savings(path=path).lifetime["calls"] == 5
def test_compaction_still_drops_out_of_retention_events(tmp_path, monkeypatch):
"""The back-off must not disable compaction for ledgers that CAN shrink."""
import json
from datetime import timedelta
import headroom.savings_ledger as sl
path = tmp_path / "events.jsonl"
stale = (sl._utc_now() - timedelta(days=sl.MAX_RETENTION_DAYS + 5)).isoformat()
with path.open("w", encoding="utf-8") as fh:
for _ in range(50):
fh.write(
json.dumps(
{
"v": 1,
"ts": stale,
"before": 1_000,
"after": 600,
"saved": 400,
"cost_usd": 0.0012,
"model": "unknown",
"client": "mcp",
"source": "mcp",
},
separators=(",", ":"),
)
+ "\n"
)
before_size = path.stat().st_size
monkeypatch.setattr(sl, "_COMPACT_SIZE_BYTES", 200)
monkeypatch.setattr(sl, "_compact_min_size", 200)
sl.record_savings_event(
tokens_before=1_000, tokens_after=600, model="unknown", source="mcp", path=path
)
assert path.stat().st_size < before_size
# Only the fresh event remains; the floor reset so compaction stays armed.
assert sl.aggregate_savings(path=path).lifetime["calls"] == 1
assert sl._compact_min_size == 200
def test_v1_events_still_aggregate_alongside_v2(tmp_path):
"""A ledger written across the upgrade must stay legible, not silently blend.
A v1 event has no cache mix and never can have one, so it contributes its
list dollar to both columns — and drags the reported basis down to `list`,
which is how a reader learns the total is not fully cache-aware.
"""
import json
import headroom.savings_ledger as sl
path = tmp_path / "events.jsonl"
with path.open("w", encoding="utf-8") as fh:
fh.write(
json.dumps(
{
"v": 1,
"ts": sl._utc_now().isoformat(),
"before": 10_000,
"after": 6_000,
"saved": 4_000,
"cost_usd": 0.012,
"model": MODEL,
"client": "claude-code",
"source": "proxy",
}
)
+ "\n"
)
sl.record_savings_event(
tokens_before=108_000,
tokens_after=100_000,
model=MODEL,
client="claude-code",
source="proxy",
saved_compression=1_000,
saved_tool_schema=7_000,
cache_read_tokens=95_000,
cache_write_tokens=4_000,
cache_write_5m_tokens=4_000,
uncached_input_tokens=1_000,
provider="anthropic",
path=path,
)
lifetime = sl.aggregate_savings(path=path).lifetime
assert lifetime["calls"] == 2
# The v2 event is discounted, so the effective total sits below the list one.
assert lifetime["cost_effective_usd"] < lifetime["cost_usd"]
# ...but the v1 event's list dollar is still counted, not dropped.
assert lifetime["cost_effective_usd"] > 0.012
assert lifetime["basis"] == "list"
def test_zero_saving_request_keeps_its_place_in_the_new_input_denominator(monkeypatch, tmp_path):
"""100 saved on 100 new input, then 0 saved on 10,000 new input. The
new-input basis is 100 / (10,100 + 100), under 1 percent, not the 50
percent that only counting requests that saved would report. The
saved-event figures do not see the second request at all."""
_events_env(monkeypatch, tmp_path)
assert L.record_savings_event(
tokens_before=200, tokens_after=100, model=None, client="claude-code", new_input_tokens=100
)
assert L.record_savings_event(
tokens_before=10_000,
tokens_after=10_000,
model=None,
client="claude-code",
new_input_tokens=10_000,
)
# Without a new-input figure a zero-saving request is still not an event.
assert not L.record_savings_event(
tokens_before=500, tokens_after=500, model=None, client="claude-code"
)
window = L.aggregate_savings().windows["today"]
assert window["tokens_saved"] == 100
assert window["new_input_tokens"] == 10_100
assert window["new_input_savings_percent"] == pytest.approx(1.0, abs=0.05)
assert window["savings_percent"] == pytest.approx(50.0)
# --------------------------------------------------------------------------- #
# failures are logged, not swallowed
# --------------------------------------------------------------------------- #
class _ListHandler(logging.Handler):
def __init__(self) -> None:
super().__init__(level=logging.DEBUG)
self.records: list[logging.LogRecord] = []
def emit(self, record: logging.LogRecord) -> None:
self.records.append(record)
@pytest.fixture
def ledger_records():
# Attach directly: caplog misses `headroom.*` loggers once the proxy has
# configured logging earlier in the session.
logger = logging.getLogger("headroom.savings_ledger")
handler = _ListHandler()
previous = logger.level
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
yield handler.records
logger.removeHandler(handler)
logger.setLevel(previous)
@pytest.fixture
def fresh_append_warnings(monkeypatch):
monkeypatch.setattr(L, "_append_warnings", WarnOnce(64, "unwritable savings ledgers"))
def _warnings(records: list[logging.LogRecord]) -> list[str]:
return [r.getMessage() for r in records if r.levelno == logging.WARNING]
def test_unreadable_ledger_warns_instead_of_reporting_zero_silently(tmp_path, ledger_records):
readable = tmp_path / "savings_events.jsonl"
assert L.record_savings_event(
tokens_before=1000, tokens_after=400, model=None, client="c", path=readable
)
assert L.aggregate_savings(path=readable).lifetime["tokens_saved"] == 600
assert _warnings(ledger_records) == []
# A directory where the ledger file should be: unreadable.
report = L.aggregate_savings(path=tmp_path)
assert report.lifetime["tokens_saved"] == 0
warnings = _warnings(ledger_records)
assert len(warnings) == 1
assert "could not read" in warnings[0]
assert str(tmp_path) in warnings[0]
def test_failed_append_warns_once_per_ledger_path(tmp_path, ledger_records, fresh_append_warnings):
first = tmp_path / "a"
second = tmp_path / "b"
first.mkdir()
second.mkdir()
# A directory where the ledger file should be: every append fails.
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=second)
warnings = _warnings(ledger_records)
# One WARNING per path; the repeat on `first` goes to DEBUG only.
assert len(warnings) == 2
assert str(first) in warnings[0] and "append to" in warnings[0]
assert str(second) in warnings[1]
assert any(r.levelno == logging.DEBUG and "append to" in r.getMessage() for r in ledger_records)
def test_successful_append_rearms_the_warning(tmp_path, ledger_records, fresh_append_warnings):
ledger = tmp_path / "savings_events.jsonl"
ledger.mkdir()
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
ledger.rmdir()
assert L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
ledger.unlink()
ledger.mkdir()
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
# The failure after a recovery is a new episode and warns again.
assert len(_warnings(ledger_records)) == 2
def test_append_warning_names_only_the_type_of_a_non_os_error(
monkeypatch, tmp_path, ledger_records, fresh_append_warnings
):
def fail(*_args, **_kwargs):
raise TypeError("secret-looking event payload")
# Shadow open() inside the module only, so nothing else is affected.
monkeypatch.setattr(L, "open", fail, raising=False)
assert not L.record_savings_event(
tokens_before=1000, tokens_after=400, path=tmp_path / "savings_events.jsonl"
)
warnings = _warnings(ledger_records)
assert len(warnings) == 1
assert "TypeError" in warnings[0]
formatted = [logging.Formatter().format(record) for record in ledger_records]
assert all("secret-looking" not in line for line in formatted)
# The repeat goes to DEBUG, and DEBUG carries no payload either.
assert not L.record_savings_event(
tokens_before=1000, tokens_after=400, path=tmp_path / "savings_events.jsonl"
)
formatted = [logging.Formatter().format(record) for record in ledger_records]
assert any(r.levelno == logging.DEBUG and "TypeError" in r.getMessage() for r in ledger_records)
assert all("secret-looking" not in line for line in formatted)
def test_many_unwritable_ledgers_do_not_warn_on_every_cycle(
tmp_path, ledger_records, fresh_append_warnings
):
# A directory where each ledger file should be: every append fails.
ledgers = [tmp_path / f"ledger-{i}" for i in range(65)]
for ledger in ledgers:
ledger.mkdir()
for _cycle in range(3):
for ledger in ledgers:
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
warnings = _warnings(ledger_records)
# 64 per-path warnings plus one overflow notice, over all three cycles.
assert len(warnings) == 65
assert sum("append to" in w for w in warnings) == 64
assert "More than 64 warnings about unwritable savings ledgers" in warnings[-1]
_ROUTING = { # what routemegood publishes, trimmed to the routing_stats contract
"decisions": 11,
"window": "lifetime",
"pairs": [
{
"requested": "claude-opus-5-5",
"served": "claude-opus-5-5",
"direction": "same",
"count": 5,
"enforced": 0,
"holdout": 0,
"measured": 0,
"savings_usd": 0.0,
},
{
"requested": "claude-sonnet-5-5",
"served": "claude-haiku-5-5",
"direction": "downgrade",
"count": 5,
"enforced": 4,
"holdout": 1,
"measured": 4,
"savings_usd": 0.057101,
},
{
"requested": "gpt-5.6-sol",
"served": "gpt-5.6-terra",
"direction": "downgrade",
"count": 2,
"enforced": 2,
"holdout": 0,
"measured": 0,
"savings_usd": 0.0,
},
],
}
def test_cli_shows_the_proxys_routing_savings_beside_compression(monkeypatch, tmp_path):
from click.testing import CliRunner
from headroom.cli import savings as cli
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=300, model=None, client="claude-code")
monkeypatch.setattr(cli, "_proxy_routing", lambda: _ROUTING)
out = CliRunner().invoke(cli.savings, []).output
# The headline is the sum of the pairs; a holdout stayed on Sonnet and is not moved.
assert "Model routing (lifetime): 6 of 11 requests moved · $0.0571 saved" in out
assert "claude-sonnet-5-5 → claude-haiku-5-5" in out
assert "not priced (no usage reported)" in out # Sol -> Terra: a saving we cannot claim
assert "claude-opus-5-5 → claude-opus-5-5" not in out
data = json.loads(CliRunner().invoke(cli.savings, ["--json"]).output)
assert data["routing"]["pairs"][1]["savings_usd"] == 0.057101
def test_cli_shows_routing_savings_before_any_compression(monkeypatch, tmp_path):
from click.testing import CliRunner
from headroom.cli import savings as cli
_events_env(monkeypatch, tmp_path)
monkeypatch.setattr(cli, "_proxy_routing", lambda: _ROUTING)
out = CliRunner().invoke(cli.savings, []).output
assert "No savings recorded yet." not in out
assert "$0.0571 saved" in out
def test_proxy_routing_is_none_without_a_router(monkeypatch):
from headroom.cli import savings as cli
monkeypatch.setattr(
"headroom.install.health.probe_json", lambda url, timeout=2.0: {"routing": None}
)
assert cli._proxy_routing() is None
monkeypatch.setattr("headroom.install.health.probe_json", lambda url, timeout=2.0: None)
assert cli._proxy_routing() is None
def test_cli_names_the_routing_window_the_provider_reports(monkeypatch, tmp_path):
from click.testing import CliRunner
from headroom.cli import savings as cli
_events_env(monkeypatch, tmp_path)
for window, label in (("session", "this proxy session"), (None, "all decisions on record")):
monkeypatch.setattr(cli, "_proxy_routing", lambda w=window: {**_ROUTING, "window": w})
out = CliRunner().invoke(cli.savings, []).output
assert f"Model routing ({label}): 6 of 11 requests moved" in out
def test_cli_without_a_router_never_mentions_routing(monkeypatch, tmp_path):
from click.testing import CliRunner
from headroom.cli import savings as cli
_events_env(monkeypatch, tmp_path)
L.record_savings_event(tokens_before=1000, tokens_after=300, model=None, client="claude-code")
monkeypatch.setattr(cli, "_proxy_routing", lambda: None)
assert "routing" not in CliRunner().invoke(cli.savings, []).output.lower()
assert "routing" not in json.loads(CliRunner().invoke(cli.savings, ["--json"]).output)