mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(savings): read routing totals, scope and moves from the routing_stats contract
Devin review on #4304: - the headline summed served_saving_usd, a routemegood field outside the contract; it now sums the pairs' signed savings_usd, so any provider's savings show - the section always said lifetime; it now names the provider's window, and says all decisions on record when none is given, as the dashboard does - moved counted every decision of a changed pair, holdouts included; it now counts enforced, the requests whose model was rewritten Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9480b003bf
commit
8128b56e34
+17
-9
@@ -108,23 +108,31 @@ def _proxy_routing() -> dict[str, Any] | None:
|
||||
return routing
|
||||
|
||||
|
||||
#: How the routing section names the provider's ``window`` (routing_stats contract).
|
||||
_ROUTING_WINDOWS = {"lifetime": "lifetime", "session": "this proxy session"}
|
||||
|
||||
|
||||
def _echo_routing(routing: dict[str, Any]) -> None:
|
||||
moved = [p for p in routing.get("pairs") or [] if p.get("direction") != "same"]
|
||||
# Per the routing_stats contract: a pair's `enforced` requests are the ones whose
|
||||
# model was rewritten (holdouts and shadow decisions are not), and the saving is
|
||||
# the sum of the pairs' signed `savings_usd`.
|
||||
pairs = [p for p in routing.get("pairs") or [] if p.get("direction") != "same"]
|
||||
moved = sum(int(p.get("enforced") or 0) for p in pairs)
|
||||
saved = sum(float(p.get("savings_usd") or 0.0) for p in pairs)
|
||||
window = _ROUTING_WINDOWS.get(str(routing.get("window")), "all decisions on record")
|
||||
click.echo("")
|
||||
click.echo(
|
||||
f"Model routing (lifetime): {sum(int(p.get('count') or 0) for p in moved):,} "
|
||||
f"of {int(routing.get('decisions') or 0):,} requests moved · "
|
||||
f"{_money(float(routing.get('served_saving_usd') or 0.0))} saved"
|
||||
f"Model routing ({window}): {moved:,} of {int(routing.get('decisions') or 0):,} "
|
||||
f"requests moved · {_money(saved)} saved"
|
||||
)
|
||||
for pair in moved:
|
||||
for pair in pairs:
|
||||
pair_name = f"{pair.get('requested')} → {pair.get('served')}"
|
||||
priced = int(pair.get("measured") or 0)
|
||||
saved = (
|
||||
priced = (
|
||||
_money(float(pair.get("savings_usd") or 0.0))
|
||||
if priced
|
||||
if int(pair.get("measured") or 0)
|
||||
else "not priced (no usage reported)"
|
||||
)
|
||||
click.echo(f" {pair_name:<44} {int(pair.get('count') or 0):,} requests {saved}")
|
||||
click.echo(f" {pair_name:<44} {int(pair.get('enforced') or 0):,} moved {priced}")
|
||||
click.echo(
|
||||
" Saved = each moved request's usage priced on the model asked for, minus its cost "
|
||||
"on the model served."
|
||||
|
||||
@@ -668,15 +668,17 @@ def test_many_unwritable_ledgers_do_not_warn_on_every_cycle(
|
||||
assert "More than 64 warnings about unwritable savings ledgers" in warnings[-1]
|
||||
|
||||
|
||||
_ROUTING = {
|
||||
_ROUTING = { # what routemegood publishes, trimmed to the routing_stats contract
|
||||
"decisions": 11,
|
||||
"served_saving_usd": 0.057101,
|
||||
"window": "lifetime",
|
||||
"pairs": [
|
||||
{
|
||||
"requested": "claude-opus-5-5",
|
||||
"served": "claude-opus-5-5",
|
||||
"direction": "same",
|
||||
"count": 5,
|
||||
"enforced": 0,
|
||||
"holdout": 0,
|
||||
"measured": 0,
|
||||
"savings_usd": 0.0,
|
||||
},
|
||||
@@ -684,7 +686,9 @@ _ROUTING = {
|
||||
"requested": "claude-sonnet-5-5",
|
||||
"served": "claude-haiku-5-5",
|
||||
"direction": "downgrade",
|
||||
"count": 4,
|
||||
"count": 5,
|
||||
"enforced": 4,
|
||||
"holdout": 1,
|
||||
"measured": 4,
|
||||
"savings_usd": 0.057101,
|
||||
},
|
||||
@@ -693,6 +697,8 @@ _ROUTING = {
|
||||
"served": "gpt-5.6-terra",
|
||||
"direction": "downgrade",
|
||||
"count": 2,
|
||||
"enforced": 2,
|
||||
"holdout": 0,
|
||||
"measured": 0,
|
||||
"savings_usd": 0.0,
|
||||
},
|
||||
@@ -710,13 +716,14 @@ def test_cli_shows_the_proxys_routing_savings_beside_compression(monkeypatch, tm
|
||||
monkeypatch.setattr(cli, "_proxy_routing", lambda: _ROUTING)
|
||||
|
||||
out = CliRunner().invoke(cli.savings, []).output
|
||||
# The headline is the sum of the pairs; a holdout stayed on Sonnet and is not moved.
|
||||
assert "Model routing (lifetime): 6 of 11 requests moved · $0.0571 saved" in out
|
||||
assert "claude-sonnet-5-5 → claude-haiku-5-5" in out
|
||||
assert "not priced (no usage reported)" in out # Sol -> Terra: a saving we cannot claim
|
||||
assert "claude-opus-5-5 → claude-opus-5-5" not in out
|
||||
|
||||
data = json.loads(CliRunner().invoke(cli.savings, ["--json"]).output)
|
||||
assert data["routing"]["served_saving_usd"] == 0.057101
|
||||
assert data["routing"]["pairs"][1]["savings_usd"] == 0.057101
|
||||
|
||||
|
||||
def test_cli_shows_routing_savings_before_any_compression(monkeypatch, tmp_path):
|
||||
@@ -740,3 +747,15 @@ def test_proxy_routing_is_none_without_a_router(monkeypatch):
|
||||
assert cli._proxy_routing() is None
|
||||
monkeypatch.setattr("headroom.install.health.probe_json", lambda url, timeout=2.0: None)
|
||||
assert cli._proxy_routing() is None
|
||||
|
||||
|
||||
def test_cli_names_the_routing_window_the_provider_reports(monkeypatch, tmp_path):
|
||||
from click.testing import CliRunner
|
||||
|
||||
from headroom.cli import savings as cli
|
||||
|
||||
_events_env(monkeypatch, tmp_path)
|
||||
for window, label in (("session", "this proxy session"), (None, "all decisions on record")):
|
||||
monkeypatch.setattr(cli, "_proxy_routing", lambda w=window: {**_ROUTING, "window": w})
|
||||
out = CliRunner().invoke(cli.savings, []).output
|
||||
assert f"Model routing ({label}): 6 of 11 requests moved" in out
|
||||
|
||||
Reference in New Issue
Block a user