mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
perf: add savings audit output (#1211)
## Description - add a savings_audit object to perf JSON output - expose raw prompt reduction, logged savings, accounting delta, suspicious records, and top saving requests - add audit columns to aggregate CSV output and a compact text report section ## Testing - [x] Python compilation check passes - [x] Current refreshed head has no failing GitHub checks ### Test Output ```text python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py tests/test_cli_perf_format.py Result: passed GitHub status rollup: no failures ``` - python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py tests/test_cli_perf_format.py - pytest tests/test_cli_perf_format.py could not run locally because pytest is not installed in this Python environment ## Type of Change - [ ] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Documentation update - [x] Performance improvement - [ ] Code refactoring ## Changes Made - Add a savings_audit object to PERF JSON output. - Expose prompt reduction, recorded savings, accounting deltas, suspicious records, and top-saving requests. - Add audit columns to aggregate CSV output and a compact text report. ### Test Output ```text python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py tests/test_cli_perf_format.py: passed Current refreshed head: no failing GitHub checks ``` ## Real Behavior Proof - Environment: current refreshed PR head and the PERF analyzer/CLI formatting test environment. - Exact command / steps: compiled the changed analyzer, CLI, and formatting-test modules and inspected the generated JSON, CSV, and text audit surfaces. - Observed result: audit data exposes raw reduction, recorded savings, deltas, suspicious rows, and top requests without changing existing PERF records. - Not tested: a production-scale PERF log corpus. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Co-authored-by: JerrettDavis <mxjerrett@gmail.com> Co-authored-by: JD Davis <jd@jds-macbook-air.tail2a279.ts.net>
This commit is contained in:
co-authored by
JerrettDavis
JD Davis
parent
31d0344991
commit
9a72bc0242
@@ -82,8 +82,11 @@ def perf(hours: float, raw: bool, output_format: str) -> None:
|
||||
"tokens_saved",
|
||||
"message_tokens_saved",
|
||||
"tool_tokens_saved",
|
||||
"prompt_reduction_tokens",
|
||||
"accounting_delta_tokens",
|
||||
"savings_pct",
|
||||
"list_price_per_mtok",
|
||||
"estimated_list_price_savings_usd",
|
||||
]
|
||||
writer = csv.DictWriter(buf, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
+136
-1
@@ -613,6 +613,28 @@ def format_report(report: PerfReport) -> str:
|
||||
lines.append(f" · tool schemas {total_tool_saved:,}")
|
||||
lines.append("")
|
||||
|
||||
audit = build_savings_audit(report)
|
||||
lines.append("Savings Audit")
|
||||
lines.append("-" * 40)
|
||||
lines.append(
|
||||
" Raw prompt reduction: "
|
||||
f"{audit['prompt_reduction_tokens']:,} tokens "
|
||||
f"({audit['prompt_reduction_pct']:.1f}%)"
|
||||
)
|
||||
lines.append(
|
||||
" Logged savings: "
|
||||
f"{audit['logged_tokens_saved']:,} tokens "
|
||||
f"({audit['logged_savings_pct']:.1f}%)"
|
||||
)
|
||||
lines.append(f" Accounting delta: {audit['accounting_delta_tokens']:+,} tokens")
|
||||
delta_count = audit["record_counts"]["with_accounting_delta"]
|
||||
if delta_count:
|
||||
lines.append(f" Records with delta: {delta_count}/{len(records)}")
|
||||
impossible_count = audit["record_counts"]["logged_saved_gt_tokens_before"]
|
||||
if impossible_count:
|
||||
lines.append(f" Impossible records: {impossible_count} saved more than before")
|
||||
lines.append("")
|
||||
|
||||
# Per-model breakdown with list prices
|
||||
by_model: dict[str, list[PerfRecord]] = {}
|
||||
for r in records:
|
||||
@@ -973,6 +995,106 @@ def _pct(saved: int, before: int) -> float:
|
||||
return round(saved / before * 100, 1) if before > 0 else 0.0
|
||||
|
||||
|
||||
def _prompt_reduction_tokens(record: PerfRecord) -> int:
|
||||
"""Raw prompt-token delta for a record, excluding cache/accounting effects."""
|
||||
return max(record.tokens_before - record.tokens_after, 0)
|
||||
|
||||
|
||||
def _audit_record(record: PerfRecord) -> dict:
|
||||
prompt_reduction = _prompt_reduction_tokens(record)
|
||||
return {
|
||||
"timestamp": record.timestamp,
|
||||
"request_id": record.request_id,
|
||||
"model": record.model,
|
||||
"client": record.client,
|
||||
"tokens_before": record.tokens_before,
|
||||
"tokens_after": record.tokens_after,
|
||||
"logged_tokens_saved": record.tokens_saved,
|
||||
"prompt_reduction_tokens": prompt_reduction,
|
||||
"accounting_delta_tokens": record.tokens_saved - prompt_reduction,
|
||||
"cache_read_tokens": record.cache_read,
|
||||
"cache_write_tokens": record.cache_write,
|
||||
"optimization_ms": record.optimization_ms,
|
||||
"transforms": record.transforms,
|
||||
}
|
||||
|
||||
|
||||
def _record_accounting_reasons(record: PerfRecord) -> list[str]:
|
||||
reasons: list[str] = []
|
||||
if min(record.tokens_before, record.tokens_after, record.tokens_saved) < 0:
|
||||
reasons.append("negative_token_count")
|
||||
if record.tokens_before > 0 and record.tokens_saved > record.tokens_before:
|
||||
reasons.append("logged_saved_gt_tokens_before")
|
||||
if record.tokens_after > record.tokens_before and record.tokens_saved > 0:
|
||||
reasons.append("prompt_grew_but_logged_savings_positive")
|
||||
if record.tokens_saved != _prompt_reduction_tokens(record):
|
||||
reasons.append("logged_saved_ne_prompt_delta")
|
||||
return reasons
|
||||
|
||||
|
||||
def build_savings_audit(report: PerfReport, *, limit: int = 10) -> dict:
|
||||
"""Explain how headline savings relate to raw before/after token deltas."""
|
||||
records = report.perf_records
|
||||
total_before = sum(r.tokens_before for r in records)
|
||||
total_after = sum(r.tokens_after for r in records)
|
||||
logged_saved = sum(r.tokens_saved for r in records)
|
||||
prompt_reduction = sum(_prompt_reduction_tokens(r) for r in records)
|
||||
prompt_growth = sum(max(r.tokens_after - r.tokens_before, 0) for r in records)
|
||||
accounting_delta = logged_saved - prompt_reduction
|
||||
|
||||
records_with_delta = [r for r in records if r.tokens_saved != _prompt_reduction_tokens(r)]
|
||||
impossible_saved_gt_before = [
|
||||
r for r in records if r.tokens_before > 0 and r.tokens_saved > r.tokens_before
|
||||
]
|
||||
prompt_growth_with_savings = [
|
||||
r for r in records if r.tokens_after > r.tokens_before and r.tokens_saved > 0
|
||||
]
|
||||
negative_token_counts = [
|
||||
r for r in records if min(r.tokens_before, r.tokens_after, r.tokens_saved) < 0
|
||||
]
|
||||
|
||||
suspicious_records = []
|
||||
for record in sorted(
|
||||
records_with_delta,
|
||||
key=lambda r: abs(r.tokens_saved - _prompt_reduction_tokens(r)),
|
||||
reverse=True,
|
||||
)[:limit]:
|
||||
suspicious_records.append(
|
||||
{
|
||||
**_audit_record(record),
|
||||
"reasons": _record_accounting_reasons(record),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"formula": {
|
||||
"raw_prompt_reduction": "sum(max(tokens_before - tokens_after, 0))",
|
||||
"logged_savings": "sum(tokens_saved)",
|
||||
"accounting_delta": "logged_savings - raw_prompt_reduction",
|
||||
},
|
||||
"total_tokens_before": total_before,
|
||||
"total_tokens_after": total_after,
|
||||
"logged_tokens_saved": logged_saved,
|
||||
"logged_savings_pct": _pct(logged_saved, total_before),
|
||||
"prompt_reduction_tokens": prompt_reduction,
|
||||
"prompt_reduction_pct": _pct(prompt_reduction, total_before),
|
||||
"prompt_growth_tokens": prompt_growth,
|
||||
"accounting_delta_tokens": accounting_delta,
|
||||
"record_counts": {
|
||||
"total": len(records),
|
||||
"with_accounting_delta": len(records_with_delta),
|
||||
"logged_saved_gt_tokens_before": len(impossible_saved_gt_before),
|
||||
"prompt_grew_but_logged_savings_positive": len(prompt_growth_with_savings),
|
||||
"negative_token_count": len(negative_token_counts),
|
||||
},
|
||||
"top_saving_requests": [
|
||||
_audit_record(r)
|
||||
for r in sorted(records, key=lambda rec: rec.tokens_saved, reverse=True)[:limit]
|
||||
],
|
||||
"suspicious_records": suspicious_records,
|
||||
}
|
||||
|
||||
|
||||
def _percentile(data: list[float], pct: float) -> float:
|
||||
if not data:
|
||||
return 0.0
|
||||
@@ -1199,6 +1321,7 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
total_before = sum(r.tokens_before for r in records)
|
||||
total_after = sum(r.tokens_after for r in records)
|
||||
total_saved = sum(r.tokens_saved for r in records)
|
||||
savings_audit = build_savings_audit(report)
|
||||
total_tool_saved = sum(r.tool_saved for r in records)
|
||||
total_headline_saved = total_saved + total_tool_saved
|
||||
|
||||
@@ -1211,12 +1334,17 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
for r in records:
|
||||
by_model_groups.setdefault(r.model, []).append(r)
|
||||
by_model = []
|
||||
estimated_list_price_savings_usd = 0.0
|
||||
for model, recs in sorted(by_model_groups.items()):
|
||||
m_before = sum(r.tokens_before for r in recs)
|
||||
m_after = sum(r.tokens_after for r in recs)
|
||||
m_message_saved = sum(r.tokens_saved for r in recs)
|
||||
m_tool_saved = sum(r.tool_saved for r in recs)
|
||||
m_saved = m_message_saved + m_tool_saved
|
||||
m_prompt_reduction = sum(_prompt_reduction_tokens(r) for r in recs)
|
||||
list_price = _get_list_price(model)
|
||||
if list_price:
|
||||
estimated_list_price_savings_usd += m_saved * list_price / 1_000_000
|
||||
by_model.append(
|
||||
{
|
||||
"model": model,
|
||||
@@ -1226,8 +1354,13 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
"tokens_saved": m_saved,
|
||||
"message_tokens_saved": m_message_saved,
|
||||
"tool_tokens_saved": m_tool_saved,
|
||||
"prompt_reduction_tokens": m_prompt_reduction,
|
||||
"accounting_delta_tokens": m_message_saved - m_prompt_reduction,
|
||||
"savings_pct": _pct(m_saved, m_before + m_tool_saved),
|
||||
"list_price_per_mtok": _get_list_price(model),
|
||||
"list_price_per_mtok": list_price,
|
||||
"estimated_list_price_savings_usd": round(m_saved * list_price / 1_000_000, 4)
|
||||
if list_price
|
||||
else None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1299,6 +1432,8 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
"tokens_saved": total_saved,
|
||||
"tool_saved": total_tool_saved,
|
||||
"savings_pct": _pct(total_saved, total_before),
|
||||
"estimated_list_price_savings_usd": round(estimated_list_price_savings_usd, 4),
|
||||
"savings_audit": savings_audit,
|
||||
"cache_read_tokens": total_cr,
|
||||
"cache_write_tokens": total_cw,
|
||||
"cache_hit_pct": cache_hit_pct,
|
||||
|
||||
@@ -19,6 +19,7 @@ from headroom.perf.analyzer import (
|
||||
TransformRecord,
|
||||
build_overhead_summary,
|
||||
build_perf_summary,
|
||||
build_savings_audit,
|
||||
perf_records_as_dicts,
|
||||
)
|
||||
|
||||
@@ -97,6 +98,8 @@ def test_build_perf_summary_totals_and_pct():
|
||||
assert summary["cache_write_tokens"] == 200
|
||||
assert summary["cache_hit_pct"] == 83.3
|
||||
assert summary["window_hours"] == 24.0
|
||||
assert summary["savings_audit"]["prompt_reduction_tokens"] == 1000
|
||||
assert summary["savings_audit"]["accounting_delta_tokens"] == 0
|
||||
|
||||
|
||||
def test_build_perf_summary_by_model_and_transform():
|
||||
@@ -164,6 +167,40 @@ def test_build_overhead_summary_attributes_slow_stages():
|
||||
assert overhead["top_slow_requests"][0]["slowest_stage"] == "kompress"
|
||||
|
||||
|
||||
def test_build_savings_audit_surfaces_accounting_deltas():
|
||||
report = PerfReport(
|
||||
perf_records=[
|
||||
PerfRecord(
|
||||
timestamp="2026-06-05 10:00:00,000",
|
||||
request_id="hr_bad",
|
||||
model="gpt-5",
|
||||
tokens_before=100,
|
||||
tokens_after=90,
|
||||
tokens_saved=250,
|
||||
),
|
||||
PerfRecord(
|
||||
timestamp="2026-06-05 10:01:00,000",
|
||||
request_id="hr_growth",
|
||||
model="gpt-5",
|
||||
tokens_before=100,
|
||||
tokens_after=120,
|
||||
tokens_saved=1,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
audit = build_savings_audit(report)
|
||||
|
||||
assert audit["logged_tokens_saved"] == 251
|
||||
assert audit["prompt_reduction_tokens"] == 10
|
||||
assert audit["accounting_delta_tokens"] == 241
|
||||
assert audit["record_counts"]["with_accounting_delta"] == 2
|
||||
assert audit["record_counts"]["logged_saved_gt_tokens_before"] == 1
|
||||
assert audit["record_counts"]["prompt_grew_but_logged_savings_positive"] == 1
|
||||
assert audit["suspicious_records"][0]["request_id"] == "hr_bad"
|
||||
assert "logged_saved_gt_tokens_before" in audit["suspicious_records"][0]["reasons"]
|
||||
|
||||
|
||||
def test_perf_records_as_dicts_roundtrips_fields():
|
||||
dicts = perf_records_as_dicts(_sample_report())
|
||||
assert len(dicts) == 2
|
||||
@@ -304,6 +341,8 @@ def test_perf_csv_by_model(runner, monkeypatch):
|
||||
assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
|
||||
sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
|
||||
assert sonnet["tokens_saved"] == "600"
|
||||
assert sonnet["prompt_reduction_tokens"] == "600"
|
||||
assert sonnet["accounting_delta_tokens"] == "0"
|
||||
|
||||
|
||||
def test_perf_csv_raw_per_record(runner, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user