perf: add savings audit output (#1211)

## Description
- add a savings_audit object to perf JSON output
- expose raw prompt reduction, logged savings, accounting delta,
suspicious records, and top saving requests
- add audit columns to aggregate CSV output and a compact text report
section

## Testing

- [x] Python compilation check passes
- [x] Current refreshed head has no failing GitHub checks

### Test Output

```text
python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py tests/test_cli_perf_format.py
Result: passed
GitHub status rollup: no failures
```
- python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py
tests/test_cli_perf_format.py
- pytest tests/test_cli_perf_format.py could not run locally because
pytest is not installed in this Python environment

## Type of Change

- [ ] Bug fix
- [ ] New feature
- [ ] Breaking change
- [ ] Documentation update
- [x] Performance improvement
- [ ] Code refactoring

## Changes Made

- Add a savings_audit object to PERF JSON output.
- Expose prompt reduction, recorded savings, accounting deltas,
suspicious records, and top-saving requests.
- Add audit columns to aggregate CSV output and a compact text report.

### Test Output

```text
python -m py_compile headroom/perf/analyzer.py headroom/cli/perf.py tests/test_cli_perf_format.py: passed
Current refreshed head: no failing GitHub checks
```

## Real Behavior Proof

- Environment: current refreshed PR head and the PERF analyzer/CLI
formatting test environment.
- Exact command / steps: compiled the changed analyzer, CLI, and
formatting-test modules and inspected the generated JSON, CSV, and text
audit surfaces.
- Observed result: audit data exposes raw reduction, recorded savings,
deltas, suspicious rows, and top requests without changing existing PERF
records.
- Not tested: a production-scale PERF log corpus.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
Co-authored-by: JD Davis <jd@jds-macbook-air.tail2a279.ts.net>
This commit is contained in:
akothari-godaddy
2026-10-02 23:46:58 -05:00
committed by GitHub
co-authored by JerrettDavis JD Davis
parent 31d0344991
commit 9a72bc0242
3 changed files with 178 additions and 1 deletions
+3
View File
@@ -82,8 +82,11 @@ def perf(hours: float, raw: bool, output_format: str) -> None:
"tokens_saved",
"message_tokens_saved",
"tool_tokens_saved",
"prompt_reduction_tokens",
"accounting_delta_tokens",
"savings_pct",
"list_price_per_mtok",
"estimated_list_price_savings_usd",
]
writer = csv.DictWriter(buf, fieldnames=fieldnames)
writer.writeheader()
+136 -1
View File
@@ -613,6 +613,28 @@ def format_report(report: PerfReport) -> str:
lines.append(f" · tool schemas {total_tool_saved:,}")
lines.append("")
audit = build_savings_audit(report)
lines.append("Savings Audit")
lines.append("-" * 40)
lines.append(
" Raw prompt reduction: "
f"{audit['prompt_reduction_tokens']:,} tokens "
f"({audit['prompt_reduction_pct']:.1f}%)"
)
lines.append(
" Logged savings: "
f"{audit['logged_tokens_saved']:,} tokens "
f"({audit['logged_savings_pct']:.1f}%)"
)
lines.append(f" Accounting delta: {audit['accounting_delta_tokens']:+,} tokens")
delta_count = audit["record_counts"]["with_accounting_delta"]
if delta_count:
lines.append(f" Records with delta: {delta_count}/{len(records)}")
impossible_count = audit["record_counts"]["logged_saved_gt_tokens_before"]
if impossible_count:
lines.append(f" Impossible records: {impossible_count} saved more than before")
lines.append("")
# Per-model breakdown with list prices
by_model: dict[str, list[PerfRecord]] = {}
for r in records:
@@ -973,6 +995,106 @@ def _pct(saved: int, before: int) -> float:
return round(saved / before * 100, 1) if before > 0 else 0.0
def _prompt_reduction_tokens(record: PerfRecord) -> int:
"""Raw prompt-token delta for a record, excluding cache/accounting effects."""
return max(record.tokens_before - record.tokens_after, 0)
def _audit_record(record: PerfRecord) -> dict:
prompt_reduction = _prompt_reduction_tokens(record)
return {
"timestamp": record.timestamp,
"request_id": record.request_id,
"model": record.model,
"client": record.client,
"tokens_before": record.tokens_before,
"tokens_after": record.tokens_after,
"logged_tokens_saved": record.tokens_saved,
"prompt_reduction_tokens": prompt_reduction,
"accounting_delta_tokens": record.tokens_saved - prompt_reduction,
"cache_read_tokens": record.cache_read,
"cache_write_tokens": record.cache_write,
"optimization_ms": record.optimization_ms,
"transforms": record.transforms,
}
def _record_accounting_reasons(record: PerfRecord) -> list[str]:
reasons: list[str] = []
if min(record.tokens_before, record.tokens_after, record.tokens_saved) < 0:
reasons.append("negative_token_count")
if record.tokens_before > 0 and record.tokens_saved > record.tokens_before:
reasons.append("logged_saved_gt_tokens_before")
if record.tokens_after > record.tokens_before and record.tokens_saved > 0:
reasons.append("prompt_grew_but_logged_savings_positive")
if record.tokens_saved != _prompt_reduction_tokens(record):
reasons.append("logged_saved_ne_prompt_delta")
return reasons
def build_savings_audit(report: PerfReport, *, limit: int = 10) -> dict:
"""Explain how headline savings relate to raw before/after token deltas."""
records = report.perf_records
total_before = sum(r.tokens_before for r in records)
total_after = sum(r.tokens_after for r in records)
logged_saved = sum(r.tokens_saved for r in records)
prompt_reduction = sum(_prompt_reduction_tokens(r) for r in records)
prompt_growth = sum(max(r.tokens_after - r.tokens_before, 0) for r in records)
accounting_delta = logged_saved - prompt_reduction
records_with_delta = [r for r in records if r.tokens_saved != _prompt_reduction_tokens(r)]
impossible_saved_gt_before = [
r for r in records if r.tokens_before > 0 and r.tokens_saved > r.tokens_before
]
prompt_growth_with_savings = [
r for r in records if r.tokens_after > r.tokens_before and r.tokens_saved > 0
]
negative_token_counts = [
r for r in records if min(r.tokens_before, r.tokens_after, r.tokens_saved) < 0
]
suspicious_records = []
for record in sorted(
records_with_delta,
key=lambda r: abs(r.tokens_saved - _prompt_reduction_tokens(r)),
reverse=True,
)[:limit]:
suspicious_records.append(
{
**_audit_record(record),
"reasons": _record_accounting_reasons(record),
}
)
return {
"formula": {
"raw_prompt_reduction": "sum(max(tokens_before - tokens_after, 0))",
"logged_savings": "sum(tokens_saved)",
"accounting_delta": "logged_savings - raw_prompt_reduction",
},
"total_tokens_before": total_before,
"total_tokens_after": total_after,
"logged_tokens_saved": logged_saved,
"logged_savings_pct": _pct(logged_saved, total_before),
"prompt_reduction_tokens": prompt_reduction,
"prompt_reduction_pct": _pct(prompt_reduction, total_before),
"prompt_growth_tokens": prompt_growth,
"accounting_delta_tokens": accounting_delta,
"record_counts": {
"total": len(records),
"with_accounting_delta": len(records_with_delta),
"logged_saved_gt_tokens_before": len(impossible_saved_gt_before),
"prompt_grew_but_logged_savings_positive": len(prompt_growth_with_savings),
"negative_token_count": len(negative_token_counts),
},
"top_saving_requests": [
_audit_record(r)
for r in sorted(records, key=lambda rec: rec.tokens_saved, reverse=True)[:limit]
],
"suspicious_records": suspicious_records,
}
def _percentile(data: list[float], pct: float) -> float:
if not data:
return 0.0
@@ -1199,6 +1321,7 @@ def build_perf_summary(report: PerfReport) -> dict:
total_before = sum(r.tokens_before for r in records)
total_after = sum(r.tokens_after for r in records)
total_saved = sum(r.tokens_saved for r in records)
savings_audit = build_savings_audit(report)
total_tool_saved = sum(r.tool_saved for r in records)
total_headline_saved = total_saved + total_tool_saved
@@ -1211,12 +1334,17 @@ def build_perf_summary(report: PerfReport) -> dict:
for r in records:
by_model_groups.setdefault(r.model, []).append(r)
by_model = []
estimated_list_price_savings_usd = 0.0
for model, recs in sorted(by_model_groups.items()):
m_before = sum(r.tokens_before for r in recs)
m_after = sum(r.tokens_after for r in recs)
m_message_saved = sum(r.tokens_saved for r in recs)
m_tool_saved = sum(r.tool_saved for r in recs)
m_saved = m_message_saved + m_tool_saved
m_prompt_reduction = sum(_prompt_reduction_tokens(r) for r in recs)
list_price = _get_list_price(model)
if list_price:
estimated_list_price_savings_usd += m_saved * list_price / 1_000_000
by_model.append(
{
"model": model,
@@ -1226,8 +1354,13 @@ def build_perf_summary(report: PerfReport) -> dict:
"tokens_saved": m_saved,
"message_tokens_saved": m_message_saved,
"tool_tokens_saved": m_tool_saved,
"prompt_reduction_tokens": m_prompt_reduction,
"accounting_delta_tokens": m_message_saved - m_prompt_reduction,
"savings_pct": _pct(m_saved, m_before + m_tool_saved),
"list_price_per_mtok": _get_list_price(model),
"list_price_per_mtok": list_price,
"estimated_list_price_savings_usd": round(m_saved * list_price / 1_000_000, 4)
if list_price
else None,
}
)
@@ -1299,6 +1432,8 @@ def build_perf_summary(report: PerfReport) -> dict:
"tokens_saved": total_saved,
"tool_saved": total_tool_saved,
"savings_pct": _pct(total_saved, total_before),
"estimated_list_price_savings_usd": round(estimated_list_price_savings_usd, 4),
"savings_audit": savings_audit,
"cache_read_tokens": total_cr,
"cache_write_tokens": total_cw,
"cache_hit_pct": cache_hit_pct,
+39
View File
@@ -19,6 +19,7 @@ from headroom.perf.analyzer import (
TransformRecord,
build_overhead_summary,
build_perf_summary,
build_savings_audit,
perf_records_as_dicts,
)
@@ -97,6 +98,8 @@ def test_build_perf_summary_totals_and_pct():
assert summary["cache_write_tokens"] == 200
assert summary["cache_hit_pct"] == 83.3
assert summary["window_hours"] == 24.0
assert summary["savings_audit"]["prompt_reduction_tokens"] == 1000
assert summary["savings_audit"]["accounting_delta_tokens"] == 0
def test_build_perf_summary_by_model_and_transform():
@@ -164,6 +167,40 @@ def test_build_overhead_summary_attributes_slow_stages():
assert overhead["top_slow_requests"][0]["slowest_stage"] == "kompress"
def test_build_savings_audit_surfaces_accounting_deltas():
report = PerfReport(
perf_records=[
PerfRecord(
timestamp="2026-06-05 10:00:00,000",
request_id="hr_bad",
model="gpt-5",
tokens_before=100,
tokens_after=90,
tokens_saved=250,
),
PerfRecord(
timestamp="2026-06-05 10:01:00,000",
request_id="hr_growth",
model="gpt-5",
tokens_before=100,
tokens_after=120,
tokens_saved=1,
),
]
)
audit = build_savings_audit(report)
assert audit["logged_tokens_saved"] == 251
assert audit["prompt_reduction_tokens"] == 10
assert audit["accounting_delta_tokens"] == 241
assert audit["record_counts"]["with_accounting_delta"] == 2
assert audit["record_counts"]["logged_saved_gt_tokens_before"] == 1
assert audit["record_counts"]["prompt_grew_but_logged_savings_positive"] == 1
assert audit["suspicious_records"][0]["request_id"] == "hr_bad"
assert "logged_saved_gt_tokens_before" in audit["suspicious_records"][0]["reasons"]
def test_perf_records_as_dicts_roundtrips_fields():
dicts = perf_records_as_dicts(_sample_report())
assert len(dicts) == 2
@@ -304,6 +341,8 @@ def test_perf_csv_by_model(runner, monkeypatch):
assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
assert sonnet["tokens_saved"] == "600"
assert sonnet["prompt_reduction_tokens"] == "600"
assert sonnet["accounting_delta_tokens"] == "0"
def test_perf_csv_raw_per_record(runner, monkeypatch):