mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(perf): make budget checks constant-time (#3368)
## Description Replace the per-request full scan of retained cost records with an incremental current-budget-window aggregate. Budget checks now read measured/estimated totals in O(1), while hourly, daily, and monthly expiry removes each entry once for amortized O(1) maintenance. Closes #3367 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [x] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Maintain current-period measured and estimated spend as costs are recorded. - Evict expired hourly/daily/monthly entries once as the window advances. - Preserve the existing 100,000-record cap and all `count`/`ignore`/`block` policy behavior. - Reset the incremental ledger with the rest of runtime cost state. - Add regressions for no-history-scan behavior, basis totals, all period boundaries, repeated reads, and capacity eviction. ## Testing - [x] Unit tests pass (`pytest`) - [ ] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ .venv/bin/pytest -q tests/test_cost_budget_performance.py tests/test_cost_budget_basis.py tests/test_cost_budget_total_prompt.py tests/test_cost_card_attribution.py tests/test_cost_pricing_warning_dedup.py tests/test_cost_tracker_counterfactual.py tests/test_cost_tracker_totals.py tests/test_pricing.py tests/test_pricing_cache_ttl.py tests/test_pricing_from_litellm.py tests/test_pricing_litellm.py tests/test_pricing_litellm_model_resolution.py tests/test_openai_pricing_resolution.py 75 passed, 4 skipped in 1.48s $ .venv/bin/ruff check headroom/proxy/cost.py tests/test_cost_budget_performance.py All checks passed! $ .venv/bin/ruff format --check headroom/proxy/cost.py tests/test_cost_budget_performance.py 2 files already formatted $ .venv/bin/mypy --follow-imports=skip headroom/proxy/cost.py Success: no issues found in 1 source file ``` ## Real Behavior Proof - Environment: Linux, Python 3.14, current `main` before and after this patch, production `CostTracker`, 100,000 current-month measured entries. - Exact command / steps: populated the retained/current-period ledgers with production `CostEntry` objects, timed repeated `check_budget()` calls, and ran an asyncio 1 ms heartbeat beside a burst of 20 checks. - Observed result: median budget-check time fell from 13.244 ms to 0.001865 ms at 100,000 entries; the 20-check event-loop stall fell from 265.9 ms to a 1.2922 ms maximum heartbeat gap. ```text BEFORE records=100000 median_ms=13.244 max_ms=36.060 checks_per_second=75.5 20_budget_checks_ms=264.8 max_event_loop_gap_ms=265.9 AFTER records=100000 median_ms=0.001865 max_ms=0.018969 20_budget_checks_ms=0.0572 max_event_loop_gap_ms=1.2922 ``` - Not tested: full repository suite; multi-process workers (each worker already owns an independent in-memory `CostTracker`); absolute timings on non-Linux filesystems/CPUs. Four selected pricing tests were skipped because LiteLLM is not installed in this worktree; the non-LiteLLM cost and budget tests ran. ## Runtime Rollout Safety - Rollout-managed feature(s): budget enforcement and cost reporting. - Minimum rollout channel: N/A. - Stable/default behavior changed: no accounting semantics change; only how current-period totals are maintained. - Kill switch / disable path: omit `budget_limit_usd` to disable enforcement; reverting this commit restores the scan implementation. - Unsafe override required: No. - Qualification impact: hourly, daily, and monthly boundaries plus all budget basis policies are covered by regressions. - Rollback path: revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) N/A. ## Additional Notes Documentation is unchanged because the public budget configuration and accounting semantics are unchanged. This targets the remaining enforcement scan; the earlier `CostTracker.totals()` optimization removed a separate full scan from metrics bookkeeping but did not change `check_budget()`.
This commit is contained in:
+62
-23
@@ -863,6 +863,15 @@ class CostTracker:
|
||||
# Cost tracking - using deque for efficient left-side removal
|
||||
self._costs: deque[CostEntry] = deque(maxlen=self.MAX_COST_ENTRIES)
|
||||
self._last_prune_time: datetime = datetime.now()
|
||||
# Current budget-window ledger and O(1) aggregates. ``_costs`` retains
|
||||
# the reporting history, while this deque evicts entries as soon as the
|
||||
# configured hourly/daily/monthly window advances. Each entry is added
|
||||
# and removed once, making expiry amortized O(1) instead of rescanning
|
||||
# up to MAX_COST_ENTRIES before every request (#3367).
|
||||
self._budget_costs: deque[CostEntry] = deque()
|
||||
self._budget_measured_usd = 0.0
|
||||
self._budget_estimated_usd = 0.0
|
||||
self._budget_estimated_records = 0
|
||||
|
||||
# Token savings per model (exact, no dollar estimation)
|
||||
self._tokens_saved_by_model: dict[str, int] = {}
|
||||
@@ -914,6 +923,10 @@ class CostTracker:
|
||||
"""Reset in-memory cost/token counters for local test/debug use."""
|
||||
self._costs.clear()
|
||||
self._last_prune_time = datetime.now()
|
||||
self._budget_costs.clear()
|
||||
self._budget_measured_usd = 0.0
|
||||
self._budget_estimated_usd = 0.0
|
||||
self._budget_estimated_records = 0
|
||||
self._tokens_saved_by_model.clear()
|
||||
self._saved_write_5m_by_tier.clear()
|
||||
self._saved_write_1h_by_tier.clear()
|
||||
@@ -1188,12 +1201,14 @@ class CostTracker:
|
||||
cache_write_tokens=effective_cache_write,
|
||||
)
|
||||
if cost is not None:
|
||||
self._costs.append(CostEntry(datetime.now(), cost, basis))
|
||||
entry = CostEntry(datetime.now(), cost, basis)
|
||||
self._costs.append(entry)
|
||||
self._record_budget_cost(entry)
|
||||
self._prune_old_costs()
|
||||
|
||||
def _period_cutoff(self) -> datetime:
|
||||
def _period_cutoff(self, now: datetime | None = None) -> datetime:
|
||||
"""Start of the current budget period."""
|
||||
now = datetime.now()
|
||||
now = now or datetime.now()
|
||||
|
||||
if self.budget_period == "hourly":
|
||||
return now - timedelta(hours=1)
|
||||
@@ -1202,6 +1217,37 @@ class CostTracker:
|
||||
# monthly
|
||||
return now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
def _subtract_budget_cost(self, entry: CostEntry) -> None:
|
||||
if entry.basis == COST_BASIS_ESTIMATED:
|
||||
self._budget_estimated_usd -= entry.cost_usd
|
||||
self._budget_estimated_records -= 1
|
||||
else:
|
||||
self._budget_measured_usd -= entry.cost_usd
|
||||
|
||||
def _refresh_budget_window(self, now: datetime | None = None) -> None:
|
||||
"""Evict expired current-period entries, each exactly once."""
|
||||
cutoff = self._period_cutoff(now)
|
||||
while self._budget_costs and self._budget_costs[0].timestamp < cutoff:
|
||||
self._subtract_budget_cost(self._budget_costs.popleft())
|
||||
|
||||
# Repeated floating-point additions/subtractions can leave a tiny
|
||||
# negative residue after a complete window rollover.
|
||||
self._budget_measured_usd = max(0.0, self._budget_measured_usd)
|
||||
self._budget_estimated_usd = max(0.0, self._budget_estimated_usd)
|
||||
self._budget_estimated_records = max(0, self._budget_estimated_records)
|
||||
|
||||
def _record_budget_cost(self, entry: CostEntry) -> None:
|
||||
"""Add one entry to the current-period aggregate in amortized O(1)."""
|
||||
self._refresh_budget_window(entry.timestamp)
|
||||
if len(self._budget_costs) >= self.MAX_COST_ENTRIES:
|
||||
self._subtract_budget_cost(self._budget_costs.popleft())
|
||||
self._budget_costs.append(entry)
|
||||
if entry.basis == COST_BASIS_ESTIMATED:
|
||||
self._budget_estimated_usd += entry.cost_usd
|
||||
self._budget_estimated_records += 1
|
||||
else:
|
||||
self._budget_measured_usd += entry.cost_usd
|
||||
|
||||
def get_period_cost(self, basis: str | None = None) -> float:
|
||||
"""Get cost for current budget period.
|
||||
|
||||
@@ -1209,12 +1255,14 @@ class CostTracker:
|
||||
regardless of how each record's input count was derived. Pass a basis
|
||||
(``"measured"`` / ``"estimated"``) to get just that slice.
|
||||
"""
|
||||
cutoff = self._period_cutoff()
|
||||
return sum(
|
||||
entry.cost_usd
|
||||
for entry in self._costs
|
||||
if entry.timestamp >= cutoff and (basis is None or entry.basis == basis)
|
||||
)
|
||||
breakdown = self.period_cost_breakdown()
|
||||
if basis == COST_BASIS_MEASURED:
|
||||
return float(breakdown["measured_usd"])
|
||||
if basis == COST_BASIS_ESTIMATED:
|
||||
return float(breakdown["estimated_usd"])
|
||||
if basis is not None:
|
||||
return 0.0
|
||||
return float(breakdown["total_usd"])
|
||||
|
||||
def period_cost_breakdown(self) -> dict[str, Any]:
|
||||
"""Split the period's booked spend by how its input count was derived.
|
||||
@@ -1224,20 +1272,11 @@ class CostTracker:
|
||||
it separable is the point: a budget refusal driven by a guess should be
|
||||
distinguishable from one driven by provider-reported usage (#2713).
|
||||
"""
|
||||
cutoff = self._period_cutoff()
|
||||
measured_usd = 0.0
|
||||
estimated_usd = 0.0
|
||||
records = 0
|
||||
estimated_records = 0
|
||||
for entry in self._costs:
|
||||
if entry.timestamp < cutoff:
|
||||
continue
|
||||
records += 1
|
||||
if entry.basis == COST_BASIS_ESTIMATED:
|
||||
estimated_usd += entry.cost_usd
|
||||
estimated_records += 1
|
||||
else:
|
||||
measured_usd += entry.cost_usd
|
||||
self._refresh_budget_window()
|
||||
measured_usd = self._budget_measured_usd
|
||||
estimated_usd = self._budget_estimated_usd
|
||||
records = len(self._budget_costs)
|
||||
estimated_records = self._budget_estimated_records
|
||||
|
||||
total_usd = measured_usd + estimated_usd
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Incremental budget aggregation regression coverage (#3367)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.proxy.budget_basis_policy import COST_BASIS_ESTIMATED, COST_BASIS_MEASURED
|
||||
from headroom.proxy.cost import CostEntry, CostTracker
|
||||
|
||||
|
||||
def test_budget_breakdown_does_not_scan_reporting_history() -> None:
|
||||
tracker = CostTracker(budget_limit_usd=100.0)
|
||||
now = datetime.now()
|
||||
tracker._costs.extend(
|
||||
CostEntry(now, 1.0, COST_BASIS_MEASURED) for _ in range(tracker.MAX_COST_ENTRIES)
|
||||
)
|
||||
tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_MEASURED))
|
||||
|
||||
class NonIterableHistory:
|
||||
def __iter__(self):
|
||||
raise AssertionError("budget check rescanned reporting history")
|
||||
|
||||
tracker._costs = NonIterableHistory() # type: ignore[assignment]
|
||||
|
||||
assert tracker.check_budget() == (True, 98.0)
|
||||
|
||||
|
||||
def test_budget_aggregate_preserves_basis_totals_and_policy() -> None:
|
||||
tracker = CostTracker(
|
||||
budget_limit_usd=10.0,
|
||||
estimated_basis_policy="ignore",
|
||||
)
|
||||
now = datetime.now()
|
||||
tracker._record_budget_cost(CostEntry(now, 3.0, COST_BASIS_MEASURED))
|
||||
tracker._record_budget_cost(CostEntry(now, 20.0, COST_BASIS_ESTIMATED))
|
||||
|
||||
assert tracker.period_cost_breakdown() == {
|
||||
"period": "daily",
|
||||
"policy": "ignore",
|
||||
"total_usd": 23.0,
|
||||
"measured_usd": 3.0,
|
||||
"estimated_usd": 20.0,
|
||||
"estimated_pct": pytest.approx(87.0),
|
||||
"records": 2,
|
||||
"estimated_records": 1,
|
||||
}
|
||||
assert tracker.check_budget() == (True, 7.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("period", "now", "expired_at", "current_at"),
|
||||
[
|
||||
(
|
||||
"hourly",
|
||||
datetime(2026, 8, 31, 12),
|
||||
datetime(2026, 8, 31, 10),
|
||||
datetime(2026, 8, 31, 11, 30),
|
||||
),
|
||||
(
|
||||
"daily",
|
||||
datetime(2026, 8, 31, 12),
|
||||
datetime(2026, 8, 30, 23, 59),
|
||||
datetime(2026, 8, 31, 1),
|
||||
),
|
||||
(
|
||||
"monthly",
|
||||
datetime(2026, 8, 31, 12),
|
||||
datetime(2026, 7, 31, 23, 59),
|
||||
datetime(2026, 8, 1),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_budget_window_evicts_expired_entries_once(period, now, expired_at, current_at) -> None:
|
||||
tracker = CostTracker(budget_limit_usd=100.0, budget_period=period)
|
||||
tracker._record_budget_cost(CostEntry(expired_at, 40.0, COST_BASIS_MEASURED))
|
||||
tracker._record_budget_cost(CostEntry(current_at, 5.0, COST_BASIS_ESTIMATED))
|
||||
|
||||
tracker._refresh_budget_window(now)
|
||||
assert tracker._budget_measured_usd == 0.0
|
||||
assert tracker._budget_estimated_usd == 5.0
|
||||
assert tracker._budget_estimated_records == 1
|
||||
assert len(tracker._budget_costs) == 1
|
||||
|
||||
# Re-reading the same window is constant-time and must not subtract twice.
|
||||
tracker._refresh_budget_window(now)
|
||||
assert tracker._budget_estimated_usd == 5.0
|
||||
assert len(tracker._budget_costs) == 1
|
||||
|
||||
|
||||
def test_budget_aggregate_keeps_existing_100k_record_cap() -> None:
|
||||
tracker = CostTracker(budget_limit_usd=1_000_000.0)
|
||||
tracker.MAX_COST_ENTRIES = 2
|
||||
now = datetime.now()
|
||||
tracker._record_budget_cost(CostEntry(now, 1.0, COST_BASIS_MEASURED))
|
||||
tracker._record_budget_cost(CostEntry(now, 2.0, COST_BASIS_ESTIMATED))
|
||||
tracker._record_budget_cost(CostEntry(now, 4.0, COST_BASIS_MEASURED))
|
||||
|
||||
assert tracker.period_cost_breakdown()["total_usd"] == 6.0
|
||||
assert tracker.period_cost_breakdown()["records"] == 2
|
||||
assert tracker.period_cost_breakdown()["estimated_records"] == 1
|
||||
Reference in New Issue
Block a user