mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(pricing): reconcile cache billing with model-specific LiteLLM rates (#3033)
When LiteLLM omits cache prices, Headroom's billed totals can disagree with its canonical calculator. The supported LiteLLM versions and model adapters handle absent rates differently, so assuming either list price or zero is incorrect. Inferred cache writes can also double-charge tokens already counted as uncached input. Fixes #3032. The shared, bounded rate resolver now uses the model-specific `litellm.cost_per_token` calculation for independently missing billed read/write rates. Explicit catalog rates retain precedence, including explicit zero, TTL tiers, and long-context rates. Failed, negative, or nonfinite calculations contribute zero rather than an invented provider ratio. Savings estimates retain their separate defaults and honest catalog/provider provenance. Billed write counters now accumulate effective writes, excluding inferred writes already charged as uncached input. Validation: - 155 focused pricing, TTL, stats, budget, savings, and counterfactual tests pass (one existing warning). - All 24 cache-price regressions pass separately on LiteLLM 1.96.2, locked 1.101.0, and hosted-CI 1.104.2. The canonical regression reproduces the hosted failure before the correction. - Ruff lint/format and whitespace checks pass. - Actual uvicorn proxy and separate loopback TCP upstream: Fireworks DeepSeek cache-only, mixed/inferred-write, and explicit gpt-4o-mini requests reconcile `/stats` with real LiteLLM calculations. On 1.104.2 the cumulative totals are $0.006, $0.018, and $0.0203; on 1.101.0 they are $0.012, $0.030, and $0.0323. These differences demonstrate why billing must consult the installed canonical calculator. The earlier LiteLLM 1.88.1 runtime example was below the supported minimum and is superseded by this supported-version matrix and TCP replay. No live provider billing is claimed. Current-head hosted checks and review must pass before approval; no merge performed. Final Linux-platform mypy: 566 source files pass. ## Runtime Rollout Safety Missing catalog rates can produce positive, zero, or discounted charges through the installed LiteLLM model calculator. Reported billed totals can therefore increase or decrease relative to the former list-price fallback; missing metadata does not mean free cache. Explicit catalog rates retain precedence. Counterfactual savings keep their separate estimates. Cache activity and billed usage deliberately differ for inferred writes: `/stats.prefix_cache` preserves Prometheus's raw activity count, while billed `CostTracker` totals and `/stats.cost` exclude inferred writes already charged as uncached input. A fresh actual TCP replay on LiteLLM 1.104.2 verifies activity write counts of 0, 5,000, and 15,000 after the three requests, alongside zero billed write/TTL counters and reconciled dollar totals. No new flag or provider-wide heuristic is introduced; reverting the billing resolver/counter changes restores prior reporting. --------- Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JerrettDavis
parent
1fb4cbaea0
commit
463c1a43e3
@@ -55,6 +55,7 @@ beats precise and wrong.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
@@ -338,6 +339,8 @@ class CacheRates:
|
||||
write_1h: float
|
||||
uncached: float
|
||||
basis: str = BASIS_CATALOG
|
||||
read_is_catalog: bool = True
|
||||
write_is_catalog: bool = True
|
||||
|
||||
def price(self, split: TokenSplit) -> float:
|
||||
return (
|
||||
@@ -373,15 +376,38 @@ def _litellm() -> Any | None:
|
||||
_RATE_CACHE_MAXSIZE = 256
|
||||
|
||||
|
||||
def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context: bool) -> float:
|
||||
"""Resolve an absent catalog slice through the model's billing calculator.
|
||||
|
||||
LiteLLM versions and model adapters differ: missing rates can contribute
|
||||
zero, list price, or a model-specific discount. Never invent our own ratio.
|
||||
The result is cached with the other resolved rates, not probed per request.
|
||||
"""
|
||||
tokens = LONG_CONTEXT_THRESHOLD_TOKENS + 1 if long_context else 1
|
||||
try:
|
||||
input_cost, _ = litellm.cost_per_token(
|
||||
model=model, prompt_tokens=tokens, completion_tokens=0, **{field: tokens}
|
||||
)
|
||||
rate = float(input_cost) / tokens
|
||||
except Exception:
|
||||
return 0.0
|
||||
return rate if math.isfinite(rate) and rate >= 0 else 0.0
|
||||
|
||||
|
||||
@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)
|
||||
def resolve_rates(
|
||||
model: str,
|
||||
*,
|
||||
long_context: bool = False,
|
||||
provider: str | None = None,
|
||||
for_billing: bool = False,
|
||||
) -> CacheRates | None:
|
||||
"""Resolve per-bucket input rates for ``model``, or ``None`` if unpriceable.
|
||||
|
||||
``for_billing`` resolves absent cache rates through LiteLLM's model-specific
|
||||
billing calculation. Savings estimates use the default fallbacks below
|
||||
instead; those estimates must not manufacture billed cache charges.
|
||||
|
||||
Preference order, strongest first:
|
||||
|
||||
1. **Catalog** — LiteLLM's per-model published rates, including the
|
||||
@@ -414,7 +440,8 @@ def resolve_rates(
|
||||
try:
|
||||
from headroom.pricing.litellm_pricing import resolve_litellm_model
|
||||
|
||||
info = litellm.model_cost.get(resolve_litellm_model(model), {}) or {}
|
||||
resolved_model = resolve_litellm_model(model)
|
||||
info = litellm.model_cost.get(resolved_model, {}) or {}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -432,16 +459,33 @@ def resolve_rates(
|
||||
"""Read ``field``, preferring its above-200k variant on long requests."""
|
||||
if long_context:
|
||||
hi = info.get(f"{field}_above_200k_tokens")
|
||||
if hi:
|
||||
if hi is not None:
|
||||
return float(hi)
|
||||
value = info.get(field)
|
||||
return float(value) if value is not None else default
|
||||
|
||||
read = _tier("cache_read_input_token_cost", base)
|
||||
# A provider that does not bill cache writes leaves this absent; writes then
|
||||
# cost the same as ordinary input, which is exactly OpenAI's and Gemini's
|
||||
# actual behaviour.
|
||||
write_5m = _tier("cache_creation_input_token_cost", base)
|
||||
# Counterfactual savings retain their existing, labelled estimate fallback.
|
||||
# Billed slices consult the canonical model calculator below when the
|
||||
# catalog omits a rate; missing metadata does not imply free cache.
|
||||
missing_cache_rate = 0.0 if for_billing else base
|
||||
read = _tier("cache_read_input_token_cost", missing_cache_rate)
|
||||
# Savings estimates value otherwise unpriced writes at ordinary input;
|
||||
# billed provider-reported writes require their own catalog rate.
|
||||
write_5m = _tier("cache_creation_input_token_cost", missing_cache_rate)
|
||||
read_is_catalog = info.get("cache_read_input_token_cost") is not None or (
|
||||
long_context and info.get("cache_read_input_token_cost_above_200k_tokens") is not None
|
||||
)
|
||||
write_is_catalog = info.get("cache_creation_input_token_cost") is not None or (
|
||||
long_context and info.get("cache_creation_input_token_cost_above_200k_tokens") is not None
|
||||
)
|
||||
if for_billing and not read_is_catalog:
|
||||
read = _canonical_cache_rate(
|
||||
litellm, resolved_model, "cache_read_input_tokens", long_context=long_context
|
||||
)
|
||||
if for_billing and not write_is_catalog:
|
||||
write_5m = _canonical_cache_rate(
|
||||
litellm, resolved_model, "cache_creation_input_tokens", long_context=long_context
|
||||
)
|
||||
|
||||
basis = BASIS_CATALOG
|
||||
write_1h_raw = info.get("cache_creation_input_token_cost_above_1hr")
|
||||
@@ -449,7 +493,7 @@ def resolve_rates(
|
||||
# No catalog publishes a combined 1h + above-200k rate, so the long
|
||||
# tier always derives. Ratio basis, and labelled as such.
|
||||
write_1h_raw = None
|
||||
if write_1h_raw:
|
||||
if write_1h_raw is not None:
|
||||
write_1h = float(write_1h_raw)
|
||||
elif write_5m > base:
|
||||
# Real write premium present but no 1h rate: derive structurally.
|
||||
@@ -460,7 +504,14 @@ def resolve_rates(
|
||||
# price; a 1h write costs what any write costs.
|
||||
write_1h = write_5m
|
||||
|
||||
if read == base and write_5m == base and base > 0:
|
||||
if (
|
||||
not for_billing
|
||||
and not read_is_catalog
|
||||
and not write_is_catalog
|
||||
and read == base
|
||||
and write_5m == base
|
||||
and base > 0
|
||||
):
|
||||
# Catalog priced the model but published no cache rates. Fall back to
|
||||
# the provider ratio table if we can identify the provider.
|
||||
econ = CACHE_ECONOMICS.get((provider or "").split(":")[-1].strip().lower())
|
||||
@@ -471,9 +522,19 @@ def resolve_rates(
|
||||
write_1h=base * float(econ["write_multiplier"]),
|
||||
uncached=base,
|
||||
basis=BASIS_PROVIDER_RATIO,
|
||||
read_is_catalog=read_is_catalog,
|
||||
write_is_catalog=write_is_catalog,
|
||||
)
|
||||
|
||||
return CacheRates(read=read, write_5m=write_5m, write_1h=write_1h, uncached=base, basis=basis)
|
||||
return CacheRates(
|
||||
read=read,
|
||||
write_5m=write_5m,
|
||||
write_1h=write_1h,
|
||||
uncached=base,
|
||||
basis=basis,
|
||||
read_is_catalog=read_is_catalog,
|
||||
write_is_catalog=write_is_catalog,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
+29
-12
@@ -291,16 +291,25 @@ def build_prefix_cache_stats(
|
||||
price_per_1m = cost_tracker._get_list_price(model_name)
|
||||
if price_per_1m:
|
||||
input_price_per_token = price_per_1m / 1_000_000
|
||||
cache_prices = cost_tracker._get_cache_prices(model_name)
|
||||
try:
|
||||
from headroom.pricing.counterfactual import resolve_rates
|
||||
|
||||
cache_prices = resolve_rates(model_name)
|
||||
except Exception:
|
||||
cache_prices = None
|
||||
best_tokens = tokens_sent
|
||||
|
||||
# Catalog rates win; the provider ratio table is the fallback for a model
|
||||
# that publishes no cache pricing.
|
||||
pricing_source = "provider_default"
|
||||
if cache_prices and input_price_per_token:
|
||||
_cr_price, _cw5_price, _cw1h_price, _uncached_price = cache_prices
|
||||
_cr_price = cache_prices.read
|
||||
_cw5_price = cache_prices.write_5m
|
||||
_cw1h_price = cache_prices.write_1h
|
||||
_uncached_price = cache_prices.uncached
|
||||
if _uncached_price:
|
||||
read_mult = _cr_price / _uncached_price
|
||||
if cache_prices.read_is_catalog:
|
||||
read_mult = _cr_price / _uncached_price
|
||||
# Blend the two write rates by the TTL mix this provider
|
||||
# actually wrote at, rather than charging every write the 5m
|
||||
# rate. Falls back to the 5m rate when nothing was written yet.
|
||||
@@ -309,8 +318,12 @@ def build_prefix_cache_stats(
|
||||
_w5 = max(0, _w_all - _w1h)
|
||||
_w_cost = _w5 * _cw5_price + _w1h * _cw1h_price
|
||||
_blended_write = (_w_cost / _w_all) if _w_all > 0 else _cw5_price
|
||||
write_mult = _blended_write / _uncached_price
|
||||
pricing_source = "catalog"
|
||||
if cache_prices.write_is_catalog:
|
||||
write_mult = _blended_write / _uncached_price
|
||||
if cache_prices.read_is_catalog and (cache_prices.write_is_catalog or not _w_all):
|
||||
pricing_source = "catalog"
|
||||
elif cache_prices.read_is_catalog or cache_prices.write_is_catalog:
|
||||
pricing_source = "mixed"
|
||||
|
||||
# Calculate savings:
|
||||
# Cache reads save (1.0 - read_mult) per token vs uncached input price.
|
||||
@@ -1150,13 +1163,13 @@ class CostTracker:
|
||||
self._api_cache_read_by_model.get(model, 0) + cache_read_tokens
|
||||
)
|
||||
self._api_cache_write_by_model[model] = (
|
||||
self._api_cache_write_by_model.get(model, 0) + cache_write_tokens
|
||||
self._api_cache_write_by_model.get(model, 0) + write_eff
|
||||
)
|
||||
self._api_cache_write_5m_by_model[model] = (
|
||||
self._api_cache_write_5m_by_model.get(model, 0) + cache_write_5m_tokens
|
||||
self._api_cache_write_5m_by_model.get(model, 0) + write_5m_eff
|
||||
)
|
||||
self._api_cache_write_1h_by_model[model] = (
|
||||
self._api_cache_write_1h_by_model.get(model, 0) + cache_write_1h_tokens
|
||||
self._api_cache_write_1h_by_model.get(model, 0) + write_1h_eff
|
||||
)
|
||||
self._api_uncached_by_model[model] = (
|
||||
self._api_uncached_by_model.get(model, 0) + uncached_tokens
|
||||
@@ -1401,7 +1414,7 @@ class CostTracker:
|
||||
return w5m * w5m_price + w1h * w1h_price
|
||||
|
||||
def _get_cache_prices(
|
||||
self, model: str, *, long_context: bool = False
|
||||
self, model: str, *, long_context: bool = False, for_billing: bool = True
|
||||
) -> tuple[float, float, float, float] | None:
|
||||
"""Per-token prices for (cache read, 5m write, 1h write, uncached input).
|
||||
|
||||
@@ -1411,6 +1424,10 @@ class CostTracker:
|
||||
persisted tracker, and the durable ledger alike, so the three cannot
|
||||
drift apart again.
|
||||
|
||||
``for_billing`` resolves missing cache catalog rates through LiteLLM's
|
||||
billing calculator. Savings consumers opt out to keep the resolver's
|
||||
counterfactual estimates.
|
||||
|
||||
The 1h write rate is new: LiteLLM publishes it per model as
|
||||
``cache_creation_input_token_cost_above_1hr`` (Sonnet: 2.00x base), and
|
||||
where a row omits it the structural multiplier in
|
||||
@@ -1427,7 +1444,7 @@ class CostTracker:
|
||||
try:
|
||||
from headroom.pricing.counterfactual import resolve_rates
|
||||
|
||||
rates = resolve_rates(model, long_context=long_context)
|
||||
rates = resolve_rates(model, long_context=long_context, for_billing=for_billing)
|
||||
except Exception:
|
||||
return None
|
||||
if rates is None or not rates.uncached:
|
||||
@@ -1586,7 +1603,7 @@ class CostTracker:
|
||||
| set(self._saved_list_by_tier)
|
||||
):
|
||||
model, long_context = key
|
||||
prices = self._get_cache_prices(model, long_context=long_context)
|
||||
prices = self._get_cache_prices(model, long_context=long_context, for_billing=False)
|
||||
if not prices:
|
||||
continue
|
||||
_cr_price, cw5_price, cw1h_price, uncached_price = prices
|
||||
@@ -1622,7 +1639,7 @@ class CostTracker:
|
||||
| set(self._tool_saved_list_by_model)
|
||||
)
|
||||
for model in tool_models:
|
||||
prices = self._get_cache_prices(model)
|
||||
prices = self._get_cache_prices(model, for_billing=False)
|
||||
if not prices:
|
||||
continue
|
||||
cr_price, cw5_price, cw1h_price, uncached_price = prices
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Reconcile reported cache slices with model-specific LiteLLM billing.
|
||||
|
||||
Explicit catalog rates win independently. Missing rates use the canonical
|
||||
calculator, whose fallback differs across supported versions and models;
|
||||
unavailable calculations do not invent Headroom provider-wide prices.
|
||||
Counterfactual savings and cache-card provenance remain separate from billing,
|
||||
and inferred writes cannot double-charge the uncached input bucket.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.pricing.counterfactual import resolve_rates
|
||||
from headroom.proxy.cost import CostTracker
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_rate_cache():
|
||||
resolve_rates.cache_clear()
|
||||
yield
|
||||
resolve_rates.cache_clear()
|
||||
|
||||
|
||||
def _patch_litellm(monkeypatch: pytest.MonkeyPatch, model_cost: dict) -> None:
|
||||
resolve_rates.cache_clear()
|
||||
monkeypatch.setattr(
|
||||
"headroom.pricing.counterfactual._litellm",
|
||||
lambda: SimpleNamespace(model_cost=model_cost),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.pricing.litellm_pricing.resolve_litellm_model",
|
||||
lambda model: model,
|
||||
)
|
||||
|
||||
|
||||
class TestGetCachePrices:
|
||||
def test_uses_explicit_litellm_cache_fields_when_present(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
}
|
||||
},
|
||||
)
|
||||
assert CostTracker()._get_cache_prices("m") == (1e-7, 1.25e-6, 2e-6, 1e-6)
|
||||
|
||||
def test_missing_cache_fields_priced_at_zero_not_full_rate(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# A model with an input price but no cache fields: exactly the 1600+
|
||||
# long-tail models the bug affects. Before the fix both slices billed at
|
||||
# the full uncached rate; now they fail closed at $0, reconciling with
|
||||
# what litellm.cost_per_token books for the same slice.
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{"m": {"input_cost_per_token": 5e-7, "litellm_provider": "mistral"}},
|
||||
)
|
||||
assert CostTracker()._get_cache_prices("m") == (0.0, 0.0, 0.0, 5e-7)
|
||||
|
||||
def test_only_the_missing_field_is_zeroed(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Explicit read cost present, write cost absent: keep the real read,
|
||||
# fail closed on the write.
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"litellm_provider": "openai",
|
||||
}
|
||||
},
|
||||
)
|
||||
assert CostTracker()._get_cache_prices("m") == (3e-7, 0.0, 0.0, 1e-6)
|
||||
|
||||
def test_no_input_price_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch_litellm(monkeypatch, {"m": {"litellm_provider": "anthropic"}})
|
||||
assert CostTracker()._get_cache_prices("m") is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_provider",
|
||||
["bedrock", "bedrock_converse", "mistral", "fireworks_ai", "cohere_chat", None, "made-up"],
|
||||
)
|
||||
def test_heterogeneous_and_unknown_providers_do_not_inherit_anthropic_rates(
|
||||
self, monkeypatch: pytest.MonkeyPatch, litellm_provider
|
||||
) -> None:
|
||||
# Regression: a missing cache field must never be back-filled with
|
||||
# Anthropic's 0.1x/1.25x economics. Bedrock fronts Anthropic, Amazon,
|
||||
# Meta, Mistral, Cohere and others; unknown OpenAI-compatible providers
|
||||
# are likewise not Anthropic-priced.
|
||||
uncached = 4e-6
|
||||
info = {"input_cost_per_token": uncached}
|
||||
if litellm_provider is not None:
|
||||
info["litellm_provider"] = litellm_provider
|
||||
_patch_litellm(monkeypatch, {"m": info})
|
||||
cache_read, cache_write, cache_write_1h, got_uncached = CostTracker()._get_cache_prices("m")
|
||||
assert got_uncached == uncached
|
||||
assert cache_read == 0.0
|
||||
assert cache_write == 0.0
|
||||
assert cache_write_1h == 0.0
|
||||
# Explicitly not the Anthropic multipliers the old fallback would apply.
|
||||
assert cache_read != pytest.approx(uncached * 0.1)
|
||||
assert cache_write != pytest.approx(uncached * 1.25)
|
||||
|
||||
def test_missing_cache_slices_match_canonical_zero_pricing(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The missing cache slices are priced at $0, the same amount the canonical
|
||||
litellm.cost_per_token path books for a slice LiteLLM cannot price, so the
|
||||
cache breakdown reconciles with the primary accounting instead of diverging.
|
||||
"""
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{"m": {"input_cost_per_token": 5e-7, "litellm_provider": "fireworks_ai"}},
|
||||
)
|
||||
cr_price, cw_price, _, uncached_price = CostTracker()._get_cache_prices("m")
|
||||
cr_tokens, cw_tokens = 20_000, 5_000
|
||||
# Cache-slice cost contributed to totals() is exactly $0, matching canonical.
|
||||
assert cr_tokens * cr_price + cw_tokens * cw_price == 0.0
|
||||
assert uncached_price == 5e-7
|
||||
|
||||
def test_totals_and_stats_charge_only_known_slices(self, monkeypatch):
|
||||
_patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 5e-7}})
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(
|
||||
model="m",
|
||||
tokens_saved=0,
|
||||
tokens_sent=26_000,
|
||||
cache_read_tokens=20_000,
|
||||
cache_write_tokens=5_000,
|
||||
uncached_tokens=1_000,
|
||||
)
|
||||
assert tracker.totals() == (26_000, 0.0005)
|
||||
assert tracker.stats()["total_input_cost_usd"] == 0.0005
|
||||
|
||||
def test_missing_read_preserves_explicit_write_and_ttl_tier(self, monkeypatch):
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 2e-6,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-6,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-6,
|
||||
}
|
||||
},
|
||||
)
|
||||
assert CostTracker()._get_cache_prices("m") == (0.0, 1.25e-6, 2e-6, 1e-6)
|
||||
assert CostTracker()._get_cache_prices("m", long_context=True) == (
|
||||
0.0,
|
||||
2.5e-6,
|
||||
4e-6,
|
||||
2e-6,
|
||||
)
|
||||
|
||||
def test_savings_provider_ratio_remains_separate_from_billed_prices(self, monkeypatch):
|
||||
_patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})
|
||||
assert resolve_rates("m", provider="anthropic").read == 1e-7
|
||||
assert CostTracker()._get_cache_prices("m")[0] == 0.0
|
||||
|
||||
def test_missing_write_price_keeps_live_zone_and_tool_savings(self, monkeypatch):
|
||||
_patch_litellm(monkeypatch, {"m": {"input_cost_per_token": 1e-6}})
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(
|
||||
model="m",
|
||||
tokens_saved=10_000,
|
||||
tokens_sent=10_000,
|
||||
cache_write_tokens=10_000,
|
||||
tool_schema_saved=5_000,
|
||||
)
|
||||
stats = tracker.stats()
|
||||
assert stats["total_input_cost_usd"] == 0.0
|
||||
assert stats["cache_aware_savings_usd"] == pytest.approx(0.01)
|
||||
assert stats["tool_savings_usd"] == pytest.approx(0.005)
|
||||
|
||||
@pytest.mark.parametrize("cache_read_price", [None, "invalid"])
|
||||
def test_unknown_read_rate_is_not_advertised_as_catalog_free_cache(
|
||||
self, monkeypatch, cache_read_price
|
||||
):
|
||||
from headroom.proxy.cost import build_prefix_cache_stats
|
||||
from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"gpt-unknown": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": cache_read_price,
|
||||
}
|
||||
},
|
||||
)
|
||||
tracker = CostTracker()
|
||||
tracker._tokens_sent_by_model["gpt-unknown"] = 1_000_000
|
||||
monkeypatch.setattr(tracker, "_get_list_price", lambda model: 1.0)
|
||||
metrics = PrometheusMetrics()
|
||||
metrics.cache_by_provider["openai"].update(
|
||||
{
|
||||
"requests": 1,
|
||||
"hit_requests": 1,
|
||||
"cache_read_tokens": 1_000_000,
|
||||
}
|
||||
)
|
||||
row = build_prefix_cache_stats(metrics, tracker)["by_provider"]["openai"]
|
||||
assert row["cache_pricing_source"] == "provider_default"
|
||||
assert row["savings_usd"] == pytest.approx(0.5)
|
||||
|
||||
def test_canonical_litellm_cache_only_cost_matches_dashboard(self, monkeypatch):
|
||||
import litellm
|
||||
|
||||
from headroom.pricing.litellm_pricing import resolve_litellm_model
|
||||
|
||||
# Keep the actual canonical calculator, with a deterministic catalog
|
||||
# row: the shared test suite enriches real model rows with cache rates.
|
||||
model = "openai/headroom-missing-cache-rate-regression"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"input_cost_per_token": 5e-7,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
resolved = resolve_litellm_model(model)
|
||||
info = litellm.model_cost[resolved]
|
||||
assert info.get("cache_read_input_token_cost") is None
|
||||
assert info.get("cache_creation_input_token_cost") is None
|
||||
canonical, _ = litellm.cost_per_token(
|
||||
model=resolved,
|
||||
prompt_tokens=10_000,
|
||||
completion_tokens=0,
|
||||
cache_read_input_tokens=10_000,
|
||||
)
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(
|
||||
model=model, tokens_saved=0, tokens_sent=10_000, cache_read_tokens=10_000
|
||||
)
|
||||
assert tracker.totals()[1] == round(canonical, 4)
|
||||
|
||||
@pytest.mark.parametrize("long_context", [False, True])
|
||||
def test_missing_catalog_rates_use_the_model_specific_canonical_calculation(
|
||||
self, monkeypatch, long_context
|
||||
):
|
||||
resolve_rates.cache_clear()
|
||||
monkeypatch.setattr(
|
||||
"headroom.pricing.litellm_pricing.resolve_litellm_model", lambda model: model
|
||||
)
|
||||
calls = []
|
||||
|
||||
def canonical(**kwargs):
|
||||
calls.append(kwargs)
|
||||
factor = 2 if kwargs["prompt_tokens"] > 200_000 else 1
|
||||
return (
|
||||
factor
|
||||
* (
|
||||
kwargs.get("cache_read_input_tokens", 0) * 4e-7
|
||||
+ kwargs.get("cache_creation_input_tokens", 0) * 1e-6
|
||||
),
|
||||
0.0,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.pricing.counterfactual._litellm",
|
||||
lambda: SimpleNamespace(
|
||||
model_cost={
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 2e-6,
|
||||
}
|
||||
},
|
||||
cost_per_token=canonical,
|
||||
),
|
||||
)
|
||||
factor = 2 if long_context else 1
|
||||
assert CostTracker()._get_cache_prices("m", long_context=long_context) == (
|
||||
4e-7 * factor,
|
||||
1e-6 * factor,
|
||||
1e-6 * factor,
|
||||
1e-6 * factor,
|
||||
)
|
||||
assert len(calls) == 2
|
||||
assert all(
|
||||
call["model"] == "m" and call["prompt_tokens"] == (200_001 if long_context else 1)
|
||||
for call in calls
|
||||
)
|
||||
|
||||
def test_inferred_writes_are_not_billed_twice_with_canonical_rates(self, monkeypatch):
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
"cache_creation_input_token_cost": 1e-6,
|
||||
}
|
||||
},
|
||||
)
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(
|
||||
model="m",
|
||||
tokens_saved=0,
|
||||
tokens_sent=20_000,
|
||||
cache_read_tokens=10_000,
|
||||
cache_write_tokens=10_000,
|
||||
cache_write_5m_tokens=10_000,
|
||||
uncached_tokens=10_000,
|
||||
cache_inferred=True,
|
||||
)
|
||||
assert tracker.totals()[1] == pytest.approx(0.015)
|
||||
assert tracker.stats()["total_input_cost_usd"] == pytest.approx(0.015)
|
||||
assert tracker._api_cache_write_by_model["m"] == 0
|
||||
assert tracker._api_cache_write_5m_by_model["m"] == 0
|
||||
|
||||
def test_explicit_zero_cache_tier_and_one_hour_prices_are_authoritative(self, monkeypatch):
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 2e-6,
|
||||
"cache_read_input_token_cost": 1e-7,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 0.0,
|
||||
"cache_creation_input_token_cost": 1.25e-6,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 0.0,
|
||||
"cache_creation_input_token_cost_above_1hr": 0.0,
|
||||
}
|
||||
},
|
||||
)
|
||||
assert CostTracker()._get_cache_prices("m")[2] == 0.0
|
||||
assert CostTracker()._get_cache_prices("m", long_context=True) == (
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
2e-6,
|
||||
)
|
||||
|
||||
def test_explicit_list_rate_cache_prices_do_not_inherit_provider_discounts(self, monkeypatch):
|
||||
_patch_litellm(
|
||||
monkeypatch,
|
||||
{
|
||||
"m": {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 1e-6,
|
||||
"cache_creation_input_token_cost": 1e-6,
|
||||
}
|
||||
},
|
||||
)
|
||||
rates = resolve_rates("m", provider="anthropic")
|
||||
assert rates.read == rates.write_5m == 1e-6
|
||||
assert rates.read_is_catalog and rates.write_is_catalog
|
||||
Reference in New Issue
Block a user