Merge branch 'main' into cleanup/cli-integrations-proxy-start-errors

This commit is contained in:
Tejas Chopra
2026-10-09 21:47:49 -07:00
committed by GitHub
4 changed files with 70 additions and 15 deletions
+37 -13
View File
@@ -30,7 +30,8 @@ from headroom.pricing.deepseek_tiers import (
OFF_PEAK_RATES_PER_1M,
off_peak_rates,
)
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
from headroom.pricing.litellm_model_resolution import pricing_lookup_candidates
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens, get_litellm_model_cost
from headroom.tokenizers.base import (
TokenCountCache,
coerce_countable_text,
@@ -219,8 +220,9 @@ def _apply_long_context_premium(
) -> dict[str, float]:
"""Return ``pricing`` scaled by the long-context premium where it applies.
Used only on the manual fallback path; the LiteLLM path already applies the
published above-threshold rates itself.
Used on the manual path, which also serves long-context requests whose
LiteLLM entry has no above-threshold rate (see
``_litellm_lacks_long_context_rate``).
"""
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
return pricing
@@ -229,6 +231,26 @@ def _apply_long_context_premium(
return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}
def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
"""True when this request needs the long-context premium but LiteLLM has no rate for it.
LiteLLM's live price map has dropped ``input_cost_per_token_above_200k_tokens``
from the bare Anthropic ids before, and pricing such a request through it then
bills the base tier: about half the real cost of a long session. The manual
table still carries the premium, so the caller uses it instead.
"""
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
return False
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
return False
cost_data = get_litellm_model_cost()
for candidate in pricing_lookup_candidates(model):
info = cost_data.get(candidate)
if isinstance(info, dict):
return "input_cost_per_token_above_200k_tokens" not in info
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
# Default limits for pattern-based inference
# Used when a model isn't in the explicit list but matches a known pattern
_PATTERN_DEFAULTS = {
@@ -837,16 +859,18 @@ class AnthropicProvider(Provider):
"""
model = sanitize_anthropic_model_id(model)
# LiteLLM knows per-model cache and long-context rates, so let it price
# the whole request rather than rebuilding the rate card here.
cost = estimate_cost_from_tokens(
model,
input_tokens=input_tokens,
output_tokens=output_tokens,
cached_tokens=cached_tokens,
now=now,
)
if cost is not None:
return cost
# the whole request rather than rebuilding the rate card here -- unless
# its entry is missing the long-context rate this request needs.
if not _litellm_lacks_long_context_rate(model, input_tokens):
cost = estimate_cost_from_tokens(
model,
input_tokens=input_tokens,
output_tokens=output_tokens,
cached_tokens=cached_tokens,
now=now,
)
if cost is not None:
return cost
# Fall back to manual pricing
pricing = self._get_pricing(model)
+4
View File
@@ -30,6 +30,10 @@ _CANDIDATES: tuple[str, ...] = (
"claude-sonnet-4-20250514",
"claude-opus-4-5-20251101",
"claude-opus-4-5",
# Last resort. By 2026-10, litellm had dropped the 200k-tier fields from
# every bare Anthropic id; only this Bedrock id still carries the full set,
# at the same rates as claude-sonnet-4-5.
"claude-sonnet-4-5-20250929-v1:0",
)
_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")
+3 -2
View File
@@ -172,12 +172,13 @@ def test_untagged_writes_default_to_the_five_minute_ttl():
def test_long_context_derives_the_one_hour_rate_and_says_so():
"""No catalog publishes a combined 1h + above-200k rate, so it derives."""
rates = resolve_rates(SONNET, long_context=True)
model = anthropic_pricing_model("input_cost_per_token_above_200k_tokens")
rates = resolve_rates(model, long_context=True)
assert rates.basis == BASIS_CATALOG_TTL_RATIO
assert rates.write_1h / rates.uncached == pytest.approx(2.00, abs=1e-6)
# And it is the EXPENSIVE tier, not the base one.
assert rates.uncached > resolve_rates(SONNET).uncached
assert rates.uncached > resolve_rates(model).uncached
# ── Provider agnosticism: every harness, every backend ────────────────────
+26
View File
@@ -136,6 +136,32 @@ class TestLongContextPricing:
)
assert cost == pytest.approx(expected, rel=1e-4)
def test_premium_survives_litellm_dropping_the_long_context_rate(self, provider, monkeypatch):
"""A LiteLLM entry without the above-200K rate must not bill a long prompt at the base tier."""
litellm = pytest.importorskip("litellm")
import headroom.providers.anthropic as anthropic_module
# One map for both readers: the guard and litellm's own cost_per_token.
monkeypatch.setitem(
litellm.model_cost,
"claude-sonnet-4-5",
{
"litellm_provider": "anthropic",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.5e-05,
"cache_read_input_token_cost": 3e-07,
},
)
assert provider.estimate_cost(300_000, 5_000, "claude-sonnet-4-5", 0) == pytest.approx(
1.9125, rel=1e-4
)
# Below the threshold the LiteLLM path still prices it.
assert (
anthropic_module._litellm_lacks_long_context_rate("claude-sonnet-4-5", 100_000) is False
)
def test_untiered_model_is_not_charged_a_premium(self, manual_provider):
# Opus is flat-rated across its whole window: 300K*$5 + 5K*$25 = $1.625.
cost = manual_provider.estimate_cost(300_000, 5_000, "claude-opus-4-5-20251101", 0)