mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
Merge branch 'main' into cleanup/cli-integrations-proxy-start-errors
This commit is contained in:
@@ -30,7 +30,8 @@ from headroom.pricing.deepseek_tiers import (
|
||||
OFF_PEAK_RATES_PER_1M,
|
||||
off_peak_rates,
|
||||
)
|
||||
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens
|
||||
from headroom.pricing.litellm_model_resolution import pricing_lookup_candidates
|
||||
from headroom.pricing.litellm_pricing import estimate_cost_from_tokens, get_litellm_model_cost
|
||||
from headroom.tokenizers.base import (
|
||||
TokenCountCache,
|
||||
coerce_countable_text,
|
||||
@@ -219,8 +220,9 @@ def _apply_long_context_premium(
|
||||
) -> dict[str, float]:
|
||||
"""Return ``pricing`` scaled by the long-context premium where it applies.
|
||||
|
||||
Used only on the manual fallback path; the LiteLLM path already applies the
|
||||
published above-threshold rates itself.
|
||||
Used on the manual path, which also serves long-context requests whose
|
||||
LiteLLM entry has no above-threshold rate (see
|
||||
``_litellm_lacks_long_context_rate``).
|
||||
"""
|
||||
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
|
||||
return pricing
|
||||
@@ -229,6 +231,26 @@ def _apply_long_context_premium(
|
||||
return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}
|
||||
|
||||
|
||||
def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
|
||||
"""True when this request needs the long-context premium but LiteLLM has no rate for it.
|
||||
|
||||
LiteLLM's live price map has dropped ``input_cost_per_token_above_200k_tokens``
|
||||
from the bare Anthropic ids before, and pricing such a request through it then
|
||||
bills the base tier: about half the real cost of a long session. The manual
|
||||
table still carries the premium, so the caller uses it instead.
|
||||
"""
|
||||
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
|
||||
return False
|
||||
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
|
||||
return False
|
||||
cost_data = get_litellm_model_cost()
|
||||
for candidate in pricing_lookup_candidates(model):
|
||||
info = cost_data.get(candidate)
|
||||
if isinstance(info, dict):
|
||||
return "input_cost_per_token_above_200k_tokens" not in info
|
||||
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
|
||||
|
||||
|
||||
# Default limits for pattern-based inference
|
||||
# Used when a model isn't in the explicit list but matches a known pattern
|
||||
_PATTERN_DEFAULTS = {
|
||||
@@ -837,16 +859,18 @@ class AnthropicProvider(Provider):
|
||||
"""
|
||||
model = sanitize_anthropic_model_id(model)
|
||||
# LiteLLM knows per-model cache and long-context rates, so let it price
|
||||
# the whole request rather than rebuilding the rate card here.
|
||||
cost = estimate_cost_from_tokens(
|
||||
model,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
cached_tokens=cached_tokens,
|
||||
now=now,
|
||||
)
|
||||
if cost is not None:
|
||||
return cost
|
||||
# the whole request rather than rebuilding the rate card here -- unless
|
||||
# its entry is missing the long-context rate this request needs.
|
||||
if not _litellm_lacks_long_context_rate(model, input_tokens):
|
||||
cost = estimate_cost_from_tokens(
|
||||
model,
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
cached_tokens=cached_tokens,
|
||||
now=now,
|
||||
)
|
||||
if cost is not None:
|
||||
return cost
|
||||
|
||||
# Fall back to manual pricing
|
||||
pricing = self._get_pricing(model)
|
||||
|
||||
@@ -30,6 +30,10 @@ _CANDIDATES: tuple[str, ...] = (
|
||||
"claude-sonnet-4-20250514",
|
||||
"claude-opus-4-5-20251101",
|
||||
"claude-opus-4-5",
|
||||
# Last resort. By 2026-10, litellm had dropped the 200k-tier fields from
|
||||
# every bare Anthropic id; only this Bedrock id still carries the full set,
|
||||
# at the same rates as claude-sonnet-4-5.
|
||||
"claude-sonnet-4-5-20250929-v1:0",
|
||||
)
|
||||
|
||||
_BASE_FIELDS = ("input_cost_per_token", "output_cost_per_token")
|
||||
|
||||
@@ -172,12 +172,13 @@ def test_untagged_writes_default_to_the_five_minute_ttl():
|
||||
|
||||
def test_long_context_derives_the_one_hour_rate_and_says_so():
|
||||
"""No catalog publishes a combined 1h + above-200k rate, so it derives."""
|
||||
rates = resolve_rates(SONNET, long_context=True)
|
||||
model = anthropic_pricing_model("input_cost_per_token_above_200k_tokens")
|
||||
rates = resolve_rates(model, long_context=True)
|
||||
|
||||
assert rates.basis == BASIS_CATALOG_TTL_RATIO
|
||||
assert rates.write_1h / rates.uncached == pytest.approx(2.00, abs=1e-6)
|
||||
# And it is the EXPENSIVE tier, not the base one.
|
||||
assert rates.uncached > resolve_rates(SONNET).uncached
|
||||
assert rates.uncached > resolve_rates(model).uncached
|
||||
|
||||
|
||||
# ── Provider agnosticism: every harness, every backend ────────────────────
|
||||
|
||||
@@ -136,6 +136,32 @@ class TestLongContextPricing:
|
||||
)
|
||||
assert cost == pytest.approx(expected, rel=1e-4)
|
||||
|
||||
def test_premium_survives_litellm_dropping_the_long_context_rate(self, provider, monkeypatch):
|
||||
"""A LiteLLM entry without the above-200K rate must not bill a long prompt at the base tier."""
|
||||
litellm = pytest.importorskip("litellm")
|
||||
import headroom.providers.anthropic as anthropic_module
|
||||
|
||||
# One map for both readers: the guard and litellm's own cost_per_token.
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"claude-sonnet-4-5",
|
||||
{
|
||||
"litellm_provider": "anthropic",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 3e-06,
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
},
|
||||
)
|
||||
|
||||
assert provider.estimate_cost(300_000, 5_000, "claude-sonnet-4-5", 0) == pytest.approx(
|
||||
1.9125, rel=1e-4
|
||||
)
|
||||
# Below the threshold the LiteLLM path still prices it.
|
||||
assert (
|
||||
anthropic_module._litellm_lacks_long_context_rate("claude-sonnet-4-5", 100_000) is False
|
||||
)
|
||||
|
||||
def test_untiered_model_is_not_charged_a_premium(self, manual_provider):
|
||||
# Opus is flat-rated across its whole window: 300K*$5 + 5K*$25 = $1.625.
|
||||
cost = manual_provider.estimate_cost(300_000, 5_000, "claude-opus-4-5-20251101", 0)
|
||||
|
||||
Reference in New Issue
Block a user