mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): skip pricing lookup for passthrough:* models
Closes #2578 Internal passthrough accounting labels are not billable models. Skip LiteLLM cost_per_token for passthrough:* names without warning, and omit them from per-model cost stats.
This commit is contained in:
+19
-2
@@ -729,6 +729,9 @@ class CostTracker:
|
||||
cache_read_tokens: Tokens served from cache (~10% of input rate)
|
||||
cache_write_tokens: Tokens written to cache (~125% of input rate)
|
||||
"""
|
||||
if model.startswith("passthrough:"):
|
||||
return None
|
||||
|
||||
litellm = _get_litellm_module()
|
||||
if litellm is None:
|
||||
_warn_pricing_once(
|
||||
@@ -918,6 +921,8 @@ class CostTracker:
|
||||
per_model = {}
|
||||
total_saved = 0
|
||||
for model in sorted(self._tokens_saved_by_model.keys()):
|
||||
if model.startswith("passthrough:"):
|
||||
continue
|
||||
saved = self._tokens_saved_by_model[model]
|
||||
sent = self._tokens_sent_by_model.get(model, 0)
|
||||
reqs = self._requests_by_model.get(model, 0)
|
||||
@@ -941,6 +946,8 @@ class CostTracker:
|
||||
total_billed_input_tokens = 0
|
||||
total_input_tokens = 0
|
||||
for model in self._tokens_saved_by_model:
|
||||
if model.startswith("passthrough:"):
|
||||
continue
|
||||
saved = self._tokens_saved_by_model[model]
|
||||
sent = self._tokens_sent_by_model.get(model, 0)
|
||||
cr = self._api_cache_read_by_model.get(model, 0)
|
||||
@@ -967,6 +974,8 @@ class CostTracker:
|
||||
# at the published $/token rate for its model. Not affected by cache mix.
|
||||
savings_usd = 0.0
|
||||
for model in self._tokens_saved_by_model:
|
||||
if model.startswith("passthrough:"):
|
||||
continue
|
||||
saved = self._tokens_saved_by_model[model]
|
||||
if saved <= 0:
|
||||
continue
|
||||
@@ -979,8 +988,16 @@ class CostTracker:
|
||||
"total_tokens_saved": total_saved,
|
||||
"total_input_tokens": total_input_tokens,
|
||||
"total_input_cost_usd": round(cost_with_headroom, 4),
|
||||
"cache_write_5m_tokens": sum(self._api_cache_write_5m_by_model.values()),
|
||||
"cache_write_1h_tokens": sum(self._api_cache_write_1h_by_model.values()),
|
||||
"cache_write_5m_tokens": sum(
|
||||
tokens
|
||||
for model, tokens in self._api_cache_write_5m_by_model.items()
|
||||
if not model.startswith("passthrough:")
|
||||
),
|
||||
"cache_write_1h_tokens": sum(
|
||||
tokens
|
||||
for model, tokens in self._api_cache_write_1h_by_model.items()
|
||||
if not model.startswith("passthrough:")
|
||||
),
|
||||
"per_model": per_model,
|
||||
"cost_with_headroom_usd": round(cost_with_headroom, 4),
|
||||
"savings_usd": round(savings_usd, 4),
|
||||
|
||||
@@ -63,3 +63,31 @@ def test_litellm_unavailable_warns_once_per_model(monkeypatch, caplog):
|
||||
|
||||
unavailable = [r for r in caplog.records if "LiteLLM not available" in r.getMessage()]
|
||||
assert len(unavailable) == 1
|
||||
|
||||
|
||||
def test_passthrough_model_skips_pricing_without_warning(monkeypatch, caplog):
|
||||
import headroom.proxy.cost as cost_mod
|
||||
|
||||
class _FakeLiteLLM:
|
||||
@staticmethod
|
||||
def cost_per_token(**_kwargs):
|
||||
raise AssertionError("cost_per_token must not be called")
|
||||
|
||||
monkeypatch.setattr(cost_mod, "_get_litellm_module", lambda: _FakeLiteLLM())
|
||||
tracker = cost_mod.CostTracker()
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
|
||||
assert tracker.estimate_cost("passthrough:count_tokens", 100, 50) is None
|
||||
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_stats_omit_passthrough_models(cost_tracker):
|
||||
cost_tracker.record_tokens("passthrough:count_tokens", 10, 90)
|
||||
cost_tracker.record_tokens("glm-5.2", 20, 80)
|
||||
|
||||
stats = cost_tracker.stats()
|
||||
|
||||
assert set(stats["per_model"]) == {"glm-5.2"}
|
||||
assert stats["total_tokens_saved"] == 20
|
||||
assert stats["total_input_tokens"] == 80
|
||||
|
||||
Reference in New Issue
Block a user