mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(grok): attribute api.x.ai chat outcomes as provider xai, recognize grok-shell UA (#2626)
## Description Grok Build chat requests were recorded as `custom` or `openai`, preventing xAI provider rollups. This change records custom-base and directly routed Grok chat traffic as `xai`, recognizes both `grok/` and `grok-shell/` clients, and keeps API-key requests on credential-driven PAYG policy. Closes #3996 ## Type of Change - [x] Bug fix - [ ] New feature - [ ] Breaking change - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring ## Changes Made - Attribute supported xAI chat routes, including trailing-dot hosts, to `xai`. - Preserve per-model catalog cache prices and share the xAI cache fallback between dashboard costs and persisted savings. - Exclude inferred cache writes from billed aggregate totals; those tokens already belong to uncached input. Explicit provider-reported writes remain billable. - Cover custom-base outcome integration, catalog pricing precedence, legacy and current client credentials, and inferred writes with and without TTL splits. ## Testing At `29fd4f9d7a24a76f3a3f53edacc5832e357cae58`: 130 focused pricing, accounting, auth, cache, and routing tests pass (one existing Starlette warning). Ruff lint and format, whitespace checks, and Linux-target mypy over 567 source files pass. Main refresh retains `3c418f6f` ancestry. Latest documentation dependency audit found zero vulnerabilities. ## Real Behavior Proof - Environment: Windows, Python 3.12, a real uvicorn proxy and a separate loopback HTTP upstream. Subscription polling disabled. Only upstream network delivery is redirected; this is not live xAI billing traffic. - Steps: send four real TCP chat POSTs with custom-base, direct, legacy Grok, and trailing-dot routing; query the running proxy's `/stats` endpoint. The upstream returns 10,000 input tokens, 5,000 cached input tokens, and 1,000 output tokens per request. A priced xAI catalog fixture has $1/M input and $2/M output, with no explicit cache-read rate. - Observed: all four requests return HTTP 200, outcomes record `provider=xai` and `client=grok_build`, and `/stats` records `{"xai":4}`. Input spend is $0.0232 using the xAI 0.16 cache fallback. Before the aggregate-accounting correction this same replay reported $0.0432. - Evidence: `pr2626-resume15-pricing-runtime.log` and `pr2626-resume15-final-tests.log`; regressions reproduce the pricing/auth defects before correction. ## Runtime Rollout Safety - Rollout-managed features: none; attribution, credential classification, and accounting use the existing request paths. - Default behavior: supported xAI routes receive the xAI label; API keys use PAYG policy, JWT credentials retain OAuth policy. Unknown gateways retain the bounded custom provider bucket. - Pricing behavior: when a priced catalog row has no cache rates and the caller omits `provider`, `resolve_rates` now identifies the provider from `litellm_provider` and uses the shared fallback. For xAI this means cached input costs 0.16 times list input; explicit model cache rates still win. Dashboard aggregates also exclude inferred writes already counted as uncached input. These corrections change reported input costs and persisted savings, not upstream requests or billing. - Rollback: revert the pricing/auth/attribution commits; no new rollout switch or unsafe override is introduced. - Coverage: real HTTP routing and `/stats` replay plus focused default-policy and accounting tests. ## Checklist - [x] Self-review completed - [x] Added regression coverage - [x] Relevant local tests and static checks pass - [x] Did not edit CHANGELOG.md ## Additional Notes Hosted checks and automated review must qualify this newly published head before approval. No PR merge performed. --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Claude Fable 5
JD Davis
parent
489de46825
commit
b1fb162b62
@@ -0,0 +1,32 @@
|
||||
"""Lightweight provider cache fallbacks shared by all pricing surfaces.
|
||||
|
||||
Published per-model catalog rates always take precedence. These ratios are
|
||||
estimates for priced models whose catalog row has no cache rates. xAI uses
|
||||
a 0.16 read ratio and no write premium; known model rates override it.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
CACHE_ECONOMICS: dict[str, dict[str, Any]] = {
|
||||
"anthropic": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Explicit breakpoints, 5-min TTL",
|
||||
},
|
||||
"openai": {
|
||||
"read_multiplier": 0.5,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Automatic, no TTL control",
|
||||
},
|
||||
"gemini": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Explicit cachedContent, configurable TTL",
|
||||
},
|
||||
"bedrock": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Same as Anthropic (Bedrock)",
|
||||
},
|
||||
"xai": {"read_multiplier": 0.16, "write_multiplier": 1.0, "label": "Automatic, no TTL control"},
|
||||
}
|
||||
@@ -61,6 +61,7 @@ from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from headroom.cache_economics import CACHE_ECONOMICS
|
||||
from headroom.pricing.cache_ttl import CACHE_WRITE_MULTIPLIERS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -72,30 +73,8 @@ LONG_CONTEXT_THRESHOLD_TOKENS = 200_000
|
||||
|
||||
#: Provider-level cache discount ratios, as a fraction of the base input price.
|
||||
#: FALLBACK ONLY — the per-model LiteLLM catalog is always preferred, because
|
||||
#: these go stale per model and per context tier. Kept here (rather than in
|
||||
#: ``proxy/cost.py``, where they used to live) so every consumer shares one copy.
|
||||
CACHE_ECONOMICS: dict[str, dict[str, Any]] = {
|
||||
"anthropic": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Explicit breakpoints, 5-min TTL",
|
||||
},
|
||||
"openai": {
|
||||
"read_multiplier": 0.5,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Automatic, no TTL control",
|
||||
},
|
||||
"gemini": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Explicit cachedContent, configurable TTL",
|
||||
},
|
||||
"bedrock": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Same as Anthropic (Bedrock)",
|
||||
},
|
||||
}
|
||||
#: these go stale per model and per context tier. The lightweight shared table
|
||||
#: is also used by the dashboard, without importing the pricing package there.
|
||||
|
||||
|
||||
class Region(str, Enum):
|
||||
@@ -514,7 +493,15 @@ def resolve_rates(
|
||||
):
|
||||
# Catalog priced the model but published no cache rates. Fall back to
|
||||
# the provider ratio table if we can identify the provider.
|
||||
econ = CACHE_ECONOMICS.get((provider or "").split(":")[-1].strip().lower())
|
||||
# Model-only cost consumers do not have a request provider to pass.
|
||||
# Prefer explicit routing context, otherwise use the catalog identity.
|
||||
resolved_provider = provider or info.get("litellm_provider") or ""
|
||||
provider_key = (
|
||||
resolved_provider.split(":")[-1].strip().lower()
|
||||
if isinstance(resolved_provider, str)
|
||||
else ""
|
||||
)
|
||||
econ = CACHE_ECONOMICS.get(provider_key)
|
||||
if econ:
|
||||
return CacheRates(
|
||||
read=base * float(econ["read_multiplier"]),
|
||||
|
||||
@@ -19,7 +19,6 @@ SUBSCRIPTION_UA_PREFIXES: tuple[str, ...] = (
|
||||
"claude-code/",
|
||||
"codex-cli/",
|
||||
"cursor/",
|
||||
"grok/",
|
||||
"claude-vscode/",
|
||||
"github-copilot/",
|
||||
"anthropic-cli/",
|
||||
@@ -34,6 +33,11 @@ CLIENT_UA_MAP: tuple[tuple[str, str], ...] = (
|
||||
("anthropic-cli/", "anthropic-cli"),
|
||||
("codex-cli/", "codex"),
|
||||
("cursor/", "cursor"),
|
||||
# Current Grok Build releases send "grok-shell/<version>" (verified against
|
||||
# grok 0.2.112); older builds sent "grok/". Deliberately NOT added to
|
||||
# SUBSCRIPTION_UA_PREFIXES: API-key grok traffic is pay-per-token and
|
||||
# should keep the aggressive compression policy.
|
||||
("grok-shell/", "grok_build"),
|
||||
("grok/", "grok_build"),
|
||||
("zed/", "zed"),
|
||||
("aider/", "aider"),
|
||||
|
||||
+2
-22
@@ -15,6 +15,7 @@ from collections import deque
|
||||
from datetime import datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
from headroom.cache_economics import CACHE_ECONOMICS as _CACHE_ECONOMICS
|
||||
from headroom.proxy.budget_basis_policy import (
|
||||
BUDGET_BASIS_BLOCK,
|
||||
BUDGET_BASIS_IGNORE,
|
||||
@@ -109,28 +110,6 @@ class CostEntry(NamedTuple):
|
||||
# source for cache economics, and these ratios stand in when a model publishes
|
||||
# no cache pricing. Hardcoded ratios go stale per model and per context tier
|
||||
# (Anthropic's >200k rates differ), so they are never preferred over the catalog.
|
||||
_CACHE_ECONOMICS = {
|
||||
"anthropic": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Explicit breakpoints, 5-min TTL",
|
||||
},
|
||||
"openai": {
|
||||
"read_multiplier": 0.5,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Automatic, no TTL control",
|
||||
},
|
||||
"gemini": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.0,
|
||||
"label": "Explicit cachedContent, configurable TTL",
|
||||
},
|
||||
"bedrock": {
|
||||
"read_multiplier": 0.1,
|
||||
"write_multiplier": 1.25,
|
||||
"label": "Same as Anthropic (Bedrock)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
#: Context size at which the major catalogs publish a second, higher price
|
||||
@@ -285,6 +264,7 @@ def build_prefix_cache_stats(
|
||||
(provider in ("anthropic", "vertex:anthropic") and "claude" in model_name)
|
||||
or (provider == "openai" and any(p in model_name for p in _openai_prefixes))
|
||||
or (provider == "gemini" and "gemini" in model_name)
|
||||
or (provider == "xai" and "grok" in model_name)
|
||||
or (provider == "bedrock" and "claude" in model_name)
|
||||
)
|
||||
if is_match and tokens_sent > best_tokens:
|
||||
|
||||
@@ -3801,11 +3801,16 @@ class OpenAIHandlerMixin:
|
||||
handler_path,
|
||||
custom_upstream_base_url or "",
|
||||
)
|
||||
# Fixed taxonomy from the shared helper (zen, zai, meta, openai); any
|
||||
# other custom base is the shared "custom" bucket. Never derive the
|
||||
# Fixed taxonomy from the shared helper (zen, zai, meta, openai, xai);
|
||||
# any other custom base is the shared "custom" bucket. Never derive the
|
||||
# label from the request-controlled hostname — see the review on #3759.
|
||||
# Grok CLI routed to xAI without a base-url header is still xai.
|
||||
openai_chat_outcome_provider = custom_chat_provider or (
|
||||
CUSTOM_BASE_PROVIDER if custom_upstream_base_url else "openai"
|
||||
CUSTOM_BASE_PROVIDER
|
||||
if custom_upstream_base_url
|
||||
else "xai"
|
||||
if _is_xai_upstream(upstream_base_url)
|
||||
else "openai"
|
||||
)
|
||||
|
||||
# Memory: Get user ID when memory is enabled. Reads `request.headers`
|
||||
|
||||
@@ -14,6 +14,8 @@ CUSTOM_BASE_CHAT_PROVIDERS = {
|
||||
"api.openai.com": "openai",
|
||||
"api.z.ai": "zai",
|
||||
"api.meta.ai": "meta",
|
||||
# Grok Build: kept apart from OpenAI so savings rollups get their own row.
|
||||
"api.x.ai": "xai",
|
||||
}
|
||||
|
||||
# Fixed label for chat traffic on any other custom base.
|
||||
@@ -45,7 +47,8 @@ def custom_base_passthrough_telemetry(method: str, path: str, base_url: str) ->
|
||||
if method.upper() != "POST":
|
||||
return "", ""
|
||||
try:
|
||||
host = (urlparse(base_url.strip()).hostname or "").lower()
|
||||
# A fully qualified name (trailing dot) is the same host.
|
||||
host = (urlparse(base_url.strip()).hostname or "").lower().rstrip(".")
|
||||
except ValueError:
|
||||
return "", ""
|
||||
normalized_path = path[1:] if path.startswith("/") else path
|
||||
|
||||
@@ -43,13 +43,24 @@ def test_client_explicit_override_wins_over_user_agent() -> None:
|
||||
assert classify_client_signals(signals) == "aider"
|
||||
|
||||
|
||||
def test_grok_build_user_agent_is_subscription_client() -> None:
|
||||
def test_grok_build_user_agent_keeps_credential_driven_policy() -> None:
|
||||
signals = AuthSignals(user_agent="grok/1.2.3")
|
||||
|
||||
assert classify_auth_signals(signals) is AuthMode.SUBSCRIPTION
|
||||
assert classify_auth_signals(signals) is AuthMode.PAYG
|
||||
assert classify_client_signals(signals) == "grok_build"
|
||||
|
||||
|
||||
def test_grok_shell_user_agent_maps_to_grok_build_but_not_subscription() -> None:
|
||||
# Current Grok Build releases send grok-shell/<version> (grok 0.2.112).
|
||||
# Client classification must recognize it; auth mode deliberately stays
|
||||
# key-driven (API-key grok is pay-per-token and should keep the
|
||||
# aggressive policy), so grok-shell/ is NOT a subscription UA prefix.
|
||||
signals = AuthSignals(user_agent="grok-shell/0.2.112 (macos; aarch64)")
|
||||
|
||||
assert classify_client_signals(signals) == "grok_build"
|
||||
assert classify_auth_signals(signals) is not AuthMode.SUBSCRIPTION
|
||||
|
||||
|
||||
def test_codex_stamp_only_for_unidentified_responses_callers() -> None:
|
||||
assert should_stamp_codex_client_signals("/v1/responses", AuthSignals()) is True
|
||||
assert (
|
||||
|
||||
@@ -498,3 +498,23 @@ def test_reset_runtime_clears_miss_attribution() -> None:
|
||||
asyncio.run(metrics.record_cache_miss_attribution("anthropic", "ttl_expiry"))
|
||||
asyncio.run(metrics.reset_runtime())
|
||||
assert dict(metrics.cache_miss_attribution_by_provider) == {}
|
||||
|
||||
|
||||
def test_prefix_cache_stats_prices_xai_cache_reads_with_grok_models(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Grok traffic books under ``xai``; its cache reads must be priced with a
|
||||
Grok model, not dropped to $0 because no model matched the provider."""
|
||||
metrics = PrometheusMetrics()
|
||||
metrics.cache_by_provider["xai"].update(
|
||||
{"requests": 1, "hit_requests": 1, "cache_read_tokens": 1_000_000}
|
||||
)
|
||||
tracker = CostTracker()
|
||||
tracker._tokens_sent_by_model.update({"grok-4": 1})
|
||||
monkeypatch.setattr(CostTracker, "_get_list_price", lambda _self, _model: 3.0)
|
||||
monkeypatch.setattr(CostTracker, "_get_cache_prices", lambda _self, _model: None)
|
||||
|
||||
stats = build_prefix_cache_stats(metrics, tracker)
|
||||
|
||||
# No catalog cache rates: xAI's own fallback ratio, not Anthropic's 0.1.
|
||||
assert stats["by_provider"]["xai"]["savings_usd"] == round(1_000_000 * 3e-6 * (1 - 0.16), 4)
|
||||
|
||||
@@ -1161,11 +1161,23 @@ def test_openai_chat_outcome_provider_uses_the_fixed_taxonomy(
|
||||
_chat("https://llm.example.internal/v1")
|
||||
# No header: the plain OpenAI path keeps its own label (control).
|
||||
_chat(None)
|
||||
_chat("https://api.x.ai")
|
||||
_chat("https://api.x.ai.")
|
||||
|
||||
# Grok CLI on the default OpenAI target is routed to api.x.ai without
|
||||
# a base-url header (#2693); it is still xAI traffic, not OpenAI's.
|
||||
response = client.post(
|
||||
"/v1/chat/completions",
|
||||
headers={"authorization": "Bearer test-key", "user-agent": "grok-shell/0.2.117"},
|
||||
json={"model": "grok-4", "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
by_provider = proxy.metrics.requests_by_provider
|
||||
assert by_provider["meta"] == 2
|
||||
assert by_provider["custom"] == 1
|
||||
assert by_provider["openai"] == 1
|
||||
assert by_provider["xai"] == 3
|
||||
|
||||
|
||||
def test_openai_chat_custom_base_flood_cannot_grow_the_provider_set(
|
||||
|
||||
@@ -118,3 +118,30 @@ def test_custom_base_passthrough_telemetry_requires_a_chat_path_segment() -> Non
|
||||
"chat/completions",
|
||||
"zai",
|
||||
)
|
||||
|
||||
|
||||
def test_custom_base_passthrough_telemetry_recognizes_xai_chat() -> None:
|
||||
# Grok Build routes plain OpenAI chat completions through custom-base
|
||||
# routing with base_url https://api.x.ai; attribute the provider as xai
|
||||
# so savings rollups separate grok traffic from OpenAI's.
|
||||
assert custom_base_passthrough_telemetry(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
"https://api.x.ai",
|
||||
) == ("chat/completions", "xai")
|
||||
assert custom_base_passthrough_telemetry(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
"https://api.x.ai.",
|
||||
) == ("chat/completions", "xai")
|
||||
# Other xai paths and methods stay unlabelled.
|
||||
assert custom_base_passthrough_telemetry(
|
||||
"GET",
|
||||
"/v1/chat/completions",
|
||||
"https://api.x.ai",
|
||||
) == ("", "")
|
||||
assert custom_base_passthrough_telemetry(
|
||||
"POST",
|
||||
"/v1/models",
|
||||
"https://api.x.ai",
|
||||
) == ("", "")
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""xAI cached input must have the same price in savings and cost totals."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("inferred, expected", [(True, 0.0058), (False, 0.0108)])
|
||||
@pytest.mark.parametrize("ttl_split", [False, True])
|
||||
def test_xai_inferred_writes_are_not_billed_twice(monkeypatch, inferred, expected, ttl_split):
|
||||
import litellm
|
||||
|
||||
from headroom.proxy.cost import CostTracker
|
||||
|
||||
model = "xai/grok-review-inferred"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"litellm_provider": "xai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(
|
||||
model,
|
||||
tokens_saved=0,
|
||||
tokens_sent=10_000,
|
||||
cache_read_tokens=5_000,
|
||||
cache_write_tokens=5_000,
|
||||
cache_write_5m_tokens=2_000 if ttl_split else 0,
|
||||
cache_write_1h_tokens=3_000 if ttl_split else 0,
|
||||
uncached_tokens=5_000,
|
||||
cache_inferred=inferred,
|
||||
)
|
||||
assert tracker.totals()[1] == pytest.approx(expected)
|
||||
assert tracker.stats()["total_input_cost_usd"] == pytest.approx(expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("catalog_read", [None, 0.05e-6])
|
||||
def test_xai_cache_read_pricing_matches_cost_totals(monkeypatch, catalog_read):
|
||||
import litellm
|
||||
|
||||
from headroom.pricing.counterfactual import CacheMix, Region, price_savings, resolve_rates
|
||||
from headroom.proxy.cost import CostTracker
|
||||
|
||||
model = "xai/grok-review-fallback" if catalog_read is None else "xai/grok-review-catalog"
|
||||
row = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"litellm_provider": "xai",
|
||||
"mode": "chat",
|
||||
}
|
||||
if catalog_read is not None:
|
||||
row["cache_read_input_token_cost"] = catalog_read
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
resolve_rates.cache_clear()
|
||||
try:
|
||||
expected_read = catalog_read if catalog_read is not None else 0.16e-6
|
||||
mix = CacheMix.from_usage(cache_read_tokens=10_000)
|
||||
saving = price_savings(10_000, model=model, mix=mix, provider="xai", region=Region.PREFIX)
|
||||
assert saving.usd == pytest.approx(10_000 * expected_read)
|
||||
tracker = CostTracker()
|
||||
tracker.record_tokens(model, tokens_saved=0, tokens_sent=10_000, cache_read_tokens=10_000)
|
||||
assert tracker.stats()["total_input_cost_usd"] == pytest.approx(10_000 * expected_read)
|
||||
finally:
|
||||
resolve_rates.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("user_agent", ["grok/1.2.3", "grok-shell/0.2.112"])
|
||||
def test_grok_api_keys_keep_payg_policy_across_client_versions(user_agent):
|
||||
from headroom.proxy.auth_policy import (
|
||||
AuthMode,
|
||||
AuthSignals,
|
||||
classify_auth_signals,
|
||||
classify_client_signals,
|
||||
)
|
||||
|
||||
signals = AuthSignals(user_agent=user_agent, authorization="Bearer xai-test-key")
|
||||
assert classify_client_signals(signals) == "grok_build"
|
||||
assert classify_auth_signals(signals) is AuthMode.PAYG
|
||||
|
||||
|
||||
@pytest.mark.parametrize("user_agent", ["grok/1.2.3", "grok-shell/0.2.112"])
|
||||
def test_grok_oauth_tokens_keep_oauth_policy_across_client_versions(user_agent):
|
||||
from headroom.proxy.auth_policy import AuthMode, AuthSignals, classify_auth_signals
|
||||
|
||||
signals = AuthSignals(user_agent=user_agent, authorization="Bearer header.payload.signature")
|
||||
assert classify_auth_signals(signals) is AuthMode.OAUTH
|
||||
Reference in New Issue
Block a user