mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
refactor(proxy): one copy of the handler helpers duplicated across providers
Several helpers existed in two or more handler modules with the same body: - _restore_frozen_prefix (anthropic.py, openai.py): now restore_frozen_prefix in headroom/cache/prefix_tracker.py, next to extract_cache_stable_delta. Both staticmethods delegate. - _extract_anthropic_cache_ttl_metrics (anthropic.py, streaming.py): now provider_usage.anthropic_cache_ttl_buckets. Both methods delegate, so instance overrides in tests still work. - _has_headroom_retrieve_tool (anthropic.py) and _has_headroom_retrieve_tool_responses (openai.py): one has_ccr_retrieve_tool in headroom/ccr/tool_calls.py. - _thinking_for_stream (streaming.py) re-implemented anthropic._thinking_tokens_for; the stream finaliser now calls that. - Six copies of the non-negative usage-count coercion (openai.py x4, streaming.py, anthropic.py) now use provider_usage.usage_int: the module's private _int, renamed and given a default. Gemini's _usage_int has different semantics and is left alone. The new helpers are not added to provider_usage.__all__. One behaviour difference: a boolean usage count (True) read as 1 at those six sites and now reads as 0, as provider_usage already did. No provider sends a boolean count. Also fixes test_thinking_tokens.py's junk-input test, which asserted 'x is None or True'. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assisted-by: claude-code:claude-fable-5 Assisted-by: claude-code:claude-opus-5-5 Headline-Session: 56d69125-fc5b-4ba0-a637-9be6fe80d456 Headline-Session: af43d0e8-fd20-4354-b4a6-edcd718ad548 Headline-AI-Share: 1% strict / 1% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
e1c0bbe6b9
Vendored
+30
@@ -589,6 +589,36 @@ def extract_cache_stable_delta(
|
||||
)
|
||||
|
||||
|
||||
def restore_frozen_prefix(
|
||||
original_messages: list[dict[str, Any]],
|
||||
candidate_messages: list[dict[str, Any]],
|
||||
*,
|
||||
frozen_message_count: int,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Force the frozen prefix of ``candidate_messages`` to match the original bytes.
|
||||
|
||||
Returns ``(restored_messages, restored_count)``. Messages a transform dropped
|
||||
from the prefix are put back; any frozen message it changed is replaced by
|
||||
the original. ``restored_count`` is how many prefix messages were restored.
|
||||
"""
|
||||
if frozen_message_count <= 0 or not original_messages:
|
||||
return candidate_messages, 0
|
||||
|
||||
frozen = min(frozen_message_count, len(original_messages))
|
||||
restored = list(candidate_messages)
|
||||
|
||||
# Defensive: if a transform dropped prefix messages, restore them.
|
||||
if len(restored) < frozen:
|
||||
return list(original_messages[:frozen]) + restored, frozen
|
||||
|
||||
changed = 0
|
||||
for idx in range(frozen):
|
||||
if restored[idx] != original_messages[idx]:
|
||||
restored[idx] = original_messages[idx]
|
||||
changed += 1
|
||||
return restored, changed
|
||||
|
||||
|
||||
def overlay_cached_prefix(
|
||||
optimized_messages: list[dict[str, Any]],
|
||||
current_original_messages: list[dict[str, Any]],
|
||||
|
||||
@@ -109,6 +109,25 @@ def is_ccr_tool_call(tool_call: dict[str, Any]) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def has_ccr_retrieve_tool(tools: Any) -> bool:
|
||||
"""Return true when a tool list defines the CCR retrieval tool.
|
||||
|
||||
Accepts flat definitions (Anthropic, OpenAI Responses: ``{"name": ...}``) and
|
||||
chat-completions definitions nested under ``"function"``.
|
||||
"""
|
||||
if not isinstance(tools, list):
|
||||
return False
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
if tool.get("name") == CCR_TOOL_NAME:
|
||||
return True
|
||||
function = tool.get("function")
|
||||
if isinstance(function, dict) and function.get("name") == CCR_TOOL_NAME:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def has_ccr_tool_calls(response: dict[str, Any], provider: str) -> bool:
|
||||
"""Return true when ``response`` contains at least one CCR tool call."""
|
||||
return any(is_ccr_tool_call(tool_call) for tool_call in extract_tool_calls(response, provider))
|
||||
|
||||
@@ -33,6 +33,7 @@ import httpx
|
||||
from headroom.agent_savings import proxy_pipeline_kwargs
|
||||
from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary
|
||||
from headroom.ccr.marker_resolution import resolve_markers_in_response
|
||||
from headroom.ccr.tool_calls import has_ccr_retrieve_tool
|
||||
from headroom.copilot_auth import apply_copilot_api_auth, is_copilot_upstream_url
|
||||
from headroom.pipeline import PipelineStage, summarize_routing_markers
|
||||
from headroom.proxy import public_errors
|
||||
@@ -68,6 +69,7 @@ from headroom.proxy.model_router import estimate_input_tokens, request_max_token
|
||||
from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
|
||||
from headroom.proxy.outcome import RequestOutcome
|
||||
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
|
||||
from headroom.proxy.provider_usage import anthropic_cache_ttl_buckets, usage_int
|
||||
from headroom.proxy.rate_limit_identity import rate_limit_identity
|
||||
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
|
||||
from headroom.proxy.tenant_key import resolve_tenant_key, set_request_tenant_key
|
||||
@@ -171,28 +173,21 @@ class _AnthropicTurnHookUsage:
|
||||
self.cache_write_1h_tokens = 0
|
||||
self.extra_calls = 0
|
||||
|
||||
@staticmethod
|
||||
def _int(value: Any) -> int:
|
||||
try:
|
||||
return max(0, int(value or 0))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
def record(self, payload: Any) -> None:
|
||||
usage = payload.get("usage") if isinstance(payload, dict) else None
|
||||
if not isinstance(usage, dict):
|
||||
self._seen.append((payload, 0, 0, 0, 0, 0, 0))
|
||||
return
|
||||
cache_read = self._int(usage.get("cache_read_input_tokens"))
|
||||
cache_write = self._int(usage.get("cache_creation_input_tokens"))
|
||||
cache_read = usage_int(usage.get("cache_read_input_tokens"))
|
||||
cache_write = usage_int(usage.get("cache_creation_input_tokens"))
|
||||
creation = usage.get("cache_creation")
|
||||
cache_write_5m = (
|
||||
self._int(creation.get("ephemeral_5m_input_tokens"))
|
||||
usage_int(creation.get("ephemeral_5m_input_tokens"))
|
||||
if isinstance(creation, dict)
|
||||
else 0
|
||||
)
|
||||
cache_write_1h = (
|
||||
self._int(creation.get("ephemeral_1h_input_tokens"))
|
||||
usage_int(creation.get("ephemeral_1h_input_tokens"))
|
||||
if isinstance(creation, dict)
|
||||
else 0
|
||||
)
|
||||
@@ -200,7 +195,7 @@ class _AnthropicTurnHookUsage:
|
||||
(
|
||||
payload,
|
||||
_anthropic_provider_input_tokens(usage),
|
||||
self._int(usage.get("output_tokens")),
|
||||
usage_int(usage.get("output_tokens")),
|
||||
cache_read,
|
||||
cache_write,
|
||||
cache_write_5m,
|
||||
@@ -240,7 +235,7 @@ def _anthropic_provider_input_tokens(usage: Any) -> int:
|
||||
if not isinstance(usage, dict):
|
||||
return 0
|
||||
return sum(
|
||||
_AnthropicTurnHookUsage._int(usage.get(key))
|
||||
usage_int(usage.get(key))
|
||||
for key in (
|
||||
"input_tokens",
|
||||
"cache_read_input_tokens",
|
||||
@@ -505,21 +500,6 @@ class AnthropicHandlerMixin:
|
||||
and name not in client_tool_names
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _has_headroom_retrieve_tool(tools: Any) -> bool:
|
||||
"""Return True when the final Anthropic tool list includes CCR retrieve."""
|
||||
if not isinstance(tools, list):
|
||||
return False
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
if tool.get("name") == "headroom_retrieve":
|
||||
return True
|
||||
function = tool.get("function")
|
||||
if isinstance(function, dict) and function.get("name") == "headroom_retrieve":
|
||||
return True
|
||||
return False
|
||||
|
||||
def _can_salvage_buffered_upstream(self, resp_json: Any) -> bool:
|
||||
"""May this upstream response be relayed when post-processing failed?
|
||||
|
||||
@@ -590,19 +570,10 @@ class AnthropicHandlerMixin:
|
||||
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
|
||||
"""Extract observed Anthropic cache-write TTL bucket usage.
|
||||
|
||||
HeadroomProxy also inherits StreamingMixin, which exposes the same
|
||||
helper for SSE usage parsing. Keep this local copy so the Anthropic
|
||||
handler remains safe when tested or embedded without StreamingMixin.
|
||||
StreamingMixin defines the same method; both delegate to one helper so
|
||||
this handler stays usable without StreamingMixin (tests, embedding).
|
||||
"""
|
||||
if not isinstance(usage, dict):
|
||||
return (0, 0)
|
||||
cache_creation = usage.get("cache_creation")
|
||||
if not isinstance(cache_creation, dict):
|
||||
return (0, 0)
|
||||
return (
|
||||
int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
|
||||
int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
|
||||
)
|
||||
return anthropic_cache_ttl_buckets(usage)
|
||||
|
||||
def _anthropic_buffered_request_timeout(self) -> httpx.Timeout:
|
||||
"""Timeout for buffered Anthropic reads."""
|
||||
@@ -790,22 +761,13 @@ class AnthropicHandlerMixin:
|
||||
frozen_message_count: int,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Force frozen prefix bytes to match the original request exactly."""
|
||||
if frozen_message_count <= 0 or not original_messages:
|
||||
return candidate_messages, 0
|
||||
from headroom.cache.prefix_tracker import restore_frozen_prefix
|
||||
|
||||
frozen = min(frozen_message_count, len(original_messages))
|
||||
restored = list(candidate_messages)
|
||||
|
||||
# Defensive: if a transform dropped prefix messages, restore them.
|
||||
if len(restored) < frozen:
|
||||
return list(original_messages[:frozen]) + restored, frozen
|
||||
|
||||
changed = 0
|
||||
for idx in range(frozen):
|
||||
if restored[idx] != original_messages[idx]:
|
||||
restored[idx] = original_messages[idx]
|
||||
changed += 1
|
||||
return restored, changed
|
||||
return restore_frozen_prefix(
|
||||
original_messages,
|
||||
candidate_messages,
|
||||
frozen_message_count=frozen_message_count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_cache_stable_delta(
|
||||
@@ -4012,9 +3974,7 @@ class AnthropicHandlerMixin:
|
||||
retrieve_tool_is_offered = (
|
||||
stream
|
||||
and ccr_response_handler_enabled
|
||||
and self._has_headroom_retrieve_tool(
|
||||
tools if tools is not None else body.get("tools")
|
||||
)
|
||||
and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
|
||||
)
|
||||
buffered_retrieval_can_help = (
|
||||
retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)
|
||||
|
||||
@@ -37,7 +37,11 @@ from headroom.proxy.helpers import (
|
||||
from headroom.proxy.identity import resolve_memory_identity
|
||||
from headroom.proxy.loopback_guard import is_loopback_host
|
||||
from headroom.proxy.modes import is_cache_mode
|
||||
from headroom.proxy.provider_usage import billed_input_for_provider, billed_input_from_usage
|
||||
from headroom.proxy.provider_usage import (
|
||||
billed_input_for_provider,
|
||||
billed_input_from_usage,
|
||||
usage_int,
|
||||
)
|
||||
from headroom.proxy.rate_limit_identity import rate_limit_identity
|
||||
from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
|
||||
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
|
||||
@@ -60,6 +64,7 @@ import httpx
|
||||
|
||||
from headroom.agent_savings import proxy_pipeline_kwargs
|
||||
from headroom.ccr.marker_resolution import resolve_markers_in_response
|
||||
from headroom.ccr.tool_calls import has_ccr_retrieve_tool
|
||||
from headroom.config import is_tool_excluded, unwrap_tool_call_name
|
||||
from headroom.copilot_auth import (
|
||||
apply_copilot_api_auth,
|
||||
@@ -361,20 +366,14 @@ class TurnHookUsage:
|
||||
500 a request by returning an odd shape."""
|
||||
usage = payload.get("usage") if isinstance(payload, dict) else None
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
if isinstance(usage, dict):
|
||||
details = usage.get(details_key)
|
||||
cached = _int(details.get("cached_tokens")) if isinstance(details, dict) else 0
|
||||
cached = usage_int(details.get("cached_tokens")) if isinstance(details, dict) else 0
|
||||
entry = (
|
||||
id(payload),
|
||||
payload,
|
||||
_int(usage.get(input_key)),
|
||||
_int(usage.get(output_key)),
|
||||
usage_int(usage.get(input_key)),
|
||||
usage_int(usage.get(output_key)),
|
||||
cached,
|
||||
)
|
||||
else:
|
||||
@@ -578,13 +577,6 @@ def _is_allowed_websocket_origin(headers: dict[str, str]) -> bool:
|
||||
return normalized_origin in normalized_allowed
|
||||
|
||||
|
||||
def _usage_int(value: Any) -> int:
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> dict[str, int]:
|
||||
"""Normalize usage from pass-through provider response shapes."""
|
||||
if not isinstance(payload, dict):
|
||||
@@ -593,9 +585,9 @@ def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> d
|
||||
usage_meta = payload.get("usageMetadata")
|
||||
if isinstance(usage_meta, dict):
|
||||
return {
|
||||
"input_tokens": _usage_int(usage_meta.get("promptTokenCount")),
|
||||
"input_tokens": usage_int(usage_meta.get("promptTokenCount")),
|
||||
"output_tokens": gemini_output_tokens(usage_meta),
|
||||
"cache_read_input_tokens": _usage_int(usage_meta.get("cachedContentTokenCount")),
|
||||
"cache_read_input_tokens": usage_int(usage_meta.get("cachedContentTokenCount")),
|
||||
}
|
||||
|
||||
usage = payload.get("usage")
|
||||
@@ -613,10 +605,10 @@ def _passthrough_usage_from_json(payload: Any, provider: str | None = None) -> d
|
||||
# the uncached tail only (cache buckets are disjoint); OpenAI's
|
||||
# headline figure is already inclusive. Callers derive uncached as
|
||||
# total - read - write, which is only right on the total.
|
||||
"input_tokens": billed_input_from_usage(usage, provider) or _usage_int(input_tokens),
|
||||
"output_tokens": _usage_int(output_tokens),
|
||||
"cache_read_input_tokens": _usage_int(usage.get("cache_read_input_tokens", cache_read)),
|
||||
"cache_creation_input_tokens": _usage_int(usage.get("cache_creation_input_tokens")),
|
||||
"input_tokens": billed_input_from_usage(usage, provider) or usage_int(input_tokens),
|
||||
"output_tokens": usage_int(output_tokens),
|
||||
"cache_read_input_tokens": usage_int(usage.get("cache_read_input_tokens", cache_read)),
|
||||
"cache_creation_input_tokens": usage_int(usage.get("cache_creation_input_tokens")),
|
||||
}
|
||||
|
||||
return {}
|
||||
@@ -1283,29 +1275,6 @@ def _responses_input_to_learner_messages(
|
||||
return messages
|
||||
|
||||
|
||||
def _has_headroom_retrieve_tool_responses(tools: Any) -> bool:
|
||||
"""Return True when the Responses API tool list includes CCR retrieve.
|
||||
|
||||
Responses API tool defs are flat (``{"type": "function", "name": ...}``)
|
||||
rather than nested under a "function" key like chat-completions
|
||||
tool_calls, so this can't reuse the chat-completions tool-list check.
|
||||
Mirrors ``AnthropicHandler._has_headroom_retrieve_tool``.
|
||||
"""
|
||||
from headroom.ccr import CCR_TOOL_NAME
|
||||
|
||||
if not isinstance(tools, list):
|
||||
return False
|
||||
for tool in tools:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
if tool.get("name") == CCR_TOOL_NAME:
|
||||
return True
|
||||
function = tool.get("function")
|
||||
if isinstance(function, dict) and function.get("name") == CCR_TOOL_NAME:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _should_buffer_openai_responses_stream_ccr(
|
||||
*,
|
||||
stream: bool,
|
||||
@@ -1327,7 +1296,7 @@ def _should_buffer_openai_responses_stream_ccr(
|
||||
and ccr_response_handler_enabled
|
||||
and not is_chatgpt_auth
|
||||
and not is_opencode_zen_base(upstream_base_url)
|
||||
and _has_headroom_retrieve_tool_responses(tools)
|
||||
and has_ccr_retrieve_tool(tools)
|
||||
)
|
||||
|
||||
|
||||
@@ -1829,16 +1798,10 @@ def _extract_responses_usage(event: dict[str, Any]) -> tuple[int, int, int, int,
|
||||
if not isinstance(usage, dict):
|
||||
return 0, 0, 0, 0, 0
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
input_tokens = _int(usage.get("input_tokens"))
|
||||
output_tokens = _int(usage.get("output_tokens"))
|
||||
input_tokens = usage_int(usage.get("input_tokens"))
|
||||
output_tokens = usage_int(usage.get("output_tokens"))
|
||||
details = usage.get("input_tokens_details")
|
||||
cached_tokens = _int(details.get("cached_tokens")) if isinstance(details, dict) else 0
|
||||
cached_tokens = usage_int(details.get("cached_tokens")) if isinstance(details, dict) else 0
|
||||
cache_write_tokens = _infer_openai_cache_write_tokens(input_tokens, cached_tokens)
|
||||
uncached_tokens = max(input_tokens - cached_tokens, 0)
|
||||
return input_tokens, output_tokens, cached_tokens, cache_write_tokens, uncached_tokens
|
||||
@@ -2115,22 +2078,14 @@ class OpenAIHandlerMixin:
|
||||
*,
|
||||
frozen_message_count: int,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Force frozen prefix bytes to match original request exactly."""
|
||||
if frozen_message_count <= 0 or not original_messages:
|
||||
return candidate_messages, 0
|
||||
"""Force frozen prefix bytes to match the original request exactly."""
|
||||
from headroom.cache.prefix_tracker import restore_frozen_prefix
|
||||
|
||||
frozen = min(frozen_message_count, len(original_messages))
|
||||
restored = list(candidate_messages)
|
||||
|
||||
if len(restored) < frozen:
|
||||
return list(original_messages[:frozen]) + restored, frozen
|
||||
|
||||
changed = 0
|
||||
for idx in range(frozen):
|
||||
if restored[idx] != original_messages[idx]:
|
||||
restored[idx] = original_messages[idx]
|
||||
changed += 1
|
||||
return restored, changed
|
||||
return restore_frozen_prefix(
|
||||
original_messages,
|
||||
candidate_messages,
|
||||
frozen_message_count=frozen_message_count,
|
||||
)
|
||||
|
||||
def _compress_openai_responses_live_text_units_with_router(
|
||||
self,
|
||||
@@ -5202,12 +5157,12 @@ class OpenAIHandlerMixin:
|
||||
# absent; a present-but-null count (some OpenAI-compatible
|
||||
# backends emit these on a stopped/empty turn) would return
|
||||
# None and crash the downstream `max(...)` arithmetic and the
|
||||
# int-typed outcome/metrics. `_usage_int` coerces both cases,
|
||||
# int-typed outcome/metrics. `usage_int` coerces both cases,
|
||||
# matching the streaming path and the guarded cache keys below
|
||||
# (same class as the gemini fix in #2347).
|
||||
output_tokens = _usage_int(usage.get("completion_tokens"))
|
||||
output_tokens = usage_int(usage.get("completion_tokens"))
|
||||
_thinking = extract_from_usage(usage)
|
||||
total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
|
||||
total_input_tokens = usage_int(usage.get("prompt_tokens")) or optimized_tokens
|
||||
|
||||
# Cache stats: prefer the Anthropic/Bedrock top-level
|
||||
# keys when present (authoritative). Fall back to
|
||||
@@ -5709,13 +5664,13 @@ class OpenAIHandlerMixin:
|
||||
# outside this try, so a null `prompt_tokens`/`cached_tokens`
|
||||
# would otherwise raise an uncaught TypeError and 500 the
|
||||
# request (same class as the gemini fix in #2347).
|
||||
total_input_tokens = _usage_int(usage.get("prompt_tokens")) or optimized_tokens
|
||||
output_tokens = _usage_int(usage.get("completion_tokens"))
|
||||
total_input_tokens = usage_int(usage.get("prompt_tokens")) or optimized_tokens
|
||||
output_tokens = usage_int(usage.get("completion_tokens"))
|
||||
_thinking = extract_from_usage(usage)
|
||||
# OpenAI returns cached_tokens in prompt_tokens_details
|
||||
# These are charged at 50% of the input price
|
||||
prompt_details = usage.get("prompt_tokens_details") or {}
|
||||
cache_read_tokens = _usage_int(prompt_details.get("cached_tokens"))
|
||||
cache_read_tokens = usage_int(prompt_details.get("cached_tokens"))
|
||||
except (KeyError, TypeError, AttributeError) as e:
|
||||
logger.debug(
|
||||
f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
|
||||
@@ -6983,21 +6938,15 @@ class OpenAIHandlerMixin:
|
||||
resp_json = response.json()
|
||||
usage = resp_json.get("usage", {})
|
||||
|
||||
def _usage_int(value: Any, default: int = 0) -> int:
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
total_input_tokens = _usage_int(
|
||||
total_input_tokens = usage_int(
|
||||
usage.get("input_tokens"),
|
||||
original_tokens,
|
||||
)
|
||||
output_tokens = _usage_int(usage.get("output_tokens"))
|
||||
output_tokens = usage_int(usage.get("output_tokens"))
|
||||
_thinking = extract_from_usage(usage)
|
||||
details = usage.get("input_tokens_details")
|
||||
if isinstance(details, dict):
|
||||
cache_read_tokens = _usage_int(details.get("cached_tokens"))
|
||||
cache_read_tokens = usage_int(details.get("cached_tokens"))
|
||||
except (
|
||||
json.JSONDecodeError,
|
||||
ValueError,
|
||||
|
||||
@@ -22,7 +22,11 @@ from headroom.proxy.helpers import (
|
||||
overload_retry_is_futile,
|
||||
retry_after_ms,
|
||||
)
|
||||
from headroom.proxy.provider_usage import billed_input_for_provider
|
||||
from headroom.proxy.provider_usage import (
|
||||
anthropic_cache_ttl_buckets,
|
||||
billed_input_for_provider,
|
||||
usage_int,
|
||||
)
|
||||
from headroom.proxy.token_counting import gemini_output_tokens
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -38,7 +42,6 @@ from headroom.proxy.memory_tool_stream import (
|
||||
MemoryToolStreamOverflowError,
|
||||
)
|
||||
from headroom.proxy.stream_output_tokens import estimate_output_tokens
|
||||
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
|
||||
from headroom.utils import format_exception_message
|
||||
|
||||
logger = logging.getLogger("headroom.proxy")
|
||||
@@ -181,23 +184,6 @@ def _upstream_error_frame(error_bytes: bytes, request_id: str) -> bytes:
|
||||
return f"event: error\ndata: {json.dumps(payload)}\n\n".encode()
|
||||
|
||||
|
||||
def _thinking_for_stream(payload: object) -> ThinkingTokens:
|
||||
"""Thinking-token split for a reassembled streaming response.
|
||||
|
||||
Shares the estimator with the non-streaming path so a streamed and an
|
||||
unstreamed response of the same shape produce the same number — otherwise
|
||||
the stratum would mix two different rulers.
|
||||
|
||||
Never raises: accounting must not cost a caller their response.
|
||||
"""
|
||||
try:
|
||||
from headroom.proxy.handlers.anthropic import _thinking_estimator
|
||||
|
||||
return extract_thinking_tokens(payload, estimator=_thinking_estimator())
|
||||
except Exception: # noqa: BLE001 - accounting must never break a response
|
||||
return ThinkingTokens()
|
||||
|
||||
|
||||
def _sse_dict(value: Any) -> dict[str, Any]:
|
||||
"""Return an upstream SSE object, or an empty object for a wrong-shaped value."""
|
||||
return value if isinstance(value, dict) else {}
|
||||
@@ -310,15 +296,7 @@ class StreamingMixin:
|
||||
@staticmethod
|
||||
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
|
||||
"""Extract observed Anthropic cache-write TTL bucket usage."""
|
||||
if not isinstance(usage, dict):
|
||||
return (0, 0)
|
||||
cache_creation = usage.get("cache_creation")
|
||||
if not isinstance(cache_creation, dict):
|
||||
return (0, 0)
|
||||
return (
|
||||
int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
|
||||
int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
|
||||
)
|
||||
return anthropic_cache_ttl_buckets(usage)
|
||||
|
||||
def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
|
||||
"""Parse usage information from SSE chunk.
|
||||
@@ -504,13 +482,6 @@ class StreamingMixin:
|
||||
if isinstance(response, dict):
|
||||
chunk_usage = response.get("usage")
|
||||
if isinstance(chunk_usage, dict):
|
||||
|
||||
def _usage_int(value: Any) -> int:
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
# Chat Completions streams report prompt/completion tokens.
|
||||
# Responses streams report input/output tokens under
|
||||
# response.usage on response.completed.
|
||||
@@ -520,15 +491,15 @@ class StreamingMixin:
|
||||
output_tokens = chunk_usage.get("completion_tokens")
|
||||
if output_tokens is None:
|
||||
output_tokens = chunk_usage.get("output_tokens", 0)
|
||||
usage_found["input_tokens"] = _usage_int(input_tokens)
|
||||
usage_found["output_tokens"] = _usage_int(output_tokens)
|
||||
usage_found["input_tokens"] = usage_int(input_tokens)
|
||||
usage_found["output_tokens"] = usage_int(output_tokens)
|
||||
details = (
|
||||
chunk_usage.get("prompt_tokens_details")
|
||||
or chunk_usage.get("input_tokens_details")
|
||||
or {}
|
||||
)
|
||||
if isinstance(details, dict):
|
||||
usage_found["cache_read_input_tokens"] = _usage_int(
|
||||
usage_found["cache_read_input_tokens"] = usage_int(
|
||||
details.get("cached_tokens")
|
||||
)
|
||||
|
||||
@@ -1104,8 +1075,11 @@ class StreamingMixin:
|
||||
#
|
||||
# ``parsed_response`` is the reassembled SSE body: it carries the content
|
||||
# blocks (thinking among them) and stop_reason, which the raw usage chunk
|
||||
# does not.
|
||||
_stream_thinking = _thinking_for_stream(parsed_response)
|
||||
# does not. Same helper as the non-streaming path, so a streamed and an
|
||||
# unstreamed response of the same shape count the same.
|
||||
from headroom.proxy.handlers.anthropic import _thinking_tokens_for
|
||||
|
||||
_stream_thinking = _thinking_tokens_for(parsed_response)
|
||||
_stream_stop_reason = (
|
||||
parsed_response.get("stop_reason") if isinstance(parsed_response, dict) else None
|
||||
)
|
||||
|
||||
@@ -41,13 +41,36 @@ __all__ = [
|
||||
_ANTHROPIC_SHAPE = frozenset({"anthropic", "bedrock", "vertex", "vertex_ai", "claude"})
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
def usage_int(value: Any, default: int = 0) -> int:
|
||||
"""One token count from a provider usage block, as a non-negative int.
|
||||
|
||||
Missing (``None``), non-numeric and boolean values give ``default``;
|
||||
negative counts clamp to 0. Never raises: usage fields are
|
||||
upstream-controlled and must not fail a response.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return 0
|
||||
return default
|
||||
try:
|
||||
return max(int(value), 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return default
|
||||
|
||||
|
||||
def anthropic_cache_ttl_buckets(usage: Any) -> tuple[int, int]:
|
||||
"""Cache-write tokens by TTL bucket ``(5m, 1h)`` from an Anthropic usage block.
|
||||
|
||||
Reads ``usage.cache_creation.ephemeral_{5m,1h}_input_tokens``; returns
|
||||
``(0, 0)`` when the block or its ``cache_creation`` object is absent.
|
||||
"""
|
||||
if not isinstance(usage, dict):
|
||||
return (0, 0)
|
||||
cache_creation = usage.get("cache_creation")
|
||||
if not isinstance(cache_creation, dict):
|
||||
return (0, 0)
|
||||
return (
|
||||
int(cache_creation.get("ephemeral_5m_input_tokens", 0) or 0),
|
||||
int(cache_creation.get("ephemeral_1h_input_tokens", 0) or 0),
|
||||
)
|
||||
|
||||
|
||||
def is_anthropic_dialect(provider: str | None) -> bool:
|
||||
@@ -64,7 +87,7 @@ def is_anthropic_dialect(provider: str | None) -> bool:
|
||||
|
||||
def anthropic_billed_input(input_tokens: Any, cache_read: Any = 0, cache_write: Any = 0) -> int:
|
||||
"""Billed input for an Anthropic-shape usage block: the three disjoint buckets."""
|
||||
return _int(input_tokens) + _int(cache_read) + _int(cache_write)
|
||||
return usage_int(input_tokens) + usage_int(cache_read) + usage_int(cache_write)
|
||||
|
||||
|
||||
def billed_input_for_provider(
|
||||
@@ -88,7 +111,7 @@ def billed_input_for_provider(
|
||||
# message_start can carry input_tokens=0 alongside real cache buckets
|
||||
# on a fully cached turn; that is still a provider-reported count.
|
||||
return total
|
||||
return _int(input_tokens)
|
||||
return usage_int(input_tokens)
|
||||
|
||||
|
||||
def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | None) -> int:
|
||||
@@ -104,14 +127,14 @@ def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | N
|
||||
|
||||
meta = payload.get("usageMetadata")
|
||||
if isinstance(meta, Mapping):
|
||||
return _int(meta.get("promptTokenCount"))
|
||||
return usage_int(meta.get("promptTokenCount"))
|
||||
|
||||
usage = payload.get("usage") if isinstance(payload.get("usage"), Mapping) else payload
|
||||
if not isinstance(usage, Mapping):
|
||||
return 0
|
||||
|
||||
if "promptTokenCount" in usage:
|
||||
return _int(usage.get("promptTokenCount"))
|
||||
return usage_int(usage.get("promptTokenCount"))
|
||||
|
||||
if is_anthropic_dialect(provider):
|
||||
if "input_tokens" not in usage:
|
||||
@@ -124,7 +147,7 @@ def billed_input_from_usage(payload: Mapping[str, Any] | None, provider: str | N
|
||||
|
||||
# OpenAI and OpenAI-compatible dialects: the headline figure is inclusive.
|
||||
if "prompt_tokens" in usage:
|
||||
return _int(usage.get("prompt_tokens"))
|
||||
return usage_int(usage.get("prompt_tokens"))
|
||||
if "input_tokens" in usage:
|
||||
return _int(usage.get("input_tokens"))
|
||||
return usage_int(usage.get("input_tokens"))
|
||||
return 0
|
||||
|
||||
@@ -29,6 +29,7 @@ from headroom.proxy.prometheus_metrics import PrometheusMetrics
|
||||
from headroom.proxy.provider_usage import (
|
||||
billed_input_for_provider,
|
||||
billed_input_from_usage,
|
||||
usage_int,
|
||||
)
|
||||
from headroom.proxy.server import HeadroomProxy
|
||||
from headroom.telemetry.reporter import UsageReporter
|
||||
@@ -295,3 +296,19 @@ def test_estimated_input_is_reported_as_estimated() -> None:
|
||||
assert payload["tokens_after"] == 400
|
||||
assert payload["tokens_after_provider_reported"] == 0
|
||||
assert payload["tokens_after_estimated"] == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "default", "expected"),
|
||||
[
|
||||
(12, 0, 12),
|
||||
("7", 0, 7),
|
||||
(-3, 0, 0),
|
||||
(None, 0, 0),
|
||||
(None, 41, 41),
|
||||
("n/a", 5, 5),
|
||||
(True, 0, 0),
|
||||
],
|
||||
)
|
||||
def test_usage_int_coerces_upstream_counts(value, default, expected) -> None:
|
||||
assert usage_int(value, default) == expected
|
||||
|
||||
@@ -219,6 +219,9 @@ class TestHandlerWiring:
|
||||
|
||||
def test_helper_never_raises_on_junk(self):
|
||||
from headroom.proxy.handlers.anthropic import _thinking_tokens_for
|
||||
from headroom.proxy.thinking_tokens import ThinkingTokens
|
||||
|
||||
for junk in (None, "string", 42, [], {"content": "not a list"}):
|
||||
assert _thinking_tokens_for(junk).tokens is None or True
|
||||
result = _thinking_tokens_for(junk)
|
||||
assert isinstance(result, ThinkingTokens)
|
||||
assert not result.tokens # nothing to count: unknown (None) or zero
|
||||
|
||||
Reference in New Issue
Block a user