fix(proxy): count deferred tools in the lineage key when they reach the provider eagerly

Leaving defer_loading tools out of the prefix-tracker lineage key is only
right while they reach the provider still deferred. A third-party Anthropic
upstream gets them with defer_loading stripped (the tool-search downgrade),
and a translating backend sends every tool as a plain function. There a
deferred load does change the provider's tools prefix, so keeping the lineage
would replay a frozen prefix the provider can no longer read and give up the
recompression a new lineage gets.

The downgrade predicate (_third_party_anthropic_upstream: provider
"anthropic" and a target base URL that is not first-party per
anthropic_first_party_tool_search_supported) is now computed once, before
the lineage key, and reused by the downgrade. Deferred tools leave the key
only when that predicate is false and no anthropic_backend is configured,
the same backend condition Headroom's own tool-search injector uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-14 13:14:49 +02:00
co-authored by Claude Opus 5.5
parent af23a94ed4
commit 2cca8abe9a
2 changed files with 58 additions and 11 deletions
+18 -8
View File
@@ -1656,12 +1656,26 @@ class AnthropicHandlerMixin:
# carrying different tool sets. Sharing one tracker across those
# requests pins cache reads at the early tools segment (#2671).
# Deferred tools are not in that key, so a ToolSearch load that
# appends one keeps the conversation's lineage.
# appends one keeps the conversation's lineage, but only where they
# reach the provider still deferred. A third-party upstream gets
# them eagerly (the tool-search downgrade below uses this same
# predicate) and so does a translating backend; there a load does
# change the provider's key, and the key here counts them.
from headroom.cache.prefix_tracker import cache_key_tools, segment_fingerprint
from headroom.proxy.helpers import anthropic_first_party_tool_search_supported
affinity_tools = cache_key_tools(
self._tools_for_forwarding(body.get("tools"), preserve_order=preserve_tool_order)
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
_third_party_anthropic_upstream = provider_name == "anthropic" and (
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
)
affinity_tools = self._tools_for_forwarding(
body.get("tools"), preserve_order=preserve_tool_order
)
if (
not _third_party_anthropic_upstream
and getattr(self, "anthropic_backend", None) is None
):
affinity_tools = cache_key_tools(affinity_tools)
cache_affinity = segment_fingerprint(
{
"model": model,
@@ -3194,14 +3208,10 @@ class AnthropicHandlerMixin:
tokens_saved = max(0, original_tokens - optimized_tokens)
from headroom.proxy.helpers import (
anthropic_first_party_tool_search_supported,
strip_first_party_tool_search_tools_for_third_party_upstream,
)
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
_third_party_anthropic_upstream = provider_name == "anthropic" and (
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
)
# _third_party_anthropic_upstream is decided before the lineage key.
if _third_party_anthropic_upstream:
_tools_before_strip = body.get("tools")
_tools_after_strip = strip_first_party_tool_search_tools_for_third_party_upstream(
+40 -3
View File
@@ -7,6 +7,11 @@ the turn after a load started a new lineage at frozen=0, the pipeline re-ran
the whole history, and history it had forwarded verbatim came out compressed:
different bytes, so the provider re-wrote the message tier.
Where the deferred tools reach the provider eagerly (a third-party upstream
gets them with ``defer_loading`` stripped, a translating backend as plain
functions), a load does change the provider's key, so it must still start a
new lineage.
The upstream below models Anthropic's prompt cache over the bytes the handler
forwards, deferred tools left out of the key, so the assertions are on what
the provider would do with them.
@@ -17,6 +22,7 @@ from __future__ import annotations
import copy
import json
import logging
from types import SimpleNamespace
from typing import Any
import pytest
@@ -26,6 +32,7 @@ pytest.importorskip("fastapi")
import httpx
from fastapi.testclient import TestClient
from headroom.backends.base import BackendResponse
from headroom.cache.prefix_tracker import PrefixCacheTracker, cache_key_tools
from headroom.proxy.helpers import reset_deferred_orphan_warn_state
from headroom.proxy.server import ProxyConfig, create_app
@@ -51,6 +58,7 @@ PLACEHOLDER = {
TOOLS = [READ, TOOL_SEARCH, PLACEHOLDER]
LOADED = {"name": "mcp__docs__search", "input_schema": {"type": "object"}, "defer_loading": True}
EAGER = {"name": "WebFetch", "input_schema": {"type": "object"}}
THIRD_PARTY_URL = "https://gateway.example.com/anthropic"
def _strip(obj: Any) -> Any:
@@ -147,7 +155,12 @@ class _PromptCache:
return sum(tokens[: hit + 1]), sum(tokens[hit + 1 :])
def _drive(requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]]) -> list[dict]:
def _drive(
requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]],
*,
api_url: str | None = None,
backend: bool = False,
) -> list[dict]:
"""Send (tools, messages) through the real handler; record each request's
resolved tracker, its frozen count, what was forwarded and the provider's
cache read and write."""
@@ -159,6 +172,7 @@ def _drive(requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]]) ->
rate_limit_enabled=False,
cost_tracking_enabled=False,
log_requests=False,
anthropic_api_url=api_url,
)
)
cache = _PromptCache()
@@ -201,7 +215,14 @@ def _drive(requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]]) ->
async def _upstream(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN003
return httpx.Response(200, json=_reply(body))
async def _backend_send(body, headers): # noqa: ANN001, ANN202
# A translating backend sends every tool as a plain function.
tools = [{k: v for k, v in t.items() if k != "defer_loading"} for t in body["tools"]]
return BackendResponse(body=_reply({**body, "tools": tools}), status_code=200)
proxy._retry_request = _upstream
if backend:
proxy.anthropic_backend = SimpleNamespace(name="litellm", send_message=_backend_send)
for tools, messages in requests:
response = client.post(
"/v1/messages",
@@ -219,13 +240,13 @@ def _drive(requests: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]]) ->
return seen
def _session(added: dict[str, Any]) -> tuple[dict, dict]:
def _session(added: dict[str, Any], **upstream: Any) -> tuple[dict, dict]:
turns = [(TOOLS, _history(n)) for n in range(1, 10)]
if added is LOADED:
turns.append(([*TOOLS, added], _after_tool_search(9)))
else:
turns.append(([*TOOLS, added], _history(10)))
seen = _drive(turns)
seen = _drive(turns, **upstream)
return seen[-2], seen[-1]
@@ -260,6 +281,22 @@ def test_deferred_tool_load_keeps_the_lineage_and_its_cached_prefix(caplog) -> N
assert not [r for r in caplog.records if "tool_search_deferred_orphan" in r.getMessage()]
@pytest.mark.parametrize(
"upstream",
[{"api_url": THIRD_PARTY_URL}, {"backend": True}],
ids=["third-party-upstream", "translating-backend"],
)
def test_deferred_tool_load_sent_eagerly_starts_a_new_lineage(upstream) -> None: # noqa: ANN001
before, after = _session(LOADED, **upstream)
assert after["tracker"] is not before["tracker"]
assert after["frozen"] == 0
# The provider got the loaded tool eagerly, so its key changed anyway:
# nothing of the old prefix was readable to keep.
assert not any(t.get("defer_loading") for t in after["tools"])
assert after["read"] == 0
def test_kill_switch_keys_deferred_tools_again(monkeypatch) -> None: # noqa: ANN001
monkeypatch.setenv("HEADROOM_DEFERRED_TOOL_AFFINITY", "0")
before, after = _session(LOADED)