fix(backends): forward the standard service_tier param (litellm + anyllm) (#2468)

## Description

`service_tier` is a standard provider request field (OpenAI
`flex`/`default`/`priority`, Anthropic `auto`/`standard_only`), but the
SDK-translation backends rebuilt the upstream request from a fixed field
whitelist and **silently dropped** it — so a client that set
`service_tier` had it discarded on `--backend litellm`/`anyllm`. The
direct passthrough already forwards it; this makes the translation
backends do the same, alongside `temperature`/`top_p`/`tools`.

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- `backends/litellm.py` (Anthropic path): forward `service_tier` when
present (the OpenAI path already forwards unconsumed fields via
`_build_openai_extra_body`).
- `backends/anyllm.py`: add `service_tier` to the forwarded params (both
the Anthropic whitelist and the OpenAI param list).

## Testing

- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check`)
- [x] Type checking passes (`mypy`)
- [x] New test added

### Test Output

```text
$ python -m pytest tests/test_backend_anyllm.py -q
16 passed in 2.51s

$ ruff check headroom/backends/litellm.py headroom/backends/anyllm.py
All checks passed!

$ mypy headroom/backends/anyllm.py headroom/backends/litellm.py   # 0 new errors
```

## Real Behavior Proof

- Environment: local, FakeAnyLLM instance capturing `acompletion`
kwargs.
- Steps: `send_message({..., "service_tier": "standard_only"})`.
- Observed: `instance.calls[0]["service_tier"] == "standard_only"` — the
param reaches the SDK call (previously absent).
- Not tested: live provider round-trip (no keys in CI); relies on the
SDK forwarding standard params.

## Checklist

- [x] Self-reviewed
- [x] Tests added and passing
- [x] I did **not** edit `CHANGELOG.md`
<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/2468"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->

---------

Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
Tejas Chopra
2026-11-03 22:29:09 -06:00
committed by GitHub
co-authored by JerrettDavis
parent 7a19803bc2
commit 62cc38a6b0
5 changed files with 229 additions and 0 deletions
+6
View File
@@ -258,6 +258,8 @@ class AnyLLMBackend(Backend):
kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]]
if "tool_choice" in body:
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
if "service_tier" in body:
kwargs["service_tier"] = body["service_tier"]
logger.debug(f"any-llm request: provider={self.provider}, model={original_model}")
@@ -308,6 +310,8 @@ class AnyLLMBackend(Backend):
kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]]
if "tool_choice" in body:
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
if "service_tier" in body:
kwargs["service_tier"] = body["service_tier"]
msg_id = f"msg_{uuid.uuid4().hex[:24]}"
@@ -494,6 +498,7 @@ class AnyLLMBackend(Backend):
"response_format",
"seed",
"n",
"service_tier",
]:
if param in body:
kwargs[param] = body[param]
@@ -615,6 +620,7 @@ class AnyLLMBackend(Backend):
"response_format",
"seed",
"n",
"service_tier",
]:
if param in body:
kwargs[param] = body[param]
+6
View File
@@ -47,6 +47,7 @@ _OPENAI_STANDARD_PARAMS = (
# reasoning request routed cross-protocol to a Claude model fails.
"reasoning_effort",
"max_completion_tokens",
"service_tier",
)
_OPENAI_CONSUMED_BODY_KEYS = frozenset(
@@ -1302,6 +1303,9 @@ class LiteLLMBackend(Backend):
kwargs["tools"] = [_convert_anthropic_tool(t) for t in tools_in]
if "tool_choice" in body:
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
# Standard provider param — forward it instead of silently dropping.
if "service_tier" in body:
kwargs["service_tier"] = body["service_tier"]
# System prompt (Anthropic puts it in body, OpenAI in messages)
if "system" in body:
@@ -1404,6 +1408,8 @@ class LiteLLMBackend(Backend):
"stream": True,
}
if "service_tier" in body:
kwargs["service_tier"] = body["service_tier"]
if "max_tokens" in body:
kwargs["max_tokens"] = body["max_tokens"]
if "temperature" in body:
+34
View File
@@ -761,3 +761,37 @@ async def test_close_is_noop(monkeypatch: pytest.MonkeyPatch) -> None:
backend, _instance = make_backend(monkeypatch)
assert await backend.close() is None
assert isinstance(StreamEvent(event_type="message_start", data={}), StreamEvent)
@pytest.mark.asyncio
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
@pytest.mark.parametrize("streaming", [False, True])
async def test_send_message_forwards_service_tier(
monkeypatch: pytest.MonkeyPatch, wire_format: str, streaming: bool
) -> None:
"""service_tier is a standard provider param — forward it, don't drop it."""
backend, instance = make_backend(monkeypatch)
instance.response = (
FakeAsyncStream([])
if streaming
else make_response(
make_choice("done", "stop"),
usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2),
)
)
body = {
"model": "claude-3-7-sonnet",
"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
"max_tokens": 16,
"service_tier": "standard_only",
}
if streaming:
method = (
backend.stream_message if wire_format == "anthropic" else backend.stream_openai_message
)
events = [event async for event in method(body, {})]
assert events
else:
method = backend.send_message if wire_format == "anthropic" else backend.send_openai_message
assert (await method(body, {})).status_code == 200
assert instance.calls[0]["service_tier"] == "standard_only"
@@ -0,0 +1,63 @@
"""Both Anthropic translation paths forward the caller's service tier unchanged."""
from __future__ import annotations
from copy import deepcopy
from unittest.mock import AsyncMock
import pytest
from tests._dotenv import importorskip_no_env_leak
litellm = importorskip_no_env_leak("litellm")
from headroom.backends import litellm as backend_module # noqa: E402
@pytest.mark.asyncio
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
@pytest.mark.parametrize("streaming", [False, True])
@pytest.mark.parametrize("tier", ["absent", None, "default", "flex", "priority"])
async def test_anthropic_service_tier_is_forwarded_in_both_paths(
monkeypatch, streaming, tier, wire_format
) -> None:
backend = backend_module.LiteLLMBackend(provider="openrouter")
body = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hello"}]}
if tier != "absent":
body["service_tier"] = tier
original = deepcopy(body)
async def chunks():
for chunk in []:
yield chunk
response = (
chunks()
if streaming
else litellm.ModelResponse(
model="claude-sonnet-4-5",
choices=[{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
)
)
completion = AsyncMock(return_value=response)
monkeypatch.setattr(backend_module, "acompletion", completion)
if streaming:
method = (
backend.stream_message if wire_format == "anthropic" else backend.stream_openai_message
)
events = [event async for event in method(body, {})]
assert events
if wire_format == "anthropic":
assert events[-1].event_type == "message_stop"
else:
method = backend.send_message if wire_format == "anthropic" else backend.send_openai_message
assert (await method(body, {})).status_code == 200
completion.assert_awaited_once()
kwargs = completion.call_args.kwargs
if tier == "absent":
assert "service_tier" not in kwargs
else:
assert kwargs["service_tier"] == tier
assert "service_tier" not in kwargs.get("extra_body", {})
assert body == original
@@ -0,0 +1,120 @@
"""Exercise real SDK serialization against a local OpenAI-compatible HTTP endpoint."""
from __future__ import annotations
import json
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
import pytest
from tests._dotenv import importorskip_no_env_leak
@pytest.mark.asyncio
@pytest.mark.parametrize("sdk", ["litellm", "anyllm"])
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
@pytest.mark.parametrize("streaming", [False, True])
async def test_real_sdk_puts_service_tier_on_the_provider_request(
sdk, wire_format, streaming, monkeypatch
) -> None:
sdk_module = importorskip_no_env_leak("litellm" if sdk == "litellm" else "any_llm")
monkeypatch.delenv("HEADROOM_OFFLINE", raising=False)
received = []
class Provider(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
received.append(body)
response = {
"id": "chatcmpl-local",
"object": "chat.completion",
"created": 0,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
if body.get("stream"):
response["object"] = "chat.completion.chunk"
response["choices"][0]["delta"] = response["choices"][0].pop("message")
payload = ("data: " + json.dumps(response) + "\n\ndata: [DONE]\n\n").encode()
media = "text/event-stream"
else:
payload = json.dumps(response).encode()
media = "application/json"
self.send_response(200)
self.send_header("Content-Type", media)
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, *args):
pass
with HTTPServer(("127.0.0.1", 0), Provider) as server:
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
url = f"http://127.0.0.1:{server.server_port}/v1"
if sdk == "litellm":
from headroom.backends.litellm import LiteLLMBackend
monkeypatch.setattr(sdk_module, "api_base", url)
monkeypatch.setattr(sdk_module, "api_key", "local-fake-key")
backend = LiteLLMBackend(provider="openai")
else:
from headroom.backends.anyllm import AnyLLMBackend
backend = AnyLLMBackend(provider="openai", api_base=url, api_key="local-fake-key")
body = {
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 8,
"service_tier": "flex",
}
try:
if streaming:
method = (
backend.stream_message
if wire_format == "anthropic"
else backend.stream_openai_message
)
events = [event async for event in method(body, {})]
assert events
if wire_format == "anthropic":
assert all(event.event_type != "error" for event in events)
assert events[-1].event_type == "message_stop"
assert any(event.data.get("delta", {}).get("text") == "ok" for event in events)
else:
chunks = [
json.loads(event.removeprefix("data: ").strip())
for event in events
if event.strip() != "data: [DONE]"
]
assert all("error" not in chunk for chunk in chunks)
assert events[-1].strip() == "data: [DONE]"
assert any(
choice.get("delta", {}).get("content") == "ok"
for chunk in chunks
for choice in chunk.get("choices", [])
)
else:
method = (
backend.send_message
if wire_format == "anthropic"
else backend.send_openai_message
)
assert (await method(body, {})).status_code == 200
finally:
await backend.close()
server.shutdown()
worker.join(timeout=5)
assert len(received) == 1
assert received[0]["service_tier"] == "flex"
assert received[0].get("stream", False) is streaming
assert received[0]["messages"] == body["messages"]