mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(backends): forward the standard service_tier param (litellm + anyllm) (#2468)
## Description
`service_tier` is a standard provider request field (OpenAI
`flex`/`default`/`priority`, Anthropic `auto`/`standard_only`), but the
SDK-translation backends rebuilt the upstream request from a fixed field
whitelist and **silently dropped** it — so a client that set
`service_tier` had it discarded on `--backend litellm`/`anyllm`. The
direct passthrough already forwards it; this makes the translation
backends do the same, alongside `temperature`/`top_p`/`tools`.
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
## Changes Made
- `backends/litellm.py` (Anthropic path): forward `service_tier` when
present (the OpenAI path already forwards unconsumed fields via
`_build_openai_extra_body`).
- `backends/anyllm.py`: add `service_tier` to the forwarded params (both
the Anthropic whitelist and the OpenAI param list).
## Testing
- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check`)
- [x] Type checking passes (`mypy`)
- [x] New test added
### Test Output
```text
$ python -m pytest tests/test_backend_anyllm.py -q
16 passed in 2.51s
$ ruff check headroom/backends/litellm.py headroom/backends/anyllm.py
All checks passed!
$ mypy headroom/backends/anyllm.py headroom/backends/litellm.py # 0 new errors
```
## Real Behavior Proof
- Environment: local, FakeAnyLLM instance capturing `acompletion`
kwargs.
- Steps: `send_message({..., "service_tier": "standard_only"})`.
- Observed: `instance.calls[0]["service_tier"] == "standard_only"` — the
param reaches the SDK call (previously absent).
- Not tested: live provider round-trip (no keys in CI); relies on the
SDK forwarding standard params.
## Checklist
- [x] Self-reviewed
- [x] Tests added and passing
- [x] I did **not** edit `CHANGELOG.md`
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/2468"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
---------
Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JerrettDavis
parent
7a19803bc2
commit
62cc38a6b0
@@ -258,6 +258,8 @@ class AnyLLMBackend(Backend):
|
||||
kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]]
|
||||
if "tool_choice" in body:
|
||||
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
|
||||
if "service_tier" in body:
|
||||
kwargs["service_tier"] = body["service_tier"]
|
||||
|
||||
logger.debug(f"any-llm request: provider={self.provider}, model={original_model}")
|
||||
|
||||
@@ -308,6 +310,8 @@ class AnyLLMBackend(Backend):
|
||||
kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]]
|
||||
if "tool_choice" in body:
|
||||
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
|
||||
if "service_tier" in body:
|
||||
kwargs["service_tier"] = body["service_tier"]
|
||||
|
||||
msg_id = f"msg_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
@@ -494,6 +498,7 @@ class AnyLLMBackend(Backend):
|
||||
"response_format",
|
||||
"seed",
|
||||
"n",
|
||||
"service_tier",
|
||||
]:
|
||||
if param in body:
|
||||
kwargs[param] = body[param]
|
||||
@@ -615,6 +620,7 @@ class AnyLLMBackend(Backend):
|
||||
"response_format",
|
||||
"seed",
|
||||
"n",
|
||||
"service_tier",
|
||||
]:
|
||||
if param in body:
|
||||
kwargs[param] = body[param]
|
||||
|
||||
@@ -47,6 +47,7 @@ _OPENAI_STANDARD_PARAMS = (
|
||||
# reasoning request routed cross-protocol to a Claude model fails.
|
||||
"reasoning_effort",
|
||||
"max_completion_tokens",
|
||||
"service_tier",
|
||||
)
|
||||
|
||||
_OPENAI_CONSUMED_BODY_KEYS = frozenset(
|
||||
@@ -1302,6 +1303,9 @@ class LiteLLMBackend(Backend):
|
||||
kwargs["tools"] = [_convert_anthropic_tool(t) for t in tools_in]
|
||||
if "tool_choice" in body:
|
||||
kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"])
|
||||
# Standard provider param — forward it instead of silently dropping.
|
||||
if "service_tier" in body:
|
||||
kwargs["service_tier"] = body["service_tier"]
|
||||
|
||||
# System prompt (Anthropic puts it in body, OpenAI in messages)
|
||||
if "system" in body:
|
||||
@@ -1404,6 +1408,8 @@ class LiteLLMBackend(Backend):
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
if "service_tier" in body:
|
||||
kwargs["service_tier"] = body["service_tier"]
|
||||
if "max_tokens" in body:
|
||||
kwargs["max_tokens"] = body["max_tokens"]
|
||||
if "temperature" in body:
|
||||
|
||||
@@ -761,3 +761,37 @@ async def test_close_is_noop(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
backend, _instance = make_backend(monkeypatch)
|
||||
assert await backend.close() is None
|
||||
assert isinstance(StreamEvent(event_type="message_start", data={}), StreamEvent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_send_message_forwards_service_tier(
|
||||
monkeypatch: pytest.MonkeyPatch, wire_format: str, streaming: bool
|
||||
) -> None:
|
||||
"""service_tier is a standard provider param — forward it, don't drop it."""
|
||||
backend, instance = make_backend(monkeypatch)
|
||||
instance.response = (
|
||||
FakeAsyncStream([])
|
||||
if streaming
|
||||
else make_response(
|
||||
make_choice("done", "stop"),
|
||||
usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2),
|
||||
)
|
||||
)
|
||||
body = {
|
||||
"model": "claude-3-7-sonnet",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
|
||||
"max_tokens": 16,
|
||||
"service_tier": "standard_only",
|
||||
}
|
||||
if streaming:
|
||||
method = (
|
||||
backend.stream_message if wire_format == "anthropic" else backend.stream_openai_message
|
||||
)
|
||||
events = [event async for event in method(body, {})]
|
||||
assert events
|
||||
else:
|
||||
method = backend.send_message if wire_format == "anthropic" else backend.send_openai_message
|
||||
assert (await method(body, {})).status_code == 200
|
||||
assert instance.calls[0]["service_tier"] == "standard_only"
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Both Anthropic translation paths forward the caller's service tier unchanged."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests._dotenv import importorskip_no_env_leak
|
||||
|
||||
litellm = importorskip_no_env_leak("litellm")
|
||||
|
||||
from headroom.backends import litellm as backend_module # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
@pytest.mark.parametrize("tier", ["absent", None, "default", "flex", "priority"])
|
||||
async def test_anthropic_service_tier_is_forwarded_in_both_paths(
|
||||
monkeypatch, streaming, tier, wire_format
|
||||
) -> None:
|
||||
backend = backend_module.LiteLLMBackend(provider="openrouter")
|
||||
body = {"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hello"}]}
|
||||
if tier != "absent":
|
||||
body["service_tier"] = tier
|
||||
original = deepcopy(body)
|
||||
|
||||
async def chunks():
|
||||
for chunk in []:
|
||||
yield chunk
|
||||
|
||||
response = (
|
||||
chunks()
|
||||
if streaming
|
||||
else litellm.ModelResponse(
|
||||
model="claude-sonnet-4-5",
|
||||
choices=[{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
usage={"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
)
|
||||
)
|
||||
completion = AsyncMock(return_value=response)
|
||||
monkeypatch.setattr(backend_module, "acompletion", completion)
|
||||
if streaming:
|
||||
method = (
|
||||
backend.stream_message if wire_format == "anthropic" else backend.stream_openai_message
|
||||
)
|
||||
events = [event async for event in method(body, {})]
|
||||
assert events
|
||||
if wire_format == "anthropic":
|
||||
assert events[-1].event_type == "message_stop"
|
||||
else:
|
||||
method = backend.send_message if wire_format == "anthropic" else backend.send_openai_message
|
||||
assert (await method(body, {})).status_code == 200
|
||||
completion.assert_awaited_once()
|
||||
kwargs = completion.call_args.kwargs
|
||||
if tier == "absent":
|
||||
assert "service_tier" not in kwargs
|
||||
else:
|
||||
assert kwargs["service_tier"] == tier
|
||||
assert "service_tier" not in kwargs.get("extra_body", {})
|
||||
assert body == original
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Exercise real SDK serialization against a local OpenAI-compatible HTTP endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
import pytest
|
||||
|
||||
from tests._dotenv import importorskip_no_env_leak
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sdk", ["litellm", "anyllm"])
|
||||
@pytest.mark.parametrize("wire_format", ["anthropic", "openai"])
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_real_sdk_puts_service_tier_on_the_provider_request(
|
||||
sdk, wire_format, streaming, monkeypatch
|
||||
) -> None:
|
||||
sdk_module = importorskip_no_env_leak("litellm" if sdk == "litellm" else "any_llm")
|
||||
monkeypatch.delenv("HEADROOM_OFFLINE", raising=False)
|
||||
received = []
|
||||
|
||||
class Provider(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
received.append(body)
|
||||
response = {
|
||||
"id": "chatcmpl-local",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
if body.get("stream"):
|
||||
response["object"] = "chat.completion.chunk"
|
||||
response["choices"][0]["delta"] = response["choices"][0].pop("message")
|
||||
payload = ("data: " + json.dumps(response) + "\n\ndata: [DONE]\n\n").encode()
|
||||
media = "text/event-stream"
|
||||
else:
|
||||
payload = json.dumps(response).encode()
|
||||
media = "application/json"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", media)
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
with HTTPServer(("127.0.0.1", 0), Provider) as server:
|
||||
worker = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
worker.start()
|
||||
url = f"http://127.0.0.1:{server.server_port}/v1"
|
||||
if sdk == "litellm":
|
||||
from headroom.backends.litellm import LiteLLMBackend
|
||||
|
||||
monkeypatch.setattr(sdk_module, "api_base", url)
|
||||
monkeypatch.setattr(sdk_module, "api_key", "local-fake-key")
|
||||
backend = LiteLLMBackend(provider="openai")
|
||||
else:
|
||||
from headroom.backends.anyllm import AnyLLMBackend
|
||||
|
||||
backend = AnyLLMBackend(provider="openai", api_base=url, api_key="local-fake-key")
|
||||
body = {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 8,
|
||||
"service_tier": "flex",
|
||||
}
|
||||
try:
|
||||
if streaming:
|
||||
method = (
|
||||
backend.stream_message
|
||||
if wire_format == "anthropic"
|
||||
else backend.stream_openai_message
|
||||
)
|
||||
events = [event async for event in method(body, {})]
|
||||
assert events
|
||||
if wire_format == "anthropic":
|
||||
assert all(event.event_type != "error" for event in events)
|
||||
assert events[-1].event_type == "message_stop"
|
||||
assert any(event.data.get("delta", {}).get("text") == "ok" for event in events)
|
||||
else:
|
||||
chunks = [
|
||||
json.loads(event.removeprefix("data: ").strip())
|
||||
for event in events
|
||||
if event.strip() != "data: [DONE]"
|
||||
]
|
||||
assert all("error" not in chunk for chunk in chunks)
|
||||
assert events[-1].strip() == "data: [DONE]"
|
||||
assert any(
|
||||
choice.get("delta", {}).get("content") == "ok"
|
||||
for chunk in chunks
|
||||
for choice in chunk.get("choices", [])
|
||||
)
|
||||
else:
|
||||
method = (
|
||||
backend.send_message
|
||||
if wire_format == "anthropic"
|
||||
else backend.send_openai_message
|
||||
)
|
||||
assert (await method(body, {})).status_code == 200
|
||||
finally:
|
||||
await backend.close()
|
||||
server.shutdown()
|
||||
worker.join(timeout=5)
|
||||
assert len(received) == 1
|
||||
assert received[0]["service_tier"] == "flex"
|
||||
assert received[0].get("stream", False) is streaming
|
||||
assert received[0]["messages"] == body["messages"]
|
||||
Reference in New Issue
Block a user