mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 18:41:07 +08:00
## Description DeepSeek restructured its model line-up and price card, and Headroom's DeepSeek data was last verified 2026-06-19: the rows still carried the pre-restructure flat rates, `deepseek-flash` was unknown, and nothing could express the vendor's peak/off-peak split. Source (fetched and re-verified today): https://api-docs.deepseek.com/quick_start/pricing | | `deepseek-flash` (V4.1-Flash) | `deepseek-v4-pro` (V4-Pro-0813) | |---|---|---| | Context / max output | 1M / 384K | 1M / 384K | | Input (cache hit) | $0.003 off-peak / $0.006 peak | $0.022 / $0.044 | | Input (cache miss) | $0.15 / $0.30 | $0.66 / $1.32 | | Output | $0.60 / $1.20 | $1.98 / $3.96 | | Vision | yes | no | `deepseek-v4-flash` and `deepseek-v4-flash-vision-exp` are retired ids still accepted on the wire — served by V4.1-Flash and billed at the Flash price. Peak is exactly 2x off-peak; peak hours are 01:00-04:00 and 06:00-10:00 UTC, Monday-Friday. Context note: this surfaced while working on #3036, which routes DeepSeek traffic through the proxy and hard-codes the `deepseek-v4-*` ids in its routing tests. That branch is not touched here, and no issue is closed by this PR — it is a rate-card and model-data refresh. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Added `headroom/pricing/deepseek_tiers.py` as the single source of truth: the off-peak `(cache-hit, cache-miss, output)` table for `deepseek-flash` and `deepseek-v4-pro`, peak derived as exactly 2x, the retired-id aliases, the Beijing peak-window predicate with the weekend rule gated on its effective instant, and the `rates_for(model, now=None)` seam. - Made two cost seams tier-aware: `estimate_cost_from_tokens(..., now=None)` prices DeepSeek flash/pro from the tier selected by the request instant (litellm's `model_cost` holds one flat rate per model and cannot express a tier), and `PricingRegistry.estimate_cost(..., now=None)` prices them through its existing body from a synthesized rate row so there is one arithmetic body rather than two. - Gave `AnthropicProvider.estimate_cost` the same `now` passthrough and derived its DeepSeek fallback table from the tier module. - Derived every flat table from the tier module so no DeepSeek rate literal is left outside it: the `deepseek_prices` rows, the `litellm.model_cost` injection (now overriding and merging upstream rather than filling absent keys), and the Anthropic fallback table. Moved `LAST_UPDATED` / `SOURCE_URL` onto the module that owns the rates and re-exported them so existing import paths keep working. - Updated the model data: `deepseek-flash` registered with vision and both retired ids as aliases, provider context rows, the Hugging Face tokenizer map, the vendored LiteLLM snapshot, and the OpenCode guide's published rate card. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ uv run pytest tests/test_providers tests/test_pricing.py tests/test_pricing_deepseek_tiers.py \ tests/test_pricing_from_litellm.py tests/test_pricing_litellm.py \ tests/test_pricing_litellm_model_resolution.py tests/test_models.py \ tests/test_huggingface_tokenizer_timeout.py tests/test_tokenizer_count_offload.py \ tests/test_cost_pricing_warning_dedup.py -q 291 passed, 3 warnings in 9.80s $ uv run ruff check . All checks passed! $ uv run mypy headroom headroom\integrations\langchain\chat_model.py:394: error: Function "RunnableBinding" could always be true in boolean context [truthy-function] Found 1 error in 1 file (checked 532 source files) $ cargo test -p headroom-proxy --lib compression::model_limits running 6 tests test compression::model_limits::tests::vendored_json_parses_at_runtime ... ok test compression::model_limits::tests::unknown_model_returns_default ... ok test compression::model_limits::tests::sample_spec_entry_is_excluded ... ok test compression::model_limits::tests::current_claude_models_present ... ok test compression::model_limits::tests::current_gpt_models_present ... ok test compression::model_limits::tests::empty_or_garbage_string_does_not_panic ... ok test result: ok. 6 passed; 0 failed $ uv run pytest tests/test_pricing_deepseek_tiers.py -q # with litellm hidden 47 passed $ uv run mypy headroom/pricing/deepseek_tiers.py headroom/pricing/registry.py \ headroom/pricing/litellm_pricing.py headroom/providers/anthropic.py Success: no issues found in 4 source files ``` The `mypy headroom` box is unticked on purpose: the single error above is pre-existing on `main` in `headroom/integrations/langchain/chat_model.py:394`, a file this PR does not touch (see Additional Notes). ## Real Behavior Proof - Environment: Windows 10, Python 3.13 in the repo's `.venv`, `uv`; litellm 1.88.1 installed (whose bundled cost map predates these ids, so the injected rows are what price them). - Exact command / steps: pinned instants Monday 2026-08-17 02:00 UTC (10:00 Beijing = peak) and 12:00 UTC (20:00 Beijing = off-peak), then: `uv run python -c "from datetime import datetime,timezone; from headroom.pricing.litellm_pricing import estimate_cost_from_tokens as e; P=datetime(2026,8,17,2,tzinfo=timezone.utc); O=datetime(2026,8,17,12,tzinfo=timezone.utc); print(e('deepseek-flash',10**6,10**6,now=O), e('deepseek-flash',10**6,10**6,now=P), e('deepseek-flash',10**6,0,cached_tokens=10**6,now=O), e('deepseek-v4-flash-vision-exp',10**6,10**6,now=O))"` - Observed result: `0.75` (off-peak flash), `1.5` (peak flash, exactly 2x), `0.003` (fully cached prompt billed once at the cache-hit rate), `0.75` (retired id prices as Flash). Also verified: 9 mutants of the new logic (peak multiplier, weekend gate, window start, alias resolution, cached-token handling, registry branch, provider passthrough) were each caught by the new tests, and `tests/test_pricing_deepseek_tiers.py` still passes with litellm absent (47 passed). - Not tested: the full test suite (hours on this box, and much of it cannot build a `TestClient` here because `NO_PROXY` contains a bare `::1`, so `httpx.URLPattern` raises `InvalidURL: Invalid port: ':'` — a pre-existing environment fault, confirmed by stashing this branch and re-running); CNY billing; the proxy budget guard and savings ledger, which remain flat by design (see Additional Notes). ## Runtime Rollout Safety - Rollout-managed feature(s): DeepSeek pricing and model data consumed by the pricing modules, providers, model registry and the Rust proxy's context limits. No feature flags are involved. - Minimum rollout channel: Stable. - Stable/default behavior changed: Yes — DeepSeek costs, model ids and capabilities change; the litellm `model_cost` DeepSeek rows now override upstream; and one context-limit fallback now resolves longest-prefix-first. All are itemised in Additional Notes. - Kill switch / disable path: revert this branch (no flag). Operators can override context limits without a release via `HEADROOM_MODEL_LIMITS` or `~/.headroom/models.json`, and provider pricing via the existing custom pricing config. - Unsafe override required: No. - Qualification impact: None — no schema, wire-format or session-format change; the vendored JSON keeps its upstream shape. - Rollback path: revert the merge commit; the only persisted artifact change is the vendored JSON, which reverts with it. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Screenshots (if applicable) Not applicable — no UI change. ## Additional Notes **Behaviour changes worth a reviewer's attention** 1. Flat figures are the off-peak tier by design; only the two cost seams are tier-aware. The proxy budget guard (`proxy/cost.py`), the savings ledger, `ModelRegistry.estimate_cost` and `OpenAICompatibleProvider.estimate_cost` remain flat or unpriced, so a peak-hour DeepSeek request can be under-estimated by up to 2x there. Making those tier-aware needs the request instant plumbed to each call site — deliberately a separate change. 2. The litellm DeepSeek rows now override upstream instead of filling only absent keys: upstream ships `deepseek-flash` at the *peak* rate, so fill-if-absent would have made the flat figure depend on the installed litellm version. The injection also merges (`{**upstream, **ours}`) so capability flags survive. 3. DeepSeek caps moved `384_000` -> `393_216` in the litellm-shaped tables (upstream's 384x1024 encoding of the published 384K). `ModelInfo` keeps the published `384_000`; the litellm-shaped tables are authoritative wherever litellm is consulted. 4. `OpenAICompatibleProvider`'s context fallback now matches the longest prefix first, mirroring the long-standing behaviour in `OpenAIProvider`. This fixes `deepseek-v4-flash-vision-exp` (was 1_048_576 via the shorter `deepseek-v4` key, now 1_000_000) and also changes the window for unlisted non-DeepSeek variants, e.g. `llama-3.1-8b-instruct` 8192 -> 128000, `mistral-large-2407` 32768 -> 128000, `starcoder2-15b` 8192 -> 16384. 5. `PricingRegistry.estimate_cost` no longer raises for a DeepSeek flash/pro id absent from `prices` — the tier table is authoritative for those four ids. Unknown non-DeepSeek ids still raise, and a flat row for a tiered id is deliberately ignored. 6. Weekend off-peak applies only from 2026-08-22T16:00Z, so a back-dated or replayed weekend request before that instant keeps the weekday windows rather than being silently repriced. `now` exists only on `AnthropicProvider.estimate_cost`; every other provider reads the wall clock for live traffic. 7. CNY billing is not modelled. The zh-cn page lists the same tiers in CNY, but the two lists are not one FX rate apart; supporting CNY-billed accounts needs a switch over two published lists, not a baked constant. **N/A / notes on checklist items** - `Type checking passes (mypy headroom)` is unticked for the pre-existing error above. `git diff --name-only origin/main..HEAD` contains no `integrations/langchain` path, so this PR does not introduce or touch it; `mypy` is clean on all four changed modules. - Out of scope: the stale `deepseek-v3` / `deepseek-coder` registry rows and the old `deepseek-chat` / `deepseek-reasoner` / `deepseek-v3.2` context rows (the vendor no longer lists them; pruning them is a behavioural change of its own). - The design and implementation plan for this change live in a separate workspace, so the decisions they record are inlined above rather than linked. --------- Co-authored-by: Zenjibad <316719033+Zenjibad@users.noreply.github.com> Co-authored-by: DSH Agent <agent@local>
525 lines
20 KiB
Python
525 lines
20 KiB
Python
"""Tests for universal provider support.
|
|
|
|
Tests OpenAICompatibleProvider, OpenAIProvider, GoogleProvider, and
|
|
LiteLLMProvider.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from headroom.providers import (
|
|
GoogleProvider,
|
|
LiteLLMProvider,
|
|
ModelCapabilities,
|
|
OpenAICompatibleProvider,
|
|
OpenAIProvider,
|
|
create_anyscale_provider,
|
|
create_fireworks_provider,
|
|
create_groq_provider,
|
|
create_litellm_provider,
|
|
create_lmstudio_provider,
|
|
create_ollama_provider,
|
|
create_together_provider,
|
|
create_vllm_provider,
|
|
is_litellm_available,
|
|
)
|
|
|
|
|
|
def _transformers_available() -> bool:
|
|
"""Check if transformers is available."""
|
|
try:
|
|
import transformers # noqa: F401
|
|
|
|
return True
|
|
except ImportError:
|
|
return False
|
|
|
|
|
|
class TestOpenAICompatibleProvider:
|
|
"""Tests for OpenAICompatibleProvider."""
|
|
|
|
def test_init_default(self):
|
|
"""Test initialization with defaults."""
|
|
provider = OpenAICompatibleProvider()
|
|
assert provider.name == "openai_compatible"
|
|
assert provider.base_url is None
|
|
|
|
def test_init_with_config(self):
|
|
"""Test initialization with configuration."""
|
|
provider = OpenAICompatibleProvider(
|
|
name="custom",
|
|
base_url="http://localhost:8080/v1",
|
|
api_key="test-key",
|
|
)
|
|
assert provider.name == "custom"
|
|
assert provider.base_url == "http://localhost:8080/v1"
|
|
assert provider.api_key == "test-key"
|
|
|
|
def test_supports_any_model(self):
|
|
"""Test that provider supports any model."""
|
|
provider = OpenAICompatibleProvider()
|
|
assert provider.supports_model("any-model") is True
|
|
assert provider.supports_model("llama-3") is True
|
|
assert provider.supports_model("custom-finetuned") is True
|
|
|
|
@pytest.mark.skipif(
|
|
not _transformers_available(),
|
|
reason="transformers not installed - needed for HuggingFace tokenizer",
|
|
)
|
|
def test_get_token_counter(self):
|
|
"""Test getting token counter."""
|
|
provider = OpenAICompatibleProvider()
|
|
counter = provider.get_token_counter("llama-3-8b")
|
|
assert counter is not None
|
|
# Should be able to count tokens
|
|
count = counter.count_text("Hello, world!")
|
|
assert count > 0
|
|
|
|
def test_get_context_limit_known_model(self):
|
|
"""Test context limit for known models."""
|
|
provider = OpenAICompatibleProvider()
|
|
# Llama 3.1 has 128K context
|
|
limit = provider.get_context_limit("llama-3.1-8b")
|
|
assert limit == 128000
|
|
|
|
def test_get_context_limit_deepseek_v3_is_1m(self):
|
|
"""DeepSeek V3/V4 support 1M context, not 128K (#1038)."""
|
|
provider = OpenAICompatibleProvider()
|
|
assert provider.get_context_limit("deepseek-v3") == 1048576
|
|
assert provider.get_context_limit("deepseek-v4") == 1048576
|
|
assert provider.get_context_limit("deepseek") == 1048576
|
|
assert provider.get_context_limit("deepseek-v2") == 128000
|
|
assert provider.get_context_limit("deepseek-v3.2") == 128000
|
|
assert provider.get_context_limit("deepseek-v4-pro") == 1_000_000
|
|
# All three flash-family ids — the current one plus the two retired
|
|
# aliases DeepSeek still accepts — are served by V4.1-Flash at 1M on
|
|
# both providers that carry a DeepSeek row.
|
|
for p in (provider, OpenAIProvider()):
|
|
assert p.get_context_limit("deepseek-flash") == 1_000_000
|
|
assert p.get_context_limit("deepseek-v4-flash") == 1_000_000
|
|
assert p.get_context_limit("deepseek-v4-flash-vision-exp") == 1_000_000
|
|
assert provider.get_context_limit("deepseek-r1") == 131072
|
|
assert provider.get_context_limit("deepseek-coder-v2") == 128000
|
|
|
|
def test_get_context_limit_unknown_model(self):
|
|
"""Test context limit for unknown models (defaults to 128K)."""
|
|
provider = OpenAICompatibleProvider()
|
|
limit = provider.get_context_limit("unknown-model")
|
|
assert limit == 128000
|
|
|
|
def test_register_model(self):
|
|
"""Test registering a custom model."""
|
|
provider = OpenAICompatibleProvider()
|
|
provider.register_model(
|
|
"my-model",
|
|
context_window=64000,
|
|
max_output_tokens=8192,
|
|
input_cost_per_1m=1.0,
|
|
output_cost_per_1m=2.0,
|
|
)
|
|
assert provider.get_context_limit("my-model") == 64000
|
|
|
|
def test_estimate_cost_registered_model(self):
|
|
"""Test cost estimation for registered model."""
|
|
provider = OpenAICompatibleProvider()
|
|
provider.register_model(
|
|
"priced-model",
|
|
input_cost_per_1m=1.0,
|
|
output_cost_per_1m=2.0,
|
|
)
|
|
cost = provider.estimate_cost(
|
|
input_tokens=1000000,
|
|
output_tokens=500000,
|
|
model="priced-model",
|
|
)
|
|
assert cost == 2.0 # 1.0 + 1.0
|
|
|
|
def test_estimate_cost_unknown_model(self):
|
|
"""Test cost estimation returns None for unknown model."""
|
|
provider = OpenAICompatibleProvider()
|
|
cost = provider.estimate_cost(
|
|
input_tokens=1000,
|
|
output_tokens=500,
|
|
model="unknown-model",
|
|
)
|
|
assert cost is None
|
|
|
|
def test_register_model_accepts_capabilities_object(self):
|
|
provider = OpenAICompatibleProvider()
|
|
caps = ModelCapabilities(model="caps-model", context_window=16000, tokenizer_backend="test")
|
|
|
|
provider.register_model("caps-model", capabilities=caps)
|
|
|
|
assert provider.get_context_limit("caps-model") == 16000
|
|
|
|
def test_get_token_counter_uses_registered_tokenizer_backend(self, monkeypatch):
|
|
recorded: list[tuple[str, str | None]] = []
|
|
|
|
class DummyTokenizer:
|
|
def count_text(self, text: str) -> int:
|
|
return len(text.split())
|
|
|
|
monkeypatch.setattr(
|
|
"headroom.providers.openai_compatible.get_tokenizer",
|
|
lambda model, backend=None: recorded.append((model, backend)) or DummyTokenizer(),
|
|
)
|
|
provider = OpenAICompatibleProvider(
|
|
models={
|
|
"custom-model": ModelCapabilities(
|
|
model="custom-model",
|
|
tokenizer_backend="custom-backend",
|
|
)
|
|
}
|
|
)
|
|
|
|
counter = provider.get_token_counter("custom-model")
|
|
|
|
assert counter.count_text("one two three") == 3
|
|
assert recorded == [("custom-model", "custom-backend")]
|
|
|
|
def test_openai_compatible_token_counter_counts_message_parts(self, monkeypatch):
|
|
class DummyTokenizer:
|
|
def count_text(self, text: str) -> int:
|
|
return len(text)
|
|
|
|
monkeypatch.setattr(
|
|
"headroom.providers.openai_compatible.get_tokenizer",
|
|
lambda model, backend=None: DummyTokenizer(),
|
|
)
|
|
counter = OpenAICompatibleProvider().get_token_counter("demo-model")
|
|
|
|
tokens = counter.count_message(
|
|
{
|
|
"role": "user",
|
|
"content": [{"type": "text", "text": "hi"}, "there"],
|
|
"name": "tester",
|
|
"tool_calls": [{"function": {"name": "lookup", "arguments": '{"x":1}'}}],
|
|
"tool_call_id": "call_123",
|
|
}
|
|
)
|
|
total = counter.count_messages(
|
|
[
|
|
{"role": "user", "content": "hello"},
|
|
{"role": "assistant", "content": ["world"]},
|
|
]
|
|
)
|
|
|
|
assert tokens == 55
|
|
assert total == 34
|
|
|
|
def test_openai_compatible_token_counter_prices_declared_media(self, monkeypatch):
|
|
"""An image block costs tokens; a non dict/str part still contributes none.
|
|
|
|
This previously asserted that BOTH contribute 0 — i.e. it pinned the
|
|
defect. The counter handled only ``type == "text"``, so every other block
|
|
priced at ~0: measured on a 6,800-char block, tool_result / thinking /
|
|
document / mcp_tool_result all returned 8 tokens, overhead only. Counters
|
|
now delegate to the shared walker, which prices a declared image with the
|
|
pixel-based estimate (1600, the max after provider auto-resize) rather
|
|
than either ignoring it or serializing its base64 as text.
|
|
"""
|
|
|
|
class DummyTokenizer:
|
|
def count_text(self, text: str) -> int:
|
|
return len(text)
|
|
|
|
monkeypatch.setattr(
|
|
"headroom.providers.openai_compatible.get_tokenizer",
|
|
lambda model, backend=None: DummyTokenizer(),
|
|
)
|
|
counter = OpenAICompatibleProvider().get_token_counter("demo-model")
|
|
|
|
# Non-list, non-str content is still ignored.
|
|
assert counter.count_message({"role": "user", "content": {}}) == 8
|
|
# A bare int is not a block and still contributes nothing.
|
|
assert counter.count_message({"role": "user", "content": [123]}) == 8
|
|
# A declared image is now priced instead of silently free.
|
|
assert counter.count_message({"role": "user", "content": [{"type": "image"}, 123]}) == 1608
|
|
|
|
def test_get_context_limit_prefix_output_buffer_and_partial_pricing(self):
|
|
provider = OpenAICompatibleProvider(
|
|
models={
|
|
"buffered": ModelCapabilities(
|
|
model="buffered",
|
|
max_output_tokens=1200,
|
|
input_cost_per_1m=1.0,
|
|
)
|
|
}
|
|
)
|
|
|
|
assert provider.get_context_limit("mistral-custom") == 32768
|
|
assert provider.get_output_buffer("buffered", default=4000) == 1200
|
|
assert provider.get_output_buffer("unknown", default=2222) == 2222
|
|
assert provider.estimate_cost(1000, 1000, "buffered") is None
|
|
|
|
|
|
class TestModelCapabilities:
|
|
"""Tests for ModelCapabilities dataclass."""
|
|
|
|
def test_default_values(self):
|
|
"""Test default capability values."""
|
|
caps = ModelCapabilities(model="test-model")
|
|
assert caps.context_window == 128000
|
|
assert caps.max_output_tokens == 4096
|
|
assert caps.supports_tools is True
|
|
assert caps.supports_vision is False
|
|
assert caps.supports_streaming is True
|
|
|
|
def test_custom_values(self):
|
|
"""Test custom capability values."""
|
|
caps = ModelCapabilities(
|
|
model="custom-model",
|
|
context_window=32000,
|
|
max_output_tokens=16384,
|
|
supports_tools=False,
|
|
supports_vision=True,
|
|
input_cost_per_1m=0.5,
|
|
output_cost_per_1m=1.5,
|
|
)
|
|
assert caps.context_window == 32000
|
|
assert caps.max_output_tokens == 16384
|
|
assert caps.supports_tools is False
|
|
assert caps.supports_vision is True
|
|
assert caps.input_cost_per_1m == 0.5
|
|
assert caps.output_cost_per_1m == 1.5
|
|
|
|
|
|
class TestGoogleProvider:
|
|
"""Tests for GoogleProvider."""
|
|
|
|
@pytest.fixture
|
|
def provider(self):
|
|
"""Create Google provider."""
|
|
return GoogleProvider()
|
|
|
|
def test_name(self, provider):
|
|
"""Test provider name."""
|
|
assert provider.name == "google"
|
|
|
|
def test_supports_gemini_models(self, provider):
|
|
"""Test support for Gemini models."""
|
|
assert provider.supports_model("gemini-2.0-flash") is True
|
|
assert provider.supports_model("gemini-1.5-pro") is True
|
|
assert provider.supports_model("gemini-1.5-flash") is True
|
|
|
|
def test_not_supports_other_models(self, provider):
|
|
"""Test non-support for other models."""
|
|
assert provider.supports_model("gpt-4o") is False
|
|
assert provider.supports_model("claude-3") is False
|
|
|
|
def test_get_token_counter(self, provider):
|
|
"""Test getting token counter."""
|
|
counter = provider.get_token_counter("gemini-2.0-flash")
|
|
assert counter is not None
|
|
count = counter.count_text("Hello, world!")
|
|
assert count > 0
|
|
|
|
def test_get_context_limit_gemini_2(self, provider):
|
|
"""Test context limit for Gemini 2.0."""
|
|
limit = provider.get_context_limit("gemini-2.0-flash")
|
|
# LiteLLM returns 1048576 (2^20), fallback returns 1000000
|
|
assert limit in (1000000, 1048576) # ~1M tokens
|
|
|
|
def test_get_context_limit_gemini_1_5_pro(self, provider):
|
|
"""Test context limit for Gemini 1.5 Pro (2M!)."""
|
|
limit = provider.get_context_limit("gemini-1.5-pro")
|
|
# LiteLLM returns 2097152 (2^21), fallback returns 2000000
|
|
assert limit in (2000000, 2097152) # ~2M tokens!
|
|
|
|
def test_estimate_cost(self, provider):
|
|
"""Test cost estimation."""
|
|
cost = provider.estimate_cost(
|
|
input_tokens=1000000,
|
|
output_tokens=500000,
|
|
model="gemini-2.0-flash",
|
|
)
|
|
assert cost is not None
|
|
# 1M input * $0.10 + 0.5M output * $0.40 = $0.10 + $0.20 = $0.30
|
|
assert abs(cost - 0.30) < 0.01
|
|
|
|
def test_openai_compatible_url(self):
|
|
"""Test OpenAI-compatible URL."""
|
|
url = GoogleProvider.get_openai_compatible_url("test-key")
|
|
assert "generativelanguage.googleapis.com" in url
|
|
|
|
|
|
class TestProviderFactoryFunctions:
|
|
"""Tests for provider factory functions."""
|
|
|
|
def test_create_ollama_provider(self):
|
|
"""Test creating Ollama provider."""
|
|
provider = create_ollama_provider()
|
|
assert provider.name == "ollama"
|
|
assert provider.base_url == "http://localhost:11434/v1"
|
|
|
|
def test_create_ollama_provider_custom_url(self):
|
|
"""Test creating Ollama provider with custom URL."""
|
|
provider = create_ollama_provider("http://192.168.1.100:11434/v1")
|
|
assert provider.base_url == "http://192.168.1.100:11434/v1"
|
|
|
|
def test_create_together_provider(self):
|
|
"""Test creating Together provider."""
|
|
provider = create_together_provider()
|
|
assert provider.name == "together"
|
|
assert "together.xyz" in provider.base_url
|
|
|
|
def test_create_groq_provider(self):
|
|
"""Test creating Groq provider."""
|
|
provider = create_groq_provider()
|
|
assert provider.name == "groq"
|
|
assert "groq.com" in provider.base_url
|
|
|
|
def test_create_vllm_provider(self):
|
|
"""Test creating vLLM provider."""
|
|
provider = create_vllm_provider("http://localhost:8000/v1")
|
|
assert provider.name == "vllm"
|
|
assert provider.base_url == "http://localhost:8000/v1"
|
|
|
|
def test_create_lmstudio_provider(self):
|
|
"""Test creating LM Studio provider."""
|
|
provider = create_lmstudio_provider()
|
|
assert provider.name == "lmstudio"
|
|
assert provider.base_url == "http://localhost:1234/v1"
|
|
|
|
def test_create_fireworks_and_anyscale_providers(self):
|
|
fireworks = create_fireworks_provider(api_key="fireworks-key")
|
|
anyscale = create_anyscale_provider(api_key="anyscale-key")
|
|
|
|
assert fireworks.name == "fireworks"
|
|
assert fireworks.base_url == "https://api.fireworks.ai/inference/v1"
|
|
assert fireworks.api_key == "fireworks-key"
|
|
assert anyscale.name == "anyscale"
|
|
assert anyscale.base_url == "https://api.endpoints.anyscale.com/v1"
|
|
assert anyscale.api_key == "anyscale-key"
|
|
|
|
|
|
class TestLiteLLMProvider:
|
|
"""Tests for LiteLLM provider."""
|
|
|
|
def test_is_litellm_available(self):
|
|
"""Test checking LiteLLM availability."""
|
|
result = is_litellm_available()
|
|
assert isinstance(result, bool)
|
|
|
|
def test_unavailable_litellm_paths(self, monkeypatch):
|
|
import headroom.providers.litellm as litellm_module
|
|
|
|
monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", False)
|
|
|
|
assert litellm_module.is_litellm_available() is False
|
|
assert litellm_module.LiteLLMProvider.list_supported_providers() == []
|
|
with pytest.raises(RuntimeError, match="LiteLLM is required"):
|
|
litellm_module.LiteLLMTokenCounter("gpt-4o")
|
|
with pytest.raises(RuntimeError, match="LiteLLM is required"):
|
|
litellm_module.LiteLLMProvider()
|
|
|
|
def test_litellm_token_counter_fallback_paths(self, monkeypatch):
|
|
import headroom.providers.litellm as litellm_module
|
|
|
|
class DummyFallback:
|
|
def count_text(self, text: str) -> int:
|
|
return len(text.split())
|
|
|
|
monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
|
|
monkeypatch.setattr(
|
|
litellm_module,
|
|
"litellm_token_counter",
|
|
lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")),
|
|
)
|
|
monkeypatch.setattr(litellm_module, "EstimatingTokenCounter", DummyFallback)
|
|
|
|
counter = litellm_module.LiteLLMTokenCounter("gpt-4o")
|
|
|
|
assert counter.count_text("") == 0
|
|
assert counter.count_text("one two three") == 3
|
|
assert counter.count_message({"content": "one two"}) == 6
|
|
assert counter.count_messages([]) == 0
|
|
assert counter.count_messages([{"content": "one two"}, {"content": "three"}]) == 14
|
|
|
|
def test_litellm_provider_info_and_cost_fallbacks(self, monkeypatch):
|
|
import headroom.providers.litellm as litellm_module
|
|
|
|
monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
|
|
monkeypatch.setattr(
|
|
litellm_module,
|
|
"litellm_get_model_info",
|
|
lambda model: {
|
|
"ctx-model": {"max_input_tokens": 64000},
|
|
"max-model": {"max_tokens": 32000},
|
|
"none-model": {"max_input_tokens": None, "max_output_tokens": None},
|
|
"output-model": {"max_output_tokens": 6000},
|
|
}[model],
|
|
)
|
|
# Cost now resolves through the shared pricing helper rather than a
|
|
# direct `litellm.completion_cost` call, so patch that seam. The helper
|
|
# returns None (not an exception) for a model LiteLLM can't price.
|
|
monkeypatch.setattr(
|
|
litellm_module,
|
|
"estimate_cost_from_tokens",
|
|
lambda model, **kwargs: 1.23 if model == "priced-model" else None,
|
|
)
|
|
|
|
provider = litellm_module.LiteLLMProvider()
|
|
|
|
assert provider.get_context_limit("ctx-model") == 64000
|
|
assert provider.get_context_limit("max-model") == 32000
|
|
assert provider.get_context_limit("none-model") == 128000
|
|
assert provider.get_output_buffer("output-model", default=4000) == 4000
|
|
assert provider.get_output_buffer("none-model", default=2222) == 2222
|
|
assert provider.estimate_cost(1000, 1000, "priced-model") == 1.23
|
|
assert provider.estimate_cost(1000, 1000, "missing-price") is None
|
|
|
|
def test_litellm_provider_handles_info_exceptions_and_factory(self, monkeypatch):
|
|
import headroom.providers.litellm as litellm_module
|
|
|
|
monkeypatch.setattr(litellm_module, "LITELLM_AVAILABLE", True)
|
|
monkeypatch.setattr(
|
|
litellm_module,
|
|
"litellm_get_model_info",
|
|
lambda model: (_ for _ in ()).throw(RuntimeError("boom")),
|
|
)
|
|
|
|
provider = create_litellm_provider()
|
|
|
|
assert isinstance(provider, LiteLLMProvider)
|
|
assert provider.get_context_limit("gpt-4o") == 128000
|
|
assert provider.get_output_buffer("gpt-4o", default=3333) == 3333
|
|
|
|
@pytest.mark.skipif(
|
|
not is_litellm_available(),
|
|
reason="LiteLLM not installed",
|
|
)
|
|
def test_create_litellm_provider(self):
|
|
"""Test creating LiteLLM provider."""
|
|
from headroom.providers import create_litellm_provider
|
|
|
|
provider = create_litellm_provider()
|
|
assert provider.name == "litellm"
|
|
|
|
@pytest.mark.skipif(
|
|
not is_litellm_available(),
|
|
reason="LiteLLM not installed",
|
|
)
|
|
def test_litellm_supports_any_model(self):
|
|
"""Test LiteLLM supports any model."""
|
|
from headroom.providers import create_litellm_provider
|
|
|
|
provider = create_litellm_provider()
|
|
assert provider.supports_model("gpt-4o") is True
|
|
assert provider.supports_model("claude-3-sonnet") is True
|
|
assert provider.supports_model("any-model") is True
|
|
|
|
@pytest.mark.skipif(
|
|
not is_litellm_available(),
|
|
reason="LiteLLM not installed",
|
|
)
|
|
def test_litellm_list_providers(self):
|
|
"""Test listing LiteLLM providers."""
|
|
from headroom.providers import LiteLLMProvider
|
|
|
|
providers = LiteLLMProvider.list_supported_providers()
|
|
assert "openai" in providers
|
|
assert "anthropic" in providers
|
|
assert "ollama" in providers
|