mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description Make the compression-timeout regression independent of host speed. The old test required the entire HTTP request, including outcome bookkeeping, to finish within 500 ms. Hosted runs show the unchanged production path can exceed that limit after the simulated compression timeout has already occurred. ## Type of Change - [x] Test-only change; no production behavior change ## Changes Made - Assert that the compression executor is awaited exactly once. - Fail if timeout handling retries compression inline or recounts tokens. - Retain the fail-open response checks: HTTP 200, unchanged messages, zero token counts/savings, unchanged compression ratio, and compression-timeout metadata. - Replace the unrelated whole-request wall-clock assertion with those behavioral checks. - Reword Copilot's generated initial-plan commit to satisfy Conventional Commits; all file trees and contributor authorship were preserved. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - Local Windows/CPython 3.12: `G:/git/headroom/.venv/Scripts/python.exe -m pytest tests/test_platform_stabilization_functional.py -q --tb=short` — five passed, one existing Starlette deprecation warning, using the existing native extension. - Current head `14404162810068d2c4a82b8eeebaab1b313aa3da`: 32 successful hosted checks and 12 skips; no failures or running checks at qualification. - The test source is byte-identical to the locally tested original implementation commit; the subsequent changes only merge main and repair commit metadata. ## Real Behavior Proof - Environment: GitHub-hosted Linux CI plus the local Windows test environment above; no provider credentials or paid inference. - Exact command / steps: run the functional-test command in Testing, then inspect CI run 41104791110 on the current head, including all four Python shards. - Observed result: five local functional tests passed; the current head's hosted checks completed with 32 successes, 12 skips, and no failures. - Observed input to this fix: the prior timeout assertion failed at approximately 1.0–1.2 seconds against a 0.5-second limit on several otherwise unrelated PRs whose proxy implementation matched main. - After-fix evidence: the current full hosted test run succeeds with the deterministic timeout contract assertions in place. Local functional tests also pass. - Scope: this is an internal maintainer-coauthored test correction. The evidence establishes the regression-test contract; it is not a new live-provider or live-Claude session qualification. No production timeout, deadline, or bookkeeping implementation changed. - Not tested: live provider traffic or a fresh native rebuild. ## Runtime Rollout Safety - Rollout-managed feature: none; tests only. - Minimum rollout channel: not applicable; no runtime feature is added or enabled. - Stable/default behavior changed: no. - Kill switch/rollback: revert this test change; production behavior is unaffected. - Rollback path: revert the test-only commit while retaining production timeout handling. - Unsafe override required: no. - Qualification impact: replaces host-speed sensitivity with explicit fail-open behavior checks. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review Automated code qualification does not constitute final human merge approval. Nothing has been merged, tagged, or published. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: JerrettDavis <2610199+JerrettDavis@users.noreply.github.com> Co-authored-by: JD Davis <mxjerrett@gmail.com>
210 lines
7.9 KiB
Python
210 lines
7.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
|
|
pytest.importorskip("fastapi")
|
|
pytest.importorskip("headroom._core")
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from headroom.config import TransformResult
|
|
from headroom.proxy.server import ProxyConfig, create_app
|
|
from headroom.tokenizer import Tokenizer
|
|
|
|
|
|
def _proxy_config(**overrides: Any) -> ProxyConfig:
|
|
defaults: dict[str, Any] = {
|
|
"optimize": True,
|
|
"cache_enabled": False,
|
|
"rate_limit_enabled": False,
|
|
"cost_tracking_enabled": False,
|
|
"log_requests": False,
|
|
"ccr_inject_tool": False,
|
|
"ccr_handle_responses": False,
|
|
"ccr_context_tracking": False,
|
|
"image_optimize": False,
|
|
"disable_kompress": True,
|
|
"compression_max_workers": 1,
|
|
}
|
|
defaults.update(overrides)
|
|
return ProxyConfig(**defaults)
|
|
|
|
|
|
def test_proxy_health_surfaces_compression_runtime_metrics(monkeypatch) -> None:
|
|
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
|
app = create_app(_proxy_config(optimize=False))
|
|
|
|
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
|
|
live = client.get("/livez")
|
|
health = client.get("/health")
|
|
|
|
assert live.status_code == 200
|
|
assert live.json()["alive"] is True
|
|
assert health.status_code == 200
|
|
runtime = health.json()["runtime"]
|
|
assert runtime["compression_executor"]["max_workers"] == 1
|
|
assert runtime["compression_executor"]["queued"] == 0
|
|
assert runtime["compression_executor"]["queue_timeouts_total"] == 0
|
|
|
|
|
|
def test_v1_compress_success_reports_actual_metrics(monkeypatch) -> None:
|
|
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
|
app = create_app(_proxy_config())
|
|
proxy = app.state.proxy
|
|
request_messages = [{"role": "user", "content": "summarize this repeated payload"}]
|
|
compressed_messages = [{"role": "user", "content": "summary payload"}]
|
|
ccr_hash = "abc123def4567890abc123de"
|
|
|
|
def fake_apply(**kwargs):
|
|
assert kwargs["messages"] == request_messages
|
|
assert kwargs["model"] == "gpt-4o"
|
|
return TransformResult(
|
|
messages=compressed_messages,
|
|
tokens_before=100,
|
|
tokens_after=40,
|
|
transforms_applied=["test:compress"],
|
|
markers_inserted=[ccr_hash],
|
|
)
|
|
|
|
# The default /v1/compress mode runs a marker-free pipeline derived from
|
|
# `openai_pipeline`, not `openai_pipeline` itself, so patch the one the
|
|
# route actually uses. It is built eagerly at create_app() time.
|
|
monkeypatch.setattr(proxy._compress_pipeline_cache["no_ccr"], "apply", fake_apply)
|
|
|
|
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
|
|
response = client.post(
|
|
"/v1/compress",
|
|
json={"model": "gpt-4o", "messages": request_messages},
|
|
)
|
|
|
|
body = response.json()
|
|
assert response.status_code == 200
|
|
assert body["messages"] == compressed_messages
|
|
assert body["tokens_before"] == 100
|
|
assert body["tokens_after"] == 40
|
|
assert body["tokens_saved"] == 60
|
|
assert body["compression_ratio"] == 0.4
|
|
assert body["transforms_applied"] == ["test:compress"]
|
|
assert body["transforms_summary"] == {"test:compress": 1}
|
|
assert body["ccr_hashes"] == [ccr_hash]
|
|
|
|
|
|
def test_v1_compress_timeout_fails_open_without_retry_or_recount(monkeypatch) -> None:
|
|
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
|
app = create_app(_proxy_config())
|
|
proxy = app.state.proxy
|
|
request_messages = [{"role": "user", "content": "do not mutate me"}]
|
|
|
|
timeout_executor = AsyncMock(side_effect=TimeoutError("compression deadline exceeded"))
|
|
pipeline_apply = Mock(side_effect=AssertionError("compression must not retry inline"))
|
|
count_messages = Mock(side_effect=AssertionError("timeout must not recount tokens"))
|
|
monkeypatch.setattr(proxy, "_run_compression_in_executor", timeout_executor)
|
|
monkeypatch.setattr(proxy._compress_pipeline_cache["no_ccr"], "apply", pipeline_apply)
|
|
|
|
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
|
|
# Check the fail-open contract, not host speed: outcome bookkeeping
|
|
# after the timeout can include pricing and durable savings writes.
|
|
monkeypatch.setattr(Tokenizer, "count_messages", count_messages)
|
|
response = client.post(
|
|
"/v1/compress",
|
|
json={"model": "gpt-4o", "messages": request_messages},
|
|
)
|
|
|
|
body = response.json()
|
|
assert response.status_code == 200
|
|
timeout_executor.assert_awaited_once()
|
|
pipeline_apply.assert_not_called()
|
|
count_messages.assert_not_called()
|
|
assert body["messages"] == request_messages
|
|
assert body["tokens_before"] == body["tokens_after"] == 0
|
|
assert body["tokens_saved"] == 0
|
|
assert body["compression_ratio"] == 1.0
|
|
assert body["transforms_applied"] == []
|
|
assert body["compression_skipped"] is True
|
|
assert body["skip_reason"] == "compression_timeout"
|
|
|
|
|
|
def test_v1_compress_real_json_tool_payload_reduces_tokens(monkeypatch) -> None:
|
|
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
|
app = create_app(
|
|
_proxy_config(
|
|
ccr_inject_marker=False,
|
|
min_tokens_to_crush=20,
|
|
max_items_after_crush=10,
|
|
)
|
|
)
|
|
items = [
|
|
{
|
|
"id": i,
|
|
"status": "ok",
|
|
"score": i % 5,
|
|
"message": "same repeated value " * 20,
|
|
}
|
|
for i in range(80)
|
|
]
|
|
request = {
|
|
"model": "gpt-4o",
|
|
"messages": [
|
|
{"role": "user", "content": "summarize rows"},
|
|
{
|
|
"role": "assistant",
|
|
"content": None,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-1",
|
|
"type": "function",
|
|
"function": {"name": "list_rows", "arguments": "{}"},
|
|
}
|
|
],
|
|
},
|
|
{"role": "tool", "tool_call_id": "call-1", "content": json.dumps(items)},
|
|
],
|
|
}
|
|
|
|
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
|
|
response = client.post("/v1/compress", json=request)
|
|
|
|
body = response.json()
|
|
assert response.status_code == 200, response.text
|
|
assert body["tokens_before"] > body["tokens_after"], body
|
|
assert body["tokens_saved"] > 0
|
|
assert body["compression_ratio"] < 1.0
|
|
assert body["transforms_applied"], body
|
|
|
|
|
|
def test_compression_quarantine_releases_after_time_cap(monkeypatch) -> None:
|
|
"""A leaked/hung timed-out worker must not pin the quarantine open forever:
|
|
once the time cap lapses, compression resumes and the release is counted
|
|
once (#2360)."""
|
|
import asyncio
|
|
|
|
from headroom.proxy.server import CompressionQuarantinedError
|
|
|
|
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
|
|
app = create_app(_proxy_config())
|
|
proxy = app.state.proxy
|
|
|
|
# Simulate a timed-out worker that is still running (debt standing).
|
|
proxy._compression_timed_out_in_flight = 1
|
|
|
|
# Within the cap: new compression is quarantined.
|
|
proxy._compression_quarantine_deadline = time.monotonic() + 1000.0
|
|
with pytest.raises(CompressionQuarantinedError):
|
|
asyncio.run(proxy._run_compression_in_executor(lambda: "unused", timeout=5.0))
|
|
assert proxy._compression_quarantine_releases == 0
|
|
|
|
# Past the cap: the worker is presumed leaked and compression runs again;
|
|
# the release is recorded once.
|
|
proxy._compression_quarantine_deadline = time.monotonic() - 1.0
|
|
assert asyncio.run(proxy._run_compression_in_executor(lambda: "ran", timeout=5.0)) == "ran"
|
|
assert proxy._compression_quarantine_releases == 1
|
|
|
|
# A subsequent request is not re-counted and still runs.
|
|
assert asyncio.run(proxy._run_compression_in_executor(lambda: "ok", timeout=5.0)) == "ok"
|
|
assert proxy._compression_quarantine_releases == 1
|