Files
headroom/tests/test_platform_stabilization_functional.py
97e7bc3333 test(proxy): make compression timeout regression deterministic (#4587)
## Description

Make the compression-timeout regression independent of host speed. The
old test required the entire HTTP request, including outcome
bookkeeping, to finish within 500 ms. Hosted runs show the unchanged
production path can exceed that limit after the simulated compression
timeout has already occurred.

## Type of Change

- [x] Test-only change; no production behavior change

## Changes Made

- Assert that the compression executor is awaited exactly once.
- Fail if timeout handling retries compression inline or recounts
tokens.
- Retain the fail-open response checks: HTTP 200, unchanged messages,
zero token counts/savings, unchanged compression ratio, and
compression-timeout metadata.
- Replace the unrelated whole-request wall-clock assertion with those
behavioral checks.
- Reword Copilot's generated initial-plan commit to satisfy Conventional
Commits; all file trees and contributor authorship were preserved.

## Testing

- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)

- Local Windows/CPython 3.12: `G:/git/headroom/.venv/Scripts/python.exe
-m pytest tests/test_platform_stabilization_functional.py -q --tb=short`
— five passed, one existing Starlette deprecation warning, using the
existing native extension.
- Current head `14404162810068d2c4a82b8eeebaab1b313aa3da`: 32 successful
hosted checks and 12 skips; no failures or running checks at
qualification.
- The test source is byte-identical to the locally tested original
implementation commit; the subsequent changes only merge main and repair
commit metadata.

## Real Behavior Proof

- Environment: GitHub-hosted Linux CI plus the local Windows test
environment above; no provider credentials or paid inference.
- Exact command / steps: run the functional-test command in Testing,
then inspect CI run 41104791110 on the current head, including all four
Python shards.
- Observed result: five local functional tests passed; the current
head's hosted checks completed with 32 successes, 12 skips, and no
failures.
- Observed input to this fix: the prior timeout assertion failed at
approximately 1.0–1.2 seconds against a 0.5-second limit on several
otherwise unrelated PRs whose proxy implementation matched main.
- After-fix evidence: the current full hosted test run succeeds with the
deterministic timeout contract assertions in place. Local functional
tests also pass.
- Scope: this is an internal maintainer-coauthored test correction. The
evidence establishes the regression-test contract; it is not a new
live-provider or live-Claude session qualification. No production
timeout, deadline, or bookkeeping implementation changed.
- Not tested: live provider traffic or a fresh native rebuild.

## Runtime Rollout Safety

- Rollout-managed feature: none; tests only.
- Minimum rollout channel: not applicable; no runtime feature is added
or enabled.
- Stable/default behavior changed: no.
- Kill switch/rollback: revert this test change; production behavior is
unaffected.
- Rollback path: revert the test-only commit while retaining production
timeout handling.
- Unsafe override required: no.
- Qualification impact: replaces host-speed sensitivity with explicit
fail-open behavior checks.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

Automated code qualification does not constitute final human merge
approval. Nothing has been merged, tagged, or published.

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: JerrettDavis <2610199+JerrettDavis@users.noreply.github.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
2026-11-03 08:41:55 +05:30

210 lines
7.9 KiB
Python

from __future__ import annotations
import json
import time
from typing import Any
from unittest.mock import AsyncMock, Mock
import pytest
pytest.importorskip("fastapi")
pytest.importorskip("headroom._core")
from fastapi.testclient import TestClient
from headroom.config import TransformResult
from headroom.proxy.server import ProxyConfig, create_app
from headroom.tokenizer import Tokenizer
def _proxy_config(**overrides: Any) -> ProxyConfig:
defaults: dict[str, Any] = {
"optimize": True,
"cache_enabled": False,
"rate_limit_enabled": False,
"cost_tracking_enabled": False,
"log_requests": False,
"ccr_inject_tool": False,
"ccr_handle_responses": False,
"ccr_context_tracking": False,
"image_optimize": False,
"disable_kompress": True,
"compression_max_workers": 1,
}
defaults.update(overrides)
return ProxyConfig(**defaults)
def test_proxy_health_surfaces_compression_runtime_metrics(monkeypatch) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
app = create_app(_proxy_config(optimize=False))
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
live = client.get("/livez")
health = client.get("/health")
assert live.status_code == 200
assert live.json()["alive"] is True
assert health.status_code == 200
runtime = health.json()["runtime"]
assert runtime["compression_executor"]["max_workers"] == 1
assert runtime["compression_executor"]["queued"] == 0
assert runtime["compression_executor"]["queue_timeouts_total"] == 0
def test_v1_compress_success_reports_actual_metrics(monkeypatch) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
app = create_app(_proxy_config())
proxy = app.state.proxy
request_messages = [{"role": "user", "content": "summarize this repeated payload"}]
compressed_messages = [{"role": "user", "content": "summary payload"}]
ccr_hash = "abc123def4567890abc123de"
def fake_apply(**kwargs):
assert kwargs["messages"] == request_messages
assert kwargs["model"] == "gpt-4o"
return TransformResult(
messages=compressed_messages,
tokens_before=100,
tokens_after=40,
transforms_applied=["test:compress"],
markers_inserted=[ccr_hash],
)
# The default /v1/compress mode runs a marker-free pipeline derived from
# `openai_pipeline`, not `openai_pipeline` itself, so patch the one the
# route actually uses. It is built eagerly at create_app() time.
monkeypatch.setattr(proxy._compress_pipeline_cache["no_ccr"], "apply", fake_apply)
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
response = client.post(
"/v1/compress",
json={"model": "gpt-4o", "messages": request_messages},
)
body = response.json()
assert response.status_code == 200
assert body["messages"] == compressed_messages
assert body["tokens_before"] == 100
assert body["tokens_after"] == 40
assert body["tokens_saved"] == 60
assert body["compression_ratio"] == 0.4
assert body["transforms_applied"] == ["test:compress"]
assert body["transforms_summary"] == {"test:compress": 1}
assert body["ccr_hashes"] == [ccr_hash]
def test_v1_compress_timeout_fails_open_without_retry_or_recount(monkeypatch) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
app = create_app(_proxy_config())
proxy = app.state.proxy
request_messages = [{"role": "user", "content": "do not mutate me"}]
timeout_executor = AsyncMock(side_effect=TimeoutError("compression deadline exceeded"))
pipeline_apply = Mock(side_effect=AssertionError("compression must not retry inline"))
count_messages = Mock(side_effect=AssertionError("timeout must not recount tokens"))
monkeypatch.setattr(proxy, "_run_compression_in_executor", timeout_executor)
monkeypatch.setattr(proxy._compress_pipeline_cache["no_ccr"], "apply", pipeline_apply)
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
# Check the fail-open contract, not host speed: outcome bookkeeping
# after the timeout can include pricing and durable savings writes.
monkeypatch.setattr(Tokenizer, "count_messages", count_messages)
response = client.post(
"/v1/compress",
json={"model": "gpt-4o", "messages": request_messages},
)
body = response.json()
assert response.status_code == 200
timeout_executor.assert_awaited_once()
pipeline_apply.assert_not_called()
count_messages.assert_not_called()
assert body["messages"] == request_messages
assert body["tokens_before"] == body["tokens_after"] == 0
assert body["tokens_saved"] == 0
assert body["compression_ratio"] == 1.0
assert body["transforms_applied"] == []
assert body["compression_skipped"] is True
assert body["skip_reason"] == "compression_timeout"
def test_v1_compress_real_json_tool_payload_reduces_tokens(monkeypatch) -> None:
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
app = create_app(
_proxy_config(
ccr_inject_marker=False,
min_tokens_to_crush=20,
max_items_after_crush=10,
)
)
items = [
{
"id": i,
"status": "ok",
"score": i % 5,
"message": "same repeated value " * 20,
}
for i in range(80)
]
request = {
"model": "gpt-4o",
"messages": [
{"role": "user", "content": "summarize rows"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "list_rows", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call-1", "content": json.dumps(items)},
],
}
with TestClient(app, base_url="http://127.0.0.1", client=("127.0.0.1", 12345)) as client:
response = client.post("/v1/compress", json=request)
body = response.json()
assert response.status_code == 200, response.text
assert body["tokens_before"] > body["tokens_after"], body
assert body["tokens_saved"] > 0
assert body["compression_ratio"] < 1.0
assert body["transforms_applied"], body
def test_compression_quarantine_releases_after_time_cap(monkeypatch) -> None:
"""A leaked/hung timed-out worker must not pin the quarantine open forever:
once the time cap lapses, compression resumes and the release is counted
once (#2360)."""
import asyncio
from headroom.proxy.server import CompressionQuarantinedError
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
app = create_app(_proxy_config())
proxy = app.state.proxy
# Simulate a timed-out worker that is still running (debt standing).
proxy._compression_timed_out_in_flight = 1
# Within the cap: new compression is quarantined.
proxy._compression_quarantine_deadline = time.monotonic() + 1000.0
with pytest.raises(CompressionQuarantinedError):
asyncio.run(proxy._run_compression_in_executor(lambda: "unused", timeout=5.0))
assert proxy._compression_quarantine_releases == 0
# Past the cap: the worker is presumed leaked and compression runs again;
# the release is recorded once.
proxy._compression_quarantine_deadline = time.monotonic() - 1.0
assert asyncio.run(proxy._run_compression_in_executor(lambda: "ran", timeout=5.0)) == "ran"
assert proxy._compression_quarantine_releases == 1
# A subsequent request is not re-counted and still runs.
assert asyncio.run(proxy._run_compression_in_executor(lambda: "ok", timeout=5.0)) == "ok"
assert proxy._compression_quarantine_releases == 1