mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(proxy): share one compression deadline across a Responses request
An OpenAI Responses request is compressed as many ContentRouter.compress calls, one per text unit or small-unit batch, several of them on the unit thread pool. #3693 gave each request one Kompress deadline origin, but only inside ContentRouter.apply(). The Responses adapter never calls apply(), so every Kompress call started its own HEADROOM_COMPRESSION_DEADLINE_MS clock. A request with many units ran far past the compression timeout: the awaiter gave up and forwarded the request uncompressed while the worker kept running inference, and that timeout debt quarantined compression for every request behind it. Stamp one origin per Responses request before the job is queued, with the budget capped at 75% of the caller's timeout so the chunk in flight, the splice and the serialization still fit, and hand it to every unit and batch. The router binds it as the request's kompress deadline (the per-request state apply() already uses), so Kompress stops at the next chunk boundary once it is spent, and units not started by then are returned unchanged without entering the router. Pool tasks run in a copy of the submitting Context. Finished work is kept, no thread is interrupted, and the unit result cache keeps storing what was forwarded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import contextvars
|
||||
import copy
|
||||
import hashlib
|
||||
import hmac
|
||||
@@ -644,6 +645,31 @@ def _openai_responses_unit_parallelism() -> int:
|
||||
return max(1, min(_OPENAI_RESPONSES_UNIT_PARALLELISM_MAX, requested))
|
||||
|
||||
|
||||
def _openai_responses_deadline_started_at(timeout: float) -> float | None:
|
||||
"""Deadline origin shared by every compress call of one Responses request.
|
||||
|
||||
A Responses request fans out over many ``ContentRouter.compress`` calls,
|
||||
and Kompress starts a fresh ``HEADROOM_COMPRESSION_DEADLINE_MS`` clock in
|
||||
each one unless it is handed an origin. Without one, a request past its
|
||||
``timeout`` keeps running inference long after the awaiter gave up, and
|
||||
that timeout debt quarantines compression for every request behind it.
|
||||
Stamped before the job is queued, because ``timeout`` counts queue wait
|
||||
too. ``None`` when the deadline is disabled (``0``).
|
||||
|
||||
The budget is also capped at 75% of ``timeout``: the deadline only stops
|
||||
the NEXT Kompress chunk, so the chunk already running when it expires, the
|
||||
splice and the output serialization need the rest. Kompress checks
|
||||
``now - origin > deadline``, so a shorter budget is a backdated origin.
|
||||
"""
|
||||
from headroom.transforms.content_router import _compression_deadline_seconds
|
||||
|
||||
deadline_s = _compression_deadline_seconds()
|
||||
if deadline_s <= 0:
|
||||
return None
|
||||
budget_s = min(deadline_s, 0.75 * timeout)
|
||||
return time.perf_counter() - (deadline_s - budget_s)
|
||||
|
||||
|
||||
def _openai_responses_unit_executor() -> ThreadPoolExecutor:
|
||||
global _OPENAI_RESPONSES_UNIT_EXECUTOR
|
||||
with _OPENAI_RESPONSES_UNIT_EXECUTOR_LOCK:
|
||||
@@ -2110,6 +2136,7 @@ class OpenAIHandlerMixin:
|
||||
request_id: str,
|
||||
pass_id: str | None = None,
|
||||
timing: dict[str, float] | None = None,
|
||||
deadline_started_at: float | None = None,
|
||||
) -> tuple[dict[str, Any], bool, int, list[str], dict[str, int], list[str], int]:
|
||||
"""Run ContentRouter on OpenAI Responses text units.
|
||||
|
||||
@@ -2700,6 +2727,7 @@ class OpenAIHandlerMixin:
|
||||
router=router,
|
||||
tokenizer=tokenizer,
|
||||
target_ratio=unit_target_ratio,
|
||||
deadline_started_at=deadline_started_at,
|
||||
)
|
||||
elapsed_ms = (time.perf_counter() - unit_started) * 1000.0
|
||||
return routed.slot, result, elapsed_ms
|
||||
@@ -2769,12 +2797,18 @@ class OpenAIHandlerMixin:
|
||||
0.0,
|
||||
)
|
||||
|
||||
# Each pool task runs in its own copy of this Context: the router binds
|
||||
# the request's deadline per task (see share_request_deadline), and
|
||||
# pool threads would otherwise keep that binding between requests.
|
||||
parallelism = _openai_responses_unit_parallelism()
|
||||
if len(cache_misses) > 1 and parallelism > 1:
|
||||
executor = _openai_responses_unit_executor()
|
||||
for start in range(0, len(cache_misses), parallelism):
|
||||
batch = cache_misses[start : start + parallelism]
|
||||
futures = [executor.submit(_compress_and_store, *item) for item in batch]
|
||||
futures = [
|
||||
executor.submit(contextvars.copy_context().run, _compress_and_store, *item)
|
||||
for item in batch
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
unit_idx, cache_key, routed_result = future.result()
|
||||
_record_routed_result(unit_idx, cache_key, routed_result)
|
||||
@@ -2799,6 +2833,7 @@ class OpenAIHandlerMixin:
|
||||
router=router,
|
||||
tokenizer=tokenizer,
|
||||
target_ratio=unit_target_ratio,
|
||||
deadline_started_at=deadline_started_at,
|
||||
)
|
||||
return results, (time.perf_counter() - batch_started) * 1000.0
|
||||
|
||||
@@ -2812,7 +2847,10 @@ class OpenAIHandlerMixin:
|
||||
executor = _openai_responses_unit_executor()
|
||||
for start in range(0, len(small_batches), parallelism):
|
||||
batch_group = small_batches[start : start + parallelism]
|
||||
futures = [executor.submit(_compress_batch, batch) for batch in batch_group]
|
||||
futures = [
|
||||
executor.submit(contextvars.copy_context().run, _compress_batch, batch)
|
||||
for batch in batch_group
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
_record_batch_result(future.result())
|
||||
else:
|
||||
@@ -3000,6 +3038,7 @@ class OpenAIHandlerMixin:
|
||||
timing: dict[str, float] | None = None,
|
||||
client: str | None = None,
|
||||
savings_tags: dict[str, Any] | None = None,
|
||||
deadline_started_at: float | None = None,
|
||||
) -> tuple[dict[str, Any], bool, int, list[str], str | None, int, int, int]:
|
||||
"""Compress an OpenAI Responses payload through the shared router.
|
||||
|
||||
@@ -3267,6 +3306,7 @@ class OpenAIHandlerMixin:
|
||||
request_id=request_id,
|
||||
pass_id=pass_id,
|
||||
timing=timing_sink,
|
||||
deadline_started_at=deadline_started_at,
|
||||
)
|
||||
_add_timing("compression_live_units_total", live_units_started)
|
||||
if router_modified:
|
||||
@@ -3414,6 +3454,8 @@ class OpenAIHandlerMixin:
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
deadline_started_at = _openai_responses_deadline_started_at(timeout)
|
||||
|
||||
def _compress(): # noqa: ANN202
|
||||
# Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER) runs before
|
||||
# compression so the turn classifier sees the client's input as
|
||||
@@ -3433,6 +3475,7 @@ class OpenAIHandlerMixin:
|
||||
"request_id": request_id,
|
||||
"timing": timing,
|
||||
"client": client,
|
||||
"deadline_started_at": deadline_started_at,
|
||||
}
|
||||
if savings_tags is not None:
|
||||
compression_kwargs["savings_tags"] = savings_tags
|
||||
@@ -3447,7 +3490,12 @@ class OpenAIHandlerMixin:
|
||||
unsupported_kwarg = next(
|
||||
(
|
||||
name
|
||||
for name in ("savings_tags", "client", "timing")
|
||||
for name in (
|
||||
"savings_tags",
|
||||
"client",
|
||||
"timing",
|
||||
"deadline_started_at",
|
||||
)
|
||||
if f"unexpected keyword argument '{name}'" in str(exc)
|
||||
and name in compression_kwargs
|
||||
),
|
||||
|
||||
@@ -230,8 +230,15 @@ def compress_batch_with_router(
|
||||
router: ContentRouter,
|
||||
tokenizer: TokenCounterLike,
|
||||
target_ratio: float | None = None,
|
||||
deadline_started_at: float | None = None,
|
||||
) -> list[tuple[object, UnitCompressionResult]]:
|
||||
"""Compress one tagged batch and split only structurally valid output."""
|
||||
"""Compress one tagged batch and split only structurally valid output.
|
||||
|
||||
``deadline_started_at`` works as in ``compress_unit_with_router``.
|
||||
"""
|
||||
|
||||
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
|
||||
return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")
|
||||
|
||||
nonce = _batch_nonce(batch)
|
||||
batch_texts, marker_blocks = _protect_ccr_markers(batch, nonce)
|
||||
|
||||
@@ -56,6 +56,8 @@ class CompressionUnit:
|
||||
# Responses handler's unit-result cache) and is surfaced
|
||||
# via RouterCompressionResult.cache_hit, not as a reason
|
||||
# here — no code path produces this category today
|
||||
# - deadline: the request's compression deadline had passed before
|
||||
# this unit started, so it was left unchanged
|
||||
UNIT_REASON_CATEGORIES = {
|
||||
None: "applied",
|
||||
"protected_user_message": "protected_role",
|
||||
@@ -66,6 +68,7 @@ UNIT_REASON_CATEGORIES = {
|
||||
"router_no_change": "compressor_noop",
|
||||
"already_compressed": "already_compressed",
|
||||
"rejected_not_smaller": "rejected_not_smaller",
|
||||
"deadline_exceeded": "deadline",
|
||||
}
|
||||
|
||||
|
||||
@@ -221,11 +224,17 @@ def compress_unit_with_router(
|
||||
router: ContentRouter,
|
||||
tokenizer: TokenCounterLike,
|
||||
target_ratio: float | None = None,
|
||||
deadline_started_at: float | None = None,
|
||||
) -> UnitCompressionResult:
|
||||
"""Compress one safe text unit through ContentRouter.
|
||||
|
||||
The final accept/reject gate uses the provider/model tokenizer, not the
|
||||
router's internal word-count estimates.
|
||||
|
||||
``deadline_started_at`` is the deadline origin of the request this unit
|
||||
belongs to (see ``ContentRouter.share_request_deadline``). Once that
|
||||
deadline has passed the unit is returned unchanged without entering the
|
||||
router.
|
||||
"""
|
||||
|
||||
tokens_before = tokenizer.count_text(unit.text)
|
||||
@@ -266,6 +275,8 @@ def compress_unit_with_router(
|
||||
return _with_reason(reason=f"cache_zone_{unit.cache_zone}")
|
||||
if text_bytes < unit.min_bytes:
|
||||
return _with_reason(reason="below_unit_floor")
|
||||
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
|
||||
return _with_reason(reason="deadline_exceeded")
|
||||
|
||||
prior_target_ratio = getattr(router, "_runtime_target_ratio", None)
|
||||
if target_ratio is not None:
|
||||
|
||||
@@ -2394,6 +2394,27 @@ class ContentRouter(Transform):
|
||||
self._runtime_state_var.set(state)
|
||||
return state
|
||||
|
||||
def share_request_deadline(self, started_at: float) -> bool:
|
||||
"""Join a request whose kompress deadline started at ``started_at``.
|
||||
|
||||
``apply()`` stamps its own origin. A caller that fans ONE request out
|
||||
over many ``compress()`` calls (the OpenAI Responses unit adapter) calls
|
||||
this before each one instead, so every call draws down the same
|
||||
``HEADROOM_COMPRESSION_DEADLINE_MS`` budget rather than restarting it.
|
||||
Binds a fresh ``_PerRequestRuntimeState`` in the CURRENT Context, so a
|
||||
worker-pool task must run in its own ``contextvars.copy_context()``.
|
||||
|
||||
Returns ``False``, binding nothing, once the deadline has passed: the
|
||||
caller should leave the content unchanged rather than start new work.
|
||||
"""
|
||||
deadline_s = _compression_deadline_seconds()
|
||||
if deadline_s and time.perf_counter() - started_at > deadline_s:
|
||||
return False
|
||||
self._runtime_state_var.set(
|
||||
_PerRequestRuntimeState(kompress_deadline_started_at=started_at)
|
||||
)
|
||||
return True
|
||||
|
||||
@property
|
||||
def _runtime_compression_policy(self) -> Any:
|
||||
return self._runtime_state_var.get().compression_policy
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
|
||||
from headroom.transforms.compression_batches import (
|
||||
CompressionBatchEntry,
|
||||
@@ -229,3 +230,27 @@ def test_batch_uses_utf8_bytes_for_cjk_small_units():
|
||||
assert batches[0].text_bytes == 1800
|
||||
assert all(result.modified for _, result in results)
|
||||
assert [result.compressed for _, result in results] == ["短"] * 4
|
||||
|
||||
|
||||
def test_batch_after_the_request_deadline_passes_through_without_routing(monkeypatch):
|
||||
from headroom.transforms.content_router import ContentRouter
|
||||
|
||||
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
|
||||
entries = [_entry(index, "x" * 150) for index in range(4)]
|
||||
batches, _ = build_compression_batches(entries, min_batch_bytes=512)
|
||||
router = ContentRouter()
|
||||
|
||||
def _not_called(*_args, **_kwargs):
|
||||
raise AssertionError("a batch past the deadline must not enter the router")
|
||||
|
||||
router.compress = _not_called
|
||||
|
||||
results = compress_batch_with_router(
|
||||
batches[0],
|
||||
router=router,
|
||||
tokenizer=_CharacterCounter(),
|
||||
deadline_started_at=time.perf_counter() - 21,
|
||||
)
|
||||
|
||||
assert [result.compressed for _, result in results] == ["x" * 150] * 4
|
||||
assert {result.reason for _, result in results} == {"deadline_exceeded"}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from types import MethodType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -1415,3 +1417,111 @@ def test_openai_responses_adapter_compresses_historical_messages_not_current_use
|
||||
assert new_payload["input"][2]["content"][0]["text"] == current_text
|
||||
assert units_by_category == {"applied": 2}
|
||||
assert any(t.startswith("router:openai:responses:message:") for t in transforms)
|
||||
|
||||
|
||||
class _RecordingKompress:
|
||||
"""Stands in for KompressCompressor, recording the deadline origin it gets."""
|
||||
|
||||
shares_request_deadline = True
|
||||
|
||||
def __init__(self, delay: float = 0.0) -> None:
|
||||
self.delay = delay
|
||||
self.deadlines: list[float | None] = []
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return True
|
||||
|
||||
def compress(self, content: str, **kwargs):
|
||||
self.deadlines.append(kwargs.get("_deadline_started_at"))
|
||||
time.sleep(self.delay)
|
||||
compressed = " ".join(content.split()[:20])
|
||||
return SimpleNamespace(compressed=compressed, compressed_tokens=20)
|
||||
|
||||
|
||||
def _kompress_payload(units: int) -> dict:
|
||||
return {
|
||||
"model": "gpt-5",
|
||||
"input": [
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": f"call_{i}",
|
||||
"output": "\n".join(
|
||||
" ".join(f"unit{i}line{line}word{j}" for j in range(10)) for line in range(70)
|
||||
),
|
||||
}
|
||||
for i in range(units)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _compress_through_executor(router: ContentRouter, payload: dict) -> tuple:
|
||||
handler = _handler_with_router(router)
|
||||
|
||||
async def _run_compression(fn, *, timeout): # noqa: ANN001, ANN202
|
||||
return fn()
|
||||
|
||||
handler._run_compression_in_executor = _run_compression
|
||||
return asyncio.run(
|
||||
handler._compress_openai_responses_payload_in_executor(
|
||||
payload, model="gpt-5", request_id="req_deadline"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_openai_responses_units_share_one_kompress_deadline(monkeypatch):
|
||||
"""The fan-out must draw down ONE request deadline (#3693), pool threads too.
|
||||
|
||||
Each unit is its own ContentRouter.compress call, and Kompress started a
|
||||
fresh HEADROOM_COMPRESSION_DEADLINE_MS clock in every one, so a request
|
||||
that blew its timeout kept running inference for minutes afterwards.
|
||||
"""
|
||||
monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "4")
|
||||
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
|
||||
router = ContentRouter()
|
||||
kompress = _RecordingKompress()
|
||||
monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
|
||||
|
||||
_compress_through_executor(router, _kompress_payload(6))
|
||||
|
||||
assert len(kompress.deadlines) >= 6, kompress.deadlines
|
||||
assert None not in kompress.deadlines, kompress.deadlines
|
||||
assert len(set(kompress.deadlines)) == 1, kompress.deadlines
|
||||
|
||||
|
||||
def test_openai_responses_units_after_the_deadline_pass_through_unchanged(monkeypatch):
|
||||
"""Finished units are kept; units not started by the deadline are not started."""
|
||||
monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "1")
|
||||
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "200")
|
||||
router = ContentRouter()
|
||||
kompress = _RecordingKompress(delay=0.3)
|
||||
monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
|
||||
payload = _kompress_payload(4)
|
||||
original = json.loads(json.dumps(payload))
|
||||
|
||||
result = _compress_through_executor(router, payload)
|
||||
|
||||
assert len(kompress.deadlines) == 1
|
||||
outputs = [item["output"] for item in result[0]["input"]]
|
||||
assert outputs[0] != original["input"][0]["output"]
|
||||
assert outputs[1:] == [item["output"] for item in original["input"][1:]]
|
||||
assert result[2] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deadline_ms", "timeout", "budget"),
|
||||
[("20000", 30.0, 20.0), ("20000", 5.0, 3.75), ("0", 30.0, None)],
|
||||
ids=["deadline-binds", "timeout-binds", "disabled"],
|
||||
)
|
||||
def test_openai_responses_deadline_budget_stays_inside_the_timeout(
|
||||
monkeypatch, deadline_ms, timeout, budget
|
||||
):
|
||||
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", deadline_ms)
|
||||
before = time.perf_counter()
|
||||
origin = openai_handler._openai_responses_deadline_started_at(timeout)
|
||||
after = time.perf_counter()
|
||||
|
||||
if budget is None:
|
||||
assert origin is None
|
||||
return
|
||||
expires_at = origin + int(deadline_ms) / 1000
|
||||
assert before + budget <= expires_at <= after + budget
|
||||
|
||||
@@ -132,7 +132,10 @@ def test_native_responses_compressor_scopes_the_exclusion_per_call() -> None:
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
assert [{key: value for key, value in call.items() if key != "timing"} for call in seen] == [
|
||||
assert [
|
||||
{key: value for key, value in call.items() if key not in {"timing", "deadline_started_at"}}
|
||||
for call in seen
|
||||
] == [
|
||||
{"client": "opencode"},
|
||||
{"client": None},
|
||||
]
|
||||
@@ -445,4 +448,6 @@ def test_websocket_route_carries_the_client_decision(
|
||||
asyncio.run(handler.handle_openai_responses_ws(client_ws))
|
||||
|
||||
assert seen, "the Responses compressor was never reached on the WS path"
|
||||
assert {k: v for k, v in seen[0].items() if k != "timing"} == expected
|
||||
assert {
|
||||
k: v for k, v in seen[0].items() if k not in {"timing", "deadline_started_at"}
|
||||
} == expected
|
||||
|
||||
Reference in New Issue
Block a user