fix(proxy): share one compression deadline across a Responses request

An OpenAI Responses request is compressed as many ContentRouter.compress
calls, one per text unit or small-unit batch, several of them on the unit
thread pool. #3693 gave each request one Kompress deadline origin, but only
inside ContentRouter.apply(). The Responses adapter never calls apply(), so
every Kompress call started its own HEADROOM_COMPRESSION_DEADLINE_MS clock.
A request with many units ran far past the compression timeout: the awaiter
gave up and forwarded the request uncompressed while the worker kept running
inference, and that timeout debt quarantined compression for every request
behind it.

Stamp one origin per Responses request before the job is queued, with the
budget capped at 75% of the caller's timeout so the chunk in flight, the
splice and the serialization still fit, and hand it to every unit and batch.
The router binds it as the request's kompress deadline (the per-request
state apply() already uses), so Kompress stops at the next chunk boundary
once it is spent, and units not started by then are returned unchanged
without entering the router. Pool tasks run in a copy of the submitting
Context. Finished work is kept, no thread is interrupted, and the unit
result cache keeps storing what was forwarded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-02 15:49:54 +02:00
co-authored by Claude Opus 5.5
parent d5318ac235
commit cfa775a10a
7 changed files with 233 additions and 6 deletions
+51 -3
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import asyncio
import contextlib
import contextvars
import copy
import hashlib
import hmac
@@ -644,6 +645,31 @@ def _openai_responses_unit_parallelism() -> int:
return max(1, min(_OPENAI_RESPONSES_UNIT_PARALLELISM_MAX, requested))
def _openai_responses_deadline_started_at(timeout: float) -> float | None:
"""Deadline origin shared by every compress call of one Responses request.
A Responses request fans out over many ``ContentRouter.compress`` calls,
and Kompress starts a fresh ``HEADROOM_COMPRESSION_DEADLINE_MS`` clock in
each one unless it is handed an origin. Without one, a request past its
``timeout`` keeps running inference long after the awaiter gave up, and
that timeout debt quarantines compression for every request behind it.
Stamped before the job is queued, because ``timeout`` counts queue wait
too. ``None`` when the deadline is disabled (``0``).
The budget is also capped at 75% of ``timeout``: the deadline only stops
the NEXT Kompress chunk, so the chunk already running when it expires, the
splice and the output serialization need the rest. Kompress checks
``now - origin > deadline``, so a shorter budget is a backdated origin.
"""
from headroom.transforms.content_router import _compression_deadline_seconds
deadline_s = _compression_deadline_seconds()
if deadline_s <= 0:
return None
budget_s = min(deadline_s, 0.75 * timeout)
return time.perf_counter() - (deadline_s - budget_s)
def _openai_responses_unit_executor() -> ThreadPoolExecutor:
global _OPENAI_RESPONSES_UNIT_EXECUTOR
with _OPENAI_RESPONSES_UNIT_EXECUTOR_LOCK:
@@ -2110,6 +2136,7 @@ class OpenAIHandlerMixin:
request_id: str,
pass_id: str | None = None,
timing: dict[str, float] | None = None,
deadline_started_at: float | None = None,
) -> tuple[dict[str, Any], bool, int, list[str], dict[str, int], list[str], int]:
"""Run ContentRouter on OpenAI Responses text units.
@@ -2700,6 +2727,7 @@ class OpenAIHandlerMixin:
router=router,
tokenizer=tokenizer,
target_ratio=unit_target_ratio,
deadline_started_at=deadline_started_at,
)
elapsed_ms = (time.perf_counter() - unit_started) * 1000.0
return routed.slot, result, elapsed_ms
@@ -2769,12 +2797,18 @@ class OpenAIHandlerMixin:
0.0,
)
# Each pool task runs in its own copy of this Context: the router binds
# the request's deadline per task (see share_request_deadline), and
# pool threads would otherwise keep that binding between requests.
parallelism = _openai_responses_unit_parallelism()
if len(cache_misses) > 1 and parallelism > 1:
executor = _openai_responses_unit_executor()
for start in range(0, len(cache_misses), parallelism):
batch = cache_misses[start : start + parallelism]
futures = [executor.submit(_compress_and_store, *item) for item in batch]
futures = [
executor.submit(contextvars.copy_context().run, _compress_and_store, *item)
for item in batch
]
for future in as_completed(futures):
unit_idx, cache_key, routed_result = future.result()
_record_routed_result(unit_idx, cache_key, routed_result)
@@ -2799,6 +2833,7 @@ class OpenAIHandlerMixin:
router=router,
tokenizer=tokenizer,
target_ratio=unit_target_ratio,
deadline_started_at=deadline_started_at,
)
return results, (time.perf_counter() - batch_started) * 1000.0
@@ -2812,7 +2847,10 @@ class OpenAIHandlerMixin:
executor = _openai_responses_unit_executor()
for start in range(0, len(small_batches), parallelism):
batch_group = small_batches[start : start + parallelism]
futures = [executor.submit(_compress_batch, batch) for batch in batch_group]
futures = [
executor.submit(contextvars.copy_context().run, _compress_batch, batch)
for batch in batch_group
]
for future in as_completed(futures):
_record_batch_result(future.result())
else:
@@ -3000,6 +3038,7 @@ class OpenAIHandlerMixin:
timing: dict[str, float] | None = None,
client: str | None = None,
savings_tags: dict[str, Any] | None = None,
deadline_started_at: float | None = None,
) -> tuple[dict[str, Any], bool, int, list[str], str | None, int, int, int]:
"""Compress an OpenAI Responses payload through the shared router.
@@ -3267,6 +3306,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
pass_id=pass_id,
timing=timing_sink,
deadline_started_at=deadline_started_at,
)
_add_timing("compression_live_units_total", live_units_started)
if router_modified:
@@ -3414,6 +3454,8 @@ class OpenAIHandlerMixin:
exc_info=True,
)
deadline_started_at = _openai_responses_deadline_started_at(timeout)
def _compress(): # noqa: ANN202
# Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER) runs before
# compression so the turn classifier sees the client's input as
@@ -3433,6 +3475,7 @@ class OpenAIHandlerMixin:
"request_id": request_id,
"timing": timing,
"client": client,
"deadline_started_at": deadline_started_at,
}
if savings_tags is not None:
compression_kwargs["savings_tags"] = savings_tags
@@ -3447,7 +3490,12 @@ class OpenAIHandlerMixin:
unsupported_kwarg = next(
(
name
for name in ("savings_tags", "client", "timing")
for name in (
"savings_tags",
"client",
"timing",
"deadline_started_at",
)
if f"unexpected keyword argument '{name}'" in str(exc)
and name in compression_kwargs
),
+8 -1
View File
@@ -230,8 +230,15 @@ def compress_batch_with_router(
router: ContentRouter,
tokenizer: TokenCounterLike,
target_ratio: float | None = None,
deadline_started_at: float | None = None,
) -> list[tuple[object, UnitCompressionResult]]:
"""Compress one tagged batch and split only structurally valid output."""
"""Compress one tagged batch and split only structurally valid output.
``deadline_started_at`` works as in ``compress_unit_with_router``.
"""
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")
nonce = _batch_nonce(batch)
batch_texts, marker_blocks = _protect_ccr_markers(batch, nonce)
+11
View File
@@ -56,6 +56,8 @@ class CompressionUnit:
# Responses handler's unit-result cache) and is surfaced
# via RouterCompressionResult.cache_hit, not as a reason
# here — no code path produces this category today
# - deadline: the request's compression deadline had passed before
# this unit started, so it was left unchanged
UNIT_REASON_CATEGORIES = {
None: "applied",
"protected_user_message": "protected_role",
@@ -66,6 +68,7 @@ UNIT_REASON_CATEGORIES = {
"router_no_change": "compressor_noop",
"already_compressed": "already_compressed",
"rejected_not_smaller": "rejected_not_smaller",
"deadline_exceeded": "deadline",
}
@@ -221,11 +224,17 @@ def compress_unit_with_router(
router: ContentRouter,
tokenizer: TokenCounterLike,
target_ratio: float | None = None,
deadline_started_at: float | None = None,
) -> UnitCompressionResult:
"""Compress one safe text unit through ContentRouter.
The final accept/reject gate uses the provider/model tokenizer, not the
router's internal word-count estimates.
``deadline_started_at`` is the deadline origin of the request this unit
belongs to (see ``ContentRouter.share_request_deadline``). Once that
deadline has passed the unit is returned unchanged without entering the
router.
"""
tokens_before = tokenizer.count_text(unit.text)
@@ -266,6 +275,8 @@ def compress_unit_with_router(
return _with_reason(reason=f"cache_zone_{unit.cache_zone}")
if text_bytes < unit.min_bytes:
return _with_reason(reason="below_unit_floor")
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
return _with_reason(reason="deadline_exceeded")
prior_target_ratio = getattr(router, "_runtime_target_ratio", None)
if target_ratio is not None:
+21
View File
@@ -2394,6 +2394,27 @@ class ContentRouter(Transform):
self._runtime_state_var.set(state)
return state
def share_request_deadline(self, started_at: float) -> bool:
"""Join a request whose kompress deadline started at ``started_at``.
``apply()`` stamps its own origin. A caller that fans ONE request out
over many ``compress()`` calls (the OpenAI Responses unit adapter) calls
this before each one instead, so every call draws down the same
``HEADROOM_COMPRESSION_DEADLINE_MS`` budget rather than restarting it.
Binds a fresh ``_PerRequestRuntimeState`` in the CURRENT Context, so a
worker-pool task must run in its own ``contextvars.copy_context()``.
Returns ``False``, binding nothing, once the deadline has passed: the
caller should leave the content unchanged rather than start new work.
"""
deadline_s = _compression_deadline_seconds()
if deadline_s and time.perf_counter() - started_at > deadline_s:
return False
self._runtime_state_var.set(
_PerRequestRuntimeState(kompress_deadline_started_at=started_at)
)
return True
@property
def _runtime_compression_policy(self) -> Any:
return self._runtime_state_var.get().compression_policy
+25
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import re
import time
from headroom.transforms.compression_batches import (
CompressionBatchEntry,
@@ -229,3 +230,27 @@ def test_batch_uses_utf8_bytes_for_cjk_small_units():
assert batches[0].text_bytes == 1800
assert all(result.modified for _, result in results)
assert [result.compressed for _, result in results] == ["短"] * 4
def test_batch_after_the_request_deadline_passes_through_without_routing(monkeypatch):
from headroom.transforms.content_router import ContentRouter
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
entries = [_entry(index, "x" * 150) for index in range(4)]
batches, _ = build_compression_batches(entries, min_batch_bytes=512)
router = ContentRouter()
def _not_called(*_args, **_kwargs):
raise AssertionError("a batch past the deadline must not enter the router")
router.compress = _not_called
results = compress_batch_with_router(
batches[0],
router=router,
tokenizer=_CharacterCounter(),
deadline_started_at=time.perf_counter() - 21,
)
assert [result.compressed for _, result in results] == ["x" * 150] * 4
assert {result.reason for _, result in results} == {"deadline_exceeded"}
@@ -1,7 +1,9 @@
from __future__ import annotations
import asyncio
import json
import threading
import time
from types import MethodType, SimpleNamespace
import pytest
@@ -1415,3 +1417,111 @@ def test_openai_responses_adapter_compresses_historical_messages_not_current_use
assert new_payload["input"][2]["content"][0]["text"] == current_text
assert units_by_category == {"applied": 2}
assert any(t.startswith("router:openai:responses:message:") for t in transforms)
class _RecordingKompress:
"""Stands in for KompressCompressor, recording the deadline origin it gets."""
shares_request_deadline = True
def __init__(self, delay: float = 0.0) -> None:
self.delay = delay
self.deadlines: list[float | None] = []
def is_ready(self) -> bool:
return True
def compress(self, content: str, **kwargs):
self.deadlines.append(kwargs.get("_deadline_started_at"))
time.sleep(self.delay)
compressed = " ".join(content.split()[:20])
return SimpleNamespace(compressed=compressed, compressed_tokens=20)
def _kompress_payload(units: int) -> dict:
return {
"model": "gpt-5",
"input": [
{
"type": "function_call_output",
"call_id": f"call_{i}",
"output": "\n".join(
" ".join(f"unit{i}line{line}word{j}" for j in range(10)) for line in range(70)
),
}
for i in range(units)
],
}
def _compress_through_executor(router: ContentRouter, payload: dict) -> tuple:
handler = _handler_with_router(router)
async def _run_compression(fn, *, timeout): # noqa: ANN001, ANN202
return fn()
handler._run_compression_in_executor = _run_compression
return asyncio.run(
handler._compress_openai_responses_payload_in_executor(
payload, model="gpt-5", request_id="req_deadline"
)
)
def test_openai_responses_units_share_one_kompress_deadline(monkeypatch):
"""The fan-out must draw down ONE request deadline (#3693), pool threads too.
Each unit is its own ContentRouter.compress call, and Kompress started a
fresh HEADROOM_COMPRESSION_DEADLINE_MS clock in every one, so a request
that blew its timeout kept running inference for minutes afterwards.
"""
monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "4")
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
router = ContentRouter()
kompress = _RecordingKompress()
monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
_compress_through_executor(router, _kompress_payload(6))
assert len(kompress.deadlines) >= 6, kompress.deadlines
assert None not in kompress.deadlines, kompress.deadlines
assert len(set(kompress.deadlines)) == 1, kompress.deadlines
def test_openai_responses_units_after_the_deadline_pass_through_unchanged(monkeypatch):
"""Finished units are kept; units not started by the deadline are not started."""
monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "1")
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "200")
router = ContentRouter()
kompress = _RecordingKompress(delay=0.3)
monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
payload = _kompress_payload(4)
original = json.loads(json.dumps(payload))
result = _compress_through_executor(router, payload)
assert len(kompress.deadlines) == 1
outputs = [item["output"] for item in result[0]["input"]]
assert outputs[0] != original["input"][0]["output"]
assert outputs[1:] == [item["output"] for item in original["input"][1:]]
assert result[2] > 0
@pytest.mark.parametrize(
("deadline_ms", "timeout", "budget"),
[("20000", 30.0, 20.0), ("20000", 5.0, 3.75), ("0", 30.0, None)],
ids=["deadline-binds", "timeout-binds", "disabled"],
)
def test_openai_responses_deadline_budget_stays_inside_the_timeout(
monkeypatch, deadline_ms, timeout, budget
):
monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", deadline_ms)
before = time.perf_counter()
origin = openai_handler._openai_responses_deadline_started_at(timeout)
after = time.perf_counter()
if budget is None:
assert origin is None
return
expires_at = origin + int(deadline_ms) / 1000
assert before + budget <= expires_at <= after + budget
+7 -2
View File
@@ -132,7 +132,10 @@ def test_native_responses_compressor_scopes_the_exclusion_per_call() -> None:
asyncio.run(_run())
assert [{key: value for key, value in call.items() if key != "timing"} for call in seen] == [
assert [
{key: value for key, value in call.items() if key not in {"timing", "deadline_started_at"}}
for call in seen
] == [
{"client": "opencode"},
{"client": None},
]
@@ -445,4 +448,6 @@ def test_websocket_route_carries_the_client_decision(
asyncio.run(handler.handle_openai_responses_ws(client_ws))
assert seen, "the Responses compressor was never reached on the WS path"
assert {k: v for k, v in seen[0].items() if k != "timing"} == expected
assert {
k: v for k, v in seen[0].items() if k not in {"timing", "deadline_started_at"}
} == expected