fix(cache): hand the exit signals back when the proxy lifespan ends

A lifespan run without uvicorn has no capture_signals to restore the
handlers, so the exit-signal wrapper outlived its loop. A later signal
then hit call_soon_threadsafe on the closed loop and raised before the
original handler ran. shutdown() now restores each handler that is still
the wrapper, leaves one another owner installed alone, and turns any
wrapper it cannot remove into a pass-through.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-11-09 10:04:48 +01:00
co-authored by Claude Opus 5.5
parent 2fd4854edd
commit 4783339be9
2 changed files with 132 additions and 5 deletions
+33 -5
View File
@@ -602,7 +602,7 @@ class CompressionQuarantinedError(asyncio.TimeoutError):
_MULTI_WORKER_CONFIG_ENV = "HEADROOM_PROXY_CONFIG_JSON"
def _save_prefix_state_on_exit_signal(store: Any, path: str) -> None:
def _save_prefix_state_on_exit_signal(store: Any, path: str) -> Callable[[], None]:
"""Save prefix-tracker state as soon as an exit signal arrives.
``HeadroomProxy.shutdown`` saves it too, but uvicorn runs the lifespan
@@ -614,10 +614,16 @@ def _save_prefix_state_on_exit_signal(store: Any, path: str) -> None:
save still writes the newer state. The save runs on the event loop, never
inside the signal handler, so it cannot read a tracker that a paused
coroutine was midway through updating.
Returns the undo, for the lifespan's shutdown: it restores each handler
that is still this wrapper (one another owner installed since is left
alone) and turns the wrapper into a pass-through, so a wrapper that could
not be removed never schedules onto this lifespan's closed loop.
"""
if threading.current_thread() is not threading.main_thread():
return # signals are only delivered to the main thread
return lambda: None # signals are only delivered to the main thread
loop = asyncio.get_running_loop()
active = True
saved = False
def save() -> None:
@@ -628,18 +634,34 @@ def _save_prefix_state_on_exit_signal(store: Any, path: str) -> None:
def chained(previous: Callable[[int, Any], Any]) -> Callable[[int, Any], None]:
def handler(signum: int, frame: Any) -> None:
loop.call_soon_threadsafe(save)
if active:
with contextlib.suppress(RuntimeError): # loop closed without a shutdown
loop.call_soon_threadsafe(save)
previous(signum, frame)
return handler
installed: list[tuple[int, Any, Callable[[int, Any], None]]] = []
for sig in (signal.SIGINT, signal.SIGTERM, getattr(signal, "SIGBREAK", None)):
if sig is None:
continue
previous = signal.getsignal(sig)
if not callable(previous):
continue # nobody installed a handler: keep the default exit
signal.signal(sig, chained(previous))
handler = chained(previous)
signal.signal(sig, handler)
installed.append((sig, previous, handler))
def undo() -> None:
nonlocal active
active = False
if threading.current_thread() is not threading.main_thread():
return
for sig, previous, handler in installed:
if signal.getsignal(sig) is handler:
signal.signal(sig, previous)
return undo
# Env var that opts out of the Rust core deployment smoke test (Hotfix-A0).
@@ -1270,6 +1292,7 @@ class HeadroomProxy(
self.shared_compress_sessions = open_shared_sessions()
# Set by startup() when HEADROOM_PREFIX_STATE_FILE opts in.
self._prefix_state_file: str | None = None
self._undo_exit_signal_save: Callable[[], None] | None = None
# Compression cache store for token mode (session-scoped). The dict
# itself is mutated under `_compression_caches_lock`; the per-session
@@ -2168,7 +2191,9 @@ class HeadroomProxy(
self._prefix_state_file = None
if self._prefix_state_file:
self.session_tracker_store.load_state(self._prefix_state_file)
_save_prefix_state_on_exit_signal(self.session_tracker_store, self._prefix_state_file)
self._undo_exit_signal_save = _save_prefix_state_on_exit_signal(
self.session_tracker_store, self._prefix_state_file
)
self.pipeline_extensions.emit(
PipelineStage.PRE_START,
operation="proxy.startup",
@@ -2519,6 +2544,9 @@ class HeadroomProxy(
# First, before any await can hit the shutdown step timeout.
if self._prefix_state_file:
self.session_tracker_store.save_state(self._prefix_state_file)
if self._undo_exit_signal_save is not None:
self._undo_exit_signal_save()
self._undo_exit_signal_save = None
if self.http_client_h1 and self.http_client_h1 is not self.http_client:
await self.http_client_h1.aclose()
self.http_client_h1 = None
+99
View File
@@ -646,3 +646,102 @@ def test_off_the_main_thread_no_handler_is_installed(tmp_path: Path) -> None:
worker.join()
assert errors == []
assert signal.getsignal(signal.SIGTERM) is before
def _lifespan_proxy() -> Any:
from headroom.proxy.server import HeadroomProxy
proxy = HeadroomProxy(
ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
cost_tracking_enabled=False,
code_aware_enabled=False,
)
)
proxy.anthropic_pipeline.transforms = []
proxy.openai_pipeline.transforms = []
return proxy
@pytest.mark.usefixtures("_restore_exit_handlers")
def test_repeated_proxy_lifespans_hand_the_signal_back(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A lifespan run without uvicorn (an embedding app, tests) has no
capture_signals to restore handlers, so shutdown must: after two lifespans
on separate loops, SIGTERM reaches the app's own handler, not a wrapper
bound to a closed loop."""
import asyncio
import signal
monkeypatch.setenv(PREFIX_STATE_FILE_ENV, str(tmp_path / "state.json"))
calls: list[int] = []
def app_handler(signum: int, frame: Any) -> None:
calls.append(signum)
signal.signal(signal.SIGTERM, app_handler)
async def lifespan() -> None:
proxy = _lifespan_proxy()
await proxy.startup()
assert signal.getsignal(signal.SIGTERM) is not app_handler
await proxy.shutdown()
asyncio.run(lifespan())
asyncio.run(lifespan())
assert signal.getsignal(signal.SIGTERM) is app_handler
signal.raise_signal(signal.SIGTERM)
assert calls == [signal.SIGTERM]
@pytest.mark.usefixtures("_restore_exit_handlers")
def test_a_wrapper_left_behind_after_its_loop_closed_passes_through(tmp_path: Path) -> None:
"""Overlapping lifespans: the first one's undo cannot remove its wrapper
while the second wraps it. Once the first is shut down and its loop closed,
a signal still reaches the app handler and saves nothing."""
import asyncio
import signal
from headroom.proxy.server import _save_prefix_state_on_exit_signal
calls: list[Any] = []
signal.signal(signal.SIGTERM, lambda signum, frame: calls.append(("app", signum)))
first = asyncio.new_event_loop()
second = asyncio.new_event_loop()
async def install() -> Any:
return _save_prefix_state_on_exit_signal(_RecordingStore(calls), str(tmp_path / "s.json"))
undo_first = first.run_until_complete(install())
undo_second = second.run_until_complete(install())
undo_first() # the second wrapper is current: leave it in place
first.close()
undo_second() # restores the first wrapper, now inert
second.close()
signal.raise_signal(signal.SIGTERM)
assert calls == [("app", signal.SIGTERM)]
@pytest.mark.usefixtures("_restore_exit_handlers")
def test_undo_leaves_a_handler_installed_later_alone(tmp_path: Path) -> None:
import asyncio
import signal
from headroom.proxy.server import _save_prefix_state_on_exit_signal
signal.signal(signal.SIGTERM, lambda signum, frame: None)
def later(signum: int, frame: Any) -> None:
pass
async def serve() -> None:
undo = _save_prefix_state_on_exit_signal(_RecordingStore([]), str(tmp_path / "s.json"))
signal.signal(signal.SIGTERM, later)
undo()
asyncio.run(serve())
assert signal.getsignal(signal.SIGTERM) is later