fix(cli): say why the proxy failed to start

Two startup failures left the user digging for the cause.

headroom proxy on a busy port printed the full banner ending in 'Press Ctrl+C to stop', then buried uvicorn's bind error among startup and shutdown log lines. It now checks the port before the banner and exits with one line naming the port, whether a Headroom proxy holds it, and --port. The probe mirrors uvicorn's own bind (SO_REUSEADDR, every resolved address for one worker, one socket for --workers > 1); a matrix of listener/host/workers cases on macOS and Linux matched uvicorn's real bind result. Windows and errors other than EADDRINUSE are left to uvicorn as before. A conftest guard keeps CLI tests that stub run_server on 8787 from depending on the developer's own proxy.

headroom wrap, when the proxy died during startup, quoted the last 500 characters of the stdio log. The proxy logs its shutdown summary after the error, and the log is appended across runs, so that tail was the summary box or an earlier run, and the path was never shown. It now reads only this run's output, quotes from the first ERROR/CRITICAL/Traceback line, and names the full log (also on timeout).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assisted-by: claude-code:claude-opus-5-5
Assisted-by: codex:gpt-5.6-luna
Headline-Session: 019feec3-e5a5-70c0-8eca-4cfb2f02be16
Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e
Headline-Session: da826546-b901-410f-9822-7e3e61e8f4ce
Headline-AI-Share: 31% strict / 31% weighted
This commit is contained in:
Tejas Chopra
2026-10-09 17:13:21 -07:00
co-authored by Claude Opus 5.5
parent a9815bbb5f
commit 03288c0507
6 changed files with 260 additions and 9 deletions
+64
View File
@@ -1,8 +1,10 @@
"""Proxy server CLI commands."""
import errno
import logging
import os
import platform
import socket
import sys
import warnings
from importlib.util import find_spec
@@ -205,6 +207,66 @@ def _get_env_float_optional(name: str) -> float | None:
raise click.ClickException(f"{name} must be a number, got {val!r}") from None
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
"""Return the EADDRINUSE error uvicorn would hit binding ``host:port``, else None.
Mirrors uvicorn's own bind so the probe never refuses a bind uvicorn would
accept: ``SO_REUSEADDR`` (both uvicorn paths set it on POSIX, so a port in
TIME_WAIT after a restart is not "in use"); a single process binds every
address ``host`` resolves to (asyncio ``create_server``), while
``--workers > 1`` binds one AF_INET/AF_INET6 socket in the parent.
Windows ``SO_REUSEADDR`` semantics differ, so Windows is left to uvicorn,
as is any error other than EADDRINUSE (unresolvable host, address not on
this machine); uvicorn reports those as before.
"""
if sys.platform == "win32":
return None
if workers > 1:
family = socket.AF_INET6 if ":" in host else socket.AF_INET
targets: list[tuple[Any, Any]] = [(family, (host, port))]
else:
try:
infos = socket.getaddrinfo(
host or None, port, type=socket.SOCK_STREAM, flags=socket.AI_PASSIVE
)
except (OSError, UnicodeError):
return None
targets = [(info[0], info[4]) for info in infos]
for family, address in targets:
try:
with socket.socket(family, socket.SOCK_STREAM) as probe:
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if family == socket.AF_INET6 and workers <= 1:
# asyncio binds v4 and v6 separately; match it.
probe.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
probe.bind(address)
except OSError as exc:
if exc.errno == errno.EADDRINUSE:
return exc
return None
def _refuse_busy_port(host: str, port: int, *, workers: int) -> None:
"""Fail before the banner when ``host:port`` is already taken.
Without this the banner prints "Press Ctrl+C to stop", then uvicorn's bind
error is buried among startup and shutdown log lines.
"""
error = _tcp_port_in_use(host, port, workers=workers)
if error is None:
return
from headroom.cli.port_discovery import probe_headroom_proxy
if probe_headroom_proxy(port):
holder = "a running Headroom proxy (reuse it, or stop it first)"
else:
holder = f"another process ({error.strerror or error})"
raise click.ClickException(
f"Port {port} on {host} is already in use by {holder}. "
"Start this proxy on another port with --port N (env: HEADROOM_PORT)."
)
@main.command()
@click.option(
"--port",
@@ -1695,6 +1757,8 @@ Memory (Multi-Provider):
_bind = evaluate_bind_policy(config.host, config.proxy_token)
if _bind.refused:
raise click.ClickException(_bind.message())
if not config.uds:
_refuse_busy_port(config.host, config.port, workers=workers)
_auth_on = _bind.token_configured
_open_bind_note = (
f" · WARNING open bind, /v1/* UNAUTHENTICATED (acknowledged via {OPEN_BIND_ACK_ENV}=1)"
+42 -7
View File
@@ -709,6 +709,35 @@ def _get_proxy_stdio_log_path(port: int | None = None) -> Path:
return _get_log_path(port).with_name(_paths.proxy_stdio_log_path(port).name)
# A proxy that dies during startup logs its shutdown summary after the error,
# so the end of the log is the summary box, not the cause. Quote from the first
# error-looking line instead.
_PROXY_STARTUP_ERROR_RE = re.compile(r"\b(?:ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
_PROXY_STARTUP_EXCERPT_CHARS = 500
def _proxy_startup_failure_excerpt(stdio_log_path: Path, start: int) -> str:
"""Return the part of this run's stdio log that best explains a startup exit.
Reads only what was written after byte offset ``start`` (the log is shared
across runs). Prefers the first ERROR/CRITICAL/Traceback line onwards, and
falls back to the last few hundred characters.
"""
try:
with open(stdio_log_path, "rb") as fh:
fh.seek(start)
output = fh.read().decode("utf-8", errors="replace").strip()
except OSError:
return "(no log output)"
if not output:
return "(no log output)"
match = _PROXY_STARTUP_ERROR_RE.search(output)
if match is None:
return output[-_PROXY_STARTUP_EXCERPT_CHARS:]
line_start = output.rfind("\n", 0, match.start()) + 1
return output[line_start : line_start + _PROXY_STARTUP_EXCERPT_CHARS]
def _start_proxy(
port: int,
*,
@@ -790,6 +819,12 @@ def _start_proxy(
timeout_seconds = _resolve_wrap_proxy_timeout_seconds()
log_path = _get_log_path(port)
stdio_log_path = _get_proxy_stdio_log_path(port)
# The stdio log is appended to across runs; remember where this run starts
# so a startup failure quotes this run's output, not an earlier one.
try:
stdio_log_start = stdio_log_path.stat().st_size
except OSError:
stdio_log_start = 0
stdio_log_file = open(stdio_log_path, "a", encoding="utf-8") # noqa: SIM115
# Ensure proxy subprocess uses UTF-8 (Windows defaults to cp1252)
@@ -910,17 +945,17 @@ def _start_proxy(
return proc
# Check if process died
if proc.poll() is not None:
# Read last few lines of log for error context
try:
tail = _read_text(stdio_log_path)[-500:]
except Exception:
tail = "(no log output)"
raise RuntimeError(f"Proxy exited with code {proc.returncode}: {tail}")
excerpt = _proxy_startup_failure_excerpt(stdio_log_path, stdio_log_start)
raise RuntimeError(
f"Proxy exited with code {proc.returncode}: {excerpt}\n"
f" Full log: {stdio_log_path}"
)
proc.kill()
raise RuntimeError(
f"Proxy failed to start on port {port} within {timeout_seconds} seconds. "
f"Set {_WRAP_PROXY_TIMEOUT_ENV} to a larger number of seconds for slow startup."
f"Set {_WRAP_PROXY_TIMEOUT_ENV} to a larger number of seconds for slow startup.\n"
f" Full log: {stdio_log_path}"
)
finally:
stdio_log_file.close()
+1
View File
@@ -534,6 +534,7 @@ markers = [
"real_llm: tests that hit real LLM APIs; skipped unless explicitly enabled",
"live: opt-in multi-turn tests that hit real upstream APIs; require provider keys",
"proxy_dependency_gate: exercises ensure_proxy_dependencies() without mocking",
"proxy_port_preflight: exercises the proxy busy-port check against real sockets",
"windows_newline: newline-contract tests (#3698); also run on windows-latest in CI",
]
+16
View File
@@ -63,6 +63,22 @@ def _skip_proxy_dependency_gate_unless_exercised(
monkeypatch.setattr(proxy, "ensure_proxy_dependencies", lambda: None)
# `headroom proxy` refuses to start when its port is already bound. Tests that
# invoke it with a stubbed run_server use the default 8787, which a developer's
# own running proxy holds, so the check is off unless a test exercises it.
@pytest.fixture(autouse=True)
def _skip_proxy_port_preflight_unless_exercised(
request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
) -> None:
if request.node.get_closest_marker("proxy_port_preflight") is not None:
return
try:
from headroom.cli import proxy
except ModuleNotFoundError:
return
monkeypatch.setattr(proxy, "_refuse_busy_port", lambda *args, **kwargs: None)
@pytest.fixture(autouse=True)
def _scrub_developer_headroom_env(monkeypatch, tmp_path):
for key in list(os.environ):
@@ -0,0 +1,83 @@
"""`headroom proxy` refuses a busy port before printing its banner.
Without the check the banner ends "Press Ctrl+C to stop", then uvicorn's bind
error is buried among startup and shutdown log lines.
"""
from __future__ import annotations
import socket
import sys
from collections.abc import Iterator
from contextlib import contextmanager
from unittest.mock import patch
import pytest
from click.testing import CliRunner
from headroom.cli.main import main
pytestmark = [
pytest.mark.proxy_port_preflight,
pytest.mark.skipif(sys.platform == "win32", reason="the check is POSIX-only"),
]
@contextmanager
def _listening(host: str = "127.0.0.1") -> Iterator[int]:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((host, 0))
sock.listen(1)
try:
yield sock.getsockname()[1]
finally:
sock.close()
def _free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def _invoke_proxy(*args: str) -> tuple[object, list[object]]:
calls: list[object] = []
with (
patch("headroom.proxy.server.run_server", lambda config, **kw: calls.append(config)),
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=False),
):
result = CliRunner().invoke(main, ["proxy", *args])
return result, calls
@pytest.mark.parametrize("workers", ["1", "2"])
def test_busy_port_fails_before_banner_with_actionable_message(workers: str) -> None:
with _listening() as port:
result, calls = _invoke_proxy("--port", str(port), "--workers", workers)
assert result.exit_code == 1, result.output
assert f"Port {port} on 127.0.0.1 is already in use by another process" in result.output
assert "--port N" in result.output
assert "HEADROOM PROXY" not in result.output
assert calls == []
def test_busy_port_held_by_headroom_says_so() -> None:
with _listening() as port:
with (
patch("headroom.proxy.server.run_server", lambda config, **kw: None),
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=True),
):
result = CliRunner().invoke(main, ["proxy", "--port", str(port)])
assert result.exit_code == 1, result.output
assert "already in use by a running Headroom proxy" in result.output
def test_free_port_starts_normally() -> None:
port = _free_port()
result, calls = _invoke_proxy("--port", str(port))
assert result.exit_code == 0, result.output
assert "HEADROOM PROXY" in result.output
assert len(calls) == 1
+54 -2
View File
@@ -203,10 +203,15 @@ class TestCLIWrapProxyTimeout:
monkeypatch.setattr(wrap_mod, "_get_log_path", lambda port=None: tmp_path / "proxy.log")
monkeypatch.setattr(wrap_mod, "_check_proxy", lambda _port: False)
monkeypatch.setattr(wrap_mod.time, "sleep", lambda _seconds: None)
monkeypatch.setattr(wrap_mod.subprocess, "Popen", lambda *args, **kwargs: fake_proc)
def popen_writing_stdio(*args, **kwargs):
kwargs["stdout"].write("proxy stdio startup output\n")
kwargs["stdout"].flush()
return fake_proc
monkeypatch.setattr(wrap_mod.subprocess, "Popen", popen_writing_stdio)
(tmp_path / "proxy.log").write_text("canonical runtime log output")
(tmp_path / "proxy-stdio-8787.log").write_text("proxy stdio startup output")
with pytest.raises(RuntimeError) as excinfo:
wrap_mod._start_proxy(8787, agent_type="codex")
@@ -216,6 +221,52 @@ class TestCLIWrapProxyTimeout:
assert "proxy stdio startup output" in message
assert "canonical runtime log output" not in message
def test_start_proxy_exit_quotes_this_runs_error_not_summary_or_old_runs(
self, monkeypatch, tmp_path
):
"""The startup error names the cause and the log path.
The proxy logs its shutdown summary after the error, and the stdio log
is appended across runs, so the old ``[-500:]`` tail quoted the summary
box (or an earlier run) and never said where the full log was.
"""
fake_proc = _FakeProxyProcess()
fake_proc.returncode = 1
fake_proc.poll = lambda: fake_proc.returncode
monkeypatch.setenv(wrap_mod._WRAP_PROXY_TIMEOUT_ENV, "2")
monkeypatch.setattr(wrap_mod, "_get_log_path", lambda port=None: tmp_path / "proxy.log")
monkeypatch.setattr(wrap_mod, "_check_proxy", lambda _port: False)
monkeypatch.setattr(wrap_mod.time, "sleep", lambda _seconds: None)
stdio_log = tmp_path / "proxy-stdio-8787.log"
stdio_log.write_text("ERROR: stale failure from an earlier run\n")
summary = "".join(
f"2026-10-09 16:19:23,665 - headroom.proxy - INFO - Summary line {i}: 0\n"
for i in range(20)
)
def popen_failing_to_bind(*args, **kwargs):
kwargs["stdout"].write(
"2026-10-09 16:19:23,659 - headroom.proxy - INFO - Local telemetry: DISABLED\n"
"ERROR: [Errno 48] error while attempting to bind on address "
"('127.0.0.1', 8787): address already in use\n" + summary
)
kwargs["stdout"].flush()
return fake_proc
monkeypatch.setattr(wrap_mod.subprocess, "Popen", popen_failing_to_bind)
with pytest.raises(RuntimeError) as excinfo:
wrap_mod._start_proxy(8787, agent_type="codex")
message = str(excinfo.value)
assert "Proxy exited with code 1" in message
assert "[Errno 48] error while attempting to bind" in message
assert "stale failure from an earlier run" not in message
assert "Local telemetry" not in message
assert f"Full log: {stdio_log}" in message
def test_timeout_error_names_configured_timeout_and_env_var(self, monkeypatch, tmp_path):
fake_proc = _FakeProxyProcess()
@@ -231,6 +282,7 @@ class TestCLIWrapProxyTimeout:
message = str(excinfo.value)
assert "within 2 seconds" in message
assert wrap_mod._WRAP_PROXY_TIMEOUT_ENV in message
assert f"Full log: {tmp_path / 'proxy-stdio-8787.log'}" in message
assert fake_proc.killed is True