mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cli): say why the proxy failed to start
Two startup failures left the user digging for the cause. headroom proxy on a busy port printed the full banner ending in 'Press Ctrl+C to stop', then buried uvicorn's bind error among startup and shutdown log lines. It now checks the port before the banner and exits with one line naming the port, whether a Headroom proxy holds it, and --port. The probe mirrors uvicorn's own bind (SO_REUSEADDR, every resolved address for one worker, one socket for --workers > 1); a matrix of listener/host/workers cases on macOS and Linux matched uvicorn's real bind result. Windows and errors other than EADDRINUSE are left to uvicorn as before. A conftest guard keeps CLI tests that stub run_server on 8787 from depending on the developer's own proxy. headroom wrap, when the proxy died during startup, quoted the last 500 characters of the stdio log. The proxy logs its shutdown summary after the error, and the log is appended across runs, so that tail was the summary box or an earlier run, and the path was never shown. It now reads only this run's output, quotes from the first ERROR/CRITICAL/Traceback line, and names the full log (also on timeout). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assisted-by: claude-code:claude-opus-5-5 Assisted-by: codex:gpt-5.6-luna Headline-Session: 019feec3-e5a5-70c0-8eca-4cfb2f02be16 Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e Headline-Session: da826546-b901-410f-9822-7e3e61e8f4ce Headline-AI-Share: 31% strict / 31% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9815bbb5f
commit
03288c0507
@@ -1,8 +1,10 @@
|
||||
"""Proxy server CLI commands."""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import socket
|
||||
import sys
|
||||
import warnings
|
||||
from importlib.util import find_spec
|
||||
@@ -205,6 +207,66 @@ def _get_env_float_optional(name: str) -> float | None:
|
||||
raise click.ClickException(f"{name} must be a number, got {val!r}") from None
|
||||
|
||||
|
||||
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
|
||||
"""Return the EADDRINUSE error uvicorn would hit binding ``host:port``, else None.
|
||||
|
||||
Mirrors uvicorn's own bind so the probe never refuses a bind uvicorn would
|
||||
accept: ``SO_REUSEADDR`` (both uvicorn paths set it on POSIX, so a port in
|
||||
TIME_WAIT after a restart is not "in use"); a single process binds every
|
||||
address ``host`` resolves to (asyncio ``create_server``), while
|
||||
``--workers > 1`` binds one AF_INET/AF_INET6 socket in the parent.
|
||||
Windows ``SO_REUSEADDR`` semantics differ, so Windows is left to uvicorn,
|
||||
as is any error other than EADDRINUSE (unresolvable host, address not on
|
||||
this machine); uvicorn reports those as before.
|
||||
"""
|
||||
if sys.platform == "win32":
|
||||
return None
|
||||
if workers > 1:
|
||||
family = socket.AF_INET6 if ":" in host else socket.AF_INET
|
||||
targets: list[tuple[Any, Any]] = [(family, (host, port))]
|
||||
else:
|
||||
try:
|
||||
infos = socket.getaddrinfo(
|
||||
host or None, port, type=socket.SOCK_STREAM, flags=socket.AI_PASSIVE
|
||||
)
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
targets = [(info[0], info[4]) for info in infos]
|
||||
for family, address in targets:
|
||||
try:
|
||||
with socket.socket(family, socket.SOCK_STREAM) as probe:
|
||||
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
if family == socket.AF_INET6 and workers <= 1:
|
||||
# asyncio binds v4 and v6 separately; match it.
|
||||
probe.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
|
||||
probe.bind(address)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EADDRINUSE:
|
||||
return exc
|
||||
return None
|
||||
|
||||
|
||||
def _refuse_busy_port(host: str, port: int, *, workers: int) -> None:
|
||||
"""Fail before the banner when ``host:port`` is already taken.
|
||||
|
||||
Without this the banner prints "Press Ctrl+C to stop", then uvicorn's bind
|
||||
error is buried among startup and shutdown log lines.
|
||||
"""
|
||||
error = _tcp_port_in_use(host, port, workers=workers)
|
||||
if error is None:
|
||||
return
|
||||
from headroom.cli.port_discovery import probe_headroom_proxy
|
||||
|
||||
if probe_headroom_proxy(port):
|
||||
holder = "a running Headroom proxy (reuse it, or stop it first)"
|
||||
else:
|
||||
holder = f"another process ({error.strerror or error})"
|
||||
raise click.ClickException(
|
||||
f"Port {port} on {host} is already in use by {holder}. "
|
||||
"Start this proxy on another port with --port N (env: HEADROOM_PORT)."
|
||||
)
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option(
|
||||
"--port",
|
||||
@@ -1695,6 +1757,8 @@ Memory (Multi-Provider):
|
||||
_bind = evaluate_bind_policy(config.host, config.proxy_token)
|
||||
if _bind.refused:
|
||||
raise click.ClickException(_bind.message())
|
||||
if not config.uds:
|
||||
_refuse_busy_port(config.host, config.port, workers=workers)
|
||||
_auth_on = _bind.token_configured
|
||||
_open_bind_note = (
|
||||
f" · WARNING open bind, /v1/* UNAUTHENTICATED (acknowledged via {OPEN_BIND_ACK_ENV}=1)"
|
||||
|
||||
+42
-7
@@ -709,6 +709,35 @@ def _get_proxy_stdio_log_path(port: int | None = None) -> Path:
|
||||
return _get_log_path(port).with_name(_paths.proxy_stdio_log_path(port).name)
|
||||
|
||||
|
||||
# A proxy that dies during startup logs its shutdown summary after the error,
|
||||
# so the end of the log is the summary box, not the cause. Quote from the first
|
||||
# error-looking line instead.
|
||||
_PROXY_STARTUP_ERROR_RE = re.compile(r"\b(?:ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
|
||||
_PROXY_STARTUP_EXCERPT_CHARS = 500
|
||||
|
||||
|
||||
def _proxy_startup_failure_excerpt(stdio_log_path: Path, start: int) -> str:
|
||||
"""Return the part of this run's stdio log that best explains a startup exit.
|
||||
|
||||
Reads only what was written after byte offset ``start`` (the log is shared
|
||||
across runs). Prefers the first ERROR/CRITICAL/Traceback line onwards, and
|
||||
falls back to the last few hundred characters.
|
||||
"""
|
||||
try:
|
||||
with open(stdio_log_path, "rb") as fh:
|
||||
fh.seek(start)
|
||||
output = fh.read().decode("utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
return "(no log output)"
|
||||
if not output:
|
||||
return "(no log output)"
|
||||
match = _PROXY_STARTUP_ERROR_RE.search(output)
|
||||
if match is None:
|
||||
return output[-_PROXY_STARTUP_EXCERPT_CHARS:]
|
||||
line_start = output.rfind("\n", 0, match.start()) + 1
|
||||
return output[line_start : line_start + _PROXY_STARTUP_EXCERPT_CHARS]
|
||||
|
||||
|
||||
def _start_proxy(
|
||||
port: int,
|
||||
*,
|
||||
@@ -790,6 +819,12 @@ def _start_proxy(
|
||||
timeout_seconds = _resolve_wrap_proxy_timeout_seconds()
|
||||
log_path = _get_log_path(port)
|
||||
stdio_log_path = _get_proxy_stdio_log_path(port)
|
||||
# The stdio log is appended to across runs; remember where this run starts
|
||||
# so a startup failure quotes this run's output, not an earlier one.
|
||||
try:
|
||||
stdio_log_start = stdio_log_path.stat().st_size
|
||||
except OSError:
|
||||
stdio_log_start = 0
|
||||
stdio_log_file = open(stdio_log_path, "a", encoding="utf-8") # noqa: SIM115
|
||||
|
||||
# Ensure proxy subprocess uses UTF-8 (Windows defaults to cp1252)
|
||||
@@ -910,17 +945,17 @@ def _start_proxy(
|
||||
return proc
|
||||
# Check if process died
|
||||
if proc.poll() is not None:
|
||||
# Read last few lines of log for error context
|
||||
try:
|
||||
tail = _read_text(stdio_log_path)[-500:]
|
||||
except Exception:
|
||||
tail = "(no log output)"
|
||||
raise RuntimeError(f"Proxy exited with code {proc.returncode}: {tail}")
|
||||
excerpt = _proxy_startup_failure_excerpt(stdio_log_path, stdio_log_start)
|
||||
raise RuntimeError(
|
||||
f"Proxy exited with code {proc.returncode}: {excerpt}\n"
|
||||
f" Full log: {stdio_log_path}"
|
||||
)
|
||||
|
||||
proc.kill()
|
||||
raise RuntimeError(
|
||||
f"Proxy failed to start on port {port} within {timeout_seconds} seconds. "
|
||||
f"Set {_WRAP_PROXY_TIMEOUT_ENV} to a larger number of seconds for slow startup."
|
||||
f"Set {_WRAP_PROXY_TIMEOUT_ENV} to a larger number of seconds for slow startup.\n"
|
||||
f" Full log: {stdio_log_path}"
|
||||
)
|
||||
finally:
|
||||
stdio_log_file.close()
|
||||
|
||||
@@ -534,6 +534,7 @@ markers = [
|
||||
"real_llm: tests that hit real LLM APIs; skipped unless explicitly enabled",
|
||||
"live: opt-in multi-turn tests that hit real upstream APIs; require provider keys",
|
||||
"proxy_dependency_gate: exercises ensure_proxy_dependencies() without mocking",
|
||||
"proxy_port_preflight: exercises the proxy busy-port check against real sockets",
|
||||
"windows_newline: newline-contract tests (#3698); also run on windows-latest in CI",
|
||||
]
|
||||
|
||||
|
||||
@@ -63,6 +63,22 @@ def _skip_proxy_dependency_gate_unless_exercised(
|
||||
monkeypatch.setattr(proxy, "ensure_proxy_dependencies", lambda: None)
|
||||
|
||||
|
||||
# `headroom proxy` refuses to start when its port is already bound. Tests that
|
||||
# invoke it with a stubbed run_server use the default 8787, which a developer's
|
||||
# own running proxy holds, so the check is off unless a test exercises it.
|
||||
@pytest.fixture(autouse=True)
|
||||
def _skip_proxy_port_preflight_unless_exercised(
|
||||
request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
if request.node.get_closest_marker("proxy_port_preflight") is not None:
|
||||
return
|
||||
try:
|
||||
from headroom.cli import proxy
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
monkeypatch.setattr(proxy, "_refuse_busy_port", lambda *args, **kwargs: None)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _scrub_developer_headroom_env(monkeypatch, tmp_path):
|
||||
for key in list(os.environ):
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""`headroom proxy` refuses a busy port before printing its banner.
|
||||
|
||||
Without the check the banner ends "Press Ctrl+C to stop", then uvicorn's bind
|
||||
error is buried among startup and shutdown log lines.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from headroom.cli.main import main
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.proxy_port_preflight,
|
||||
pytest.mark.skipif(sys.platform == "win32", reason="the check is POSIX-only"),
|
||||
]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _listening(host: str = "127.0.0.1") -> Iterator[int]:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.bind((host, 0))
|
||||
sock.listen(1)
|
||||
try:
|
||||
yield sock.getsockname()[1]
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
def _invoke_proxy(*args: str) -> tuple[object, list[object]]:
|
||||
calls: list[object] = []
|
||||
with (
|
||||
patch("headroom.proxy.server.run_server", lambda config, **kw: calls.append(config)),
|
||||
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=False),
|
||||
):
|
||||
result = CliRunner().invoke(main, ["proxy", *args])
|
||||
return result, calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workers", ["1", "2"])
|
||||
def test_busy_port_fails_before_banner_with_actionable_message(workers: str) -> None:
|
||||
with _listening() as port:
|
||||
result, calls = _invoke_proxy("--port", str(port), "--workers", workers)
|
||||
|
||||
assert result.exit_code == 1, result.output
|
||||
assert f"Port {port} on 127.0.0.1 is already in use by another process" in result.output
|
||||
assert "--port N" in result.output
|
||||
assert "HEADROOM PROXY" not in result.output
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_busy_port_held_by_headroom_says_so() -> None:
|
||||
with _listening() as port:
|
||||
with (
|
||||
patch("headroom.proxy.server.run_server", lambda config, **kw: None),
|
||||
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=True),
|
||||
):
|
||||
result = CliRunner().invoke(main, ["proxy", "--port", str(port)])
|
||||
|
||||
assert result.exit_code == 1, result.output
|
||||
assert "already in use by a running Headroom proxy" in result.output
|
||||
|
||||
|
||||
def test_free_port_starts_normally() -> None:
|
||||
port = _free_port()
|
||||
result, calls = _invoke_proxy("--port", str(port))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "HEADROOM PROXY" in result.output
|
||||
assert len(calls) == 1
|
||||
@@ -203,10 +203,15 @@ class TestCLIWrapProxyTimeout:
|
||||
monkeypatch.setattr(wrap_mod, "_get_log_path", lambda port=None: tmp_path / "proxy.log")
|
||||
monkeypatch.setattr(wrap_mod, "_check_proxy", lambda _port: False)
|
||||
monkeypatch.setattr(wrap_mod.time, "sleep", lambda _seconds: None)
|
||||
monkeypatch.setattr(wrap_mod.subprocess, "Popen", lambda *args, **kwargs: fake_proc)
|
||||
|
||||
def popen_writing_stdio(*args, **kwargs):
|
||||
kwargs["stdout"].write("proxy stdio startup output\n")
|
||||
kwargs["stdout"].flush()
|
||||
return fake_proc
|
||||
|
||||
monkeypatch.setattr(wrap_mod.subprocess, "Popen", popen_writing_stdio)
|
||||
|
||||
(tmp_path / "proxy.log").write_text("canonical runtime log output")
|
||||
(tmp_path / "proxy-stdio-8787.log").write_text("proxy stdio startup output")
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
wrap_mod._start_proxy(8787, agent_type="codex")
|
||||
@@ -216,6 +221,52 @@ class TestCLIWrapProxyTimeout:
|
||||
assert "proxy stdio startup output" in message
|
||||
assert "canonical runtime log output" not in message
|
||||
|
||||
def test_start_proxy_exit_quotes_this_runs_error_not_summary_or_old_runs(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
"""The startup error names the cause and the log path.
|
||||
|
||||
The proxy logs its shutdown summary after the error, and the stdio log
|
||||
is appended across runs, so the old ``[-500:]`` tail quoted the summary
|
||||
box (or an earlier run) and never said where the full log was.
|
||||
"""
|
||||
fake_proc = _FakeProxyProcess()
|
||||
fake_proc.returncode = 1
|
||||
fake_proc.poll = lambda: fake_proc.returncode
|
||||
|
||||
monkeypatch.setenv(wrap_mod._WRAP_PROXY_TIMEOUT_ENV, "2")
|
||||
monkeypatch.setattr(wrap_mod, "_get_log_path", lambda port=None: tmp_path / "proxy.log")
|
||||
monkeypatch.setattr(wrap_mod, "_check_proxy", lambda _port: False)
|
||||
monkeypatch.setattr(wrap_mod.time, "sleep", lambda _seconds: None)
|
||||
|
||||
stdio_log = tmp_path / "proxy-stdio-8787.log"
|
||||
stdio_log.write_text("ERROR: stale failure from an earlier run\n")
|
||||
summary = "".join(
|
||||
f"2026-10-09 16:19:23,665 - headroom.proxy - INFO - Summary line {i}: 0\n"
|
||||
for i in range(20)
|
||||
)
|
||||
|
||||
def popen_failing_to_bind(*args, **kwargs):
|
||||
kwargs["stdout"].write(
|
||||
"2026-10-09 16:19:23,659 - headroom.proxy - INFO - Local telemetry: DISABLED\n"
|
||||
"ERROR: [Errno 48] error while attempting to bind on address "
|
||||
"('127.0.0.1', 8787): address already in use\n" + summary
|
||||
)
|
||||
kwargs["stdout"].flush()
|
||||
return fake_proc
|
||||
|
||||
monkeypatch.setattr(wrap_mod.subprocess, "Popen", popen_failing_to_bind)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
wrap_mod._start_proxy(8787, agent_type="codex")
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "Proxy exited with code 1" in message
|
||||
assert "[Errno 48] error while attempting to bind" in message
|
||||
assert "stale failure from an earlier run" not in message
|
||||
assert "Local telemetry" not in message
|
||||
assert f"Full log: {stdio_log}" in message
|
||||
|
||||
def test_timeout_error_names_configured_timeout_and_env_var(self, monkeypatch, tmp_path):
|
||||
fake_proc = _FakeProxyProcess()
|
||||
|
||||
@@ -231,6 +282,7 @@ class TestCLIWrapProxyTimeout:
|
||||
message = str(excinfo.value)
|
||||
assert "within 2 seconds" in message
|
||||
assert wrap_mod._WRAP_PROXY_TIMEOUT_ENV in message
|
||||
assert f"Full log: {tmp_path / 'proxy-stdio-8787.log'}" in message
|
||||
assert fake_proc.killed is True
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user