fix(cli): identify the busy-port owner on the requested host; bound the log read

- headroom proxy --host ::1 on a busy port probed 127.0.0.1 for Headroom, so a
  Headroom proxy on the IPv4 port made an unrelated IPv6 listener read as
  "a running Headroom proxy". The probe now targets the address that is
  actually busy (wildcards via loopback of the same family, IPv6 bracketed).
  When that address cannot be probed (scoped IPv6, unresolvable name) the
  message is the generic "already in use (<reason>)".
- The wrap startup excerpt read this run's whole stdio log. It now reads at
  most the first 64 KiB written by this run; the startup error precedes the
  shutdown summary.

Refs #4136

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assisted-by: claude-code:claude-opus-5
Assisted-by: claude-code:claude-opus-5-5
Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e
Headline-Session: 76c8edeb-416b-4be7-9896-43a71880e563
Headline-Session: 88f3a7f6-986c-4fa8-ab24-0c064dbb438a
Headline-AI-Share: 10% strict / 10% weighted
This commit is contained in:
Tejas Chopra
2026-10-09 18:29:54 -07:00
co-authored by Claude Opus 5.5
parent 03288c0507
commit c9868420b8
5 changed files with 133 additions and 16 deletions
+7 -4
View File
@@ -120,16 +120,19 @@ def candidate_ports(
return seen[:_MAX_CANDIDATES]
def probe_headroom_proxy(port: int, timeout: float = _PROBE_TIMEOUT_SECONDS) -> bool:
"""True when a Headroom proxy answers ``/livez`` on loopback ``port``.
def probe_headroom_proxy(
port: int, timeout: float = _PROBE_TIMEOUT_SECONDS, *, host: str = "127.0.0.1"
) -> bool:
"""True when a Headroom proxy answers ``/livez`` on ``host:port`` (loopback by default).
Requires the ``service: headroom-proxy`` identity (or the legacy
``alive``+``version`` shape) so an unrelated local server on the port is
never mistaken for Headroom.
never mistaken for Headroom. ``host`` is a numeric IPv4 or IPv6 address.
"""
from headroom.install.health import probe_json
return is_headroom_livez(probe_json(f"http://127.0.0.1:{port}/livez", timeout=timeout))
netloc = f"[{host}]" if ":" in host else host
return is_headroom_livez(probe_json(f"http://{netloc}:{port}/livez", timeout=timeout))
def is_headroom_livez(payload: Mapping[str, Any] | None) -> bool:
+43 -9
View File
@@ -1,6 +1,7 @@
"""Proxy server CLI commands."""
import errno
import ipaddress
import logging
import os
import platform
@@ -207,8 +208,10 @@ def _get_env_float_optional(name: str) -> float | None:
raise click.ClickException(f"{name} must be a number, got {val!r}") from None
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
"""Return the EADDRINUSE error uvicorn would hit binding ``host:port``, else None.
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> tuple[OSError, Any] | None:
"""Return ``(EADDRINUSE error, busy address)`` uvicorn would hit binding ``host:port``.
Returns None when the bind would succeed.
Mirrors uvicorn's own bind so the probe never refuses a bind uvicorn would
accept: ``SO_REUSEADDR`` (both uvicorn paths set it on POSIX, so a port in
@@ -242,27 +245,58 @@ def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
probe.bind(address)
except OSError as exc:
if exc.errno == errno.EADDRINUSE:
return exc
return exc, address
return None
_WILDCARD_PROBE_HOSTS = {"": "127.0.0.1", "0.0.0.0": "127.0.0.1", "::": "::1"}
def _probe_host_for(address: Any) -> str | None:
"""The numeric address to ask "is this Headroom?", or None when it can't be targeted.
A wildcard bind is reached through loopback of the same family. A
hostname (the ``--workers > 1`` path binds it unresolved) is resolved for
IPv4. A scoped IPv6 address cannot go in a probe URL.
"""
host = str(address[0])
if host in _WILDCARD_PROBE_HOSTS:
return _WILDCARD_PROBE_HOSTS[host]
if "%" in host:
return None
try:
ipaddress.ip_address(host)
except ValueError:
try:
infos = socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM)
except (OSError, UnicodeError):
return None
return str(infos[0][4][0]) if infos else None
return host
def _refuse_busy_port(host: str, port: int, *, workers: int) -> None:
"""Fail before the banner when ``host:port`` is already taken.
Without this the banner prints "Press Ctrl+C to stop", then uvicorn's bind
error is buried among startup and shutdown log lines.
"""
error = _tcp_port_in_use(host, port, workers=workers)
if error is None:
busy = _tcp_port_in_use(host, port, workers=workers)
if busy is None:
return
error, address = busy
from headroom.cli.port_discovery import probe_headroom_proxy
if probe_headroom_proxy(port):
holder = "a running Headroom proxy (reuse it, or stop it first)"
reason = error.strerror or str(error)
probe_host = _probe_host_for(address)
if probe_host is None:
holder = f"({reason})"
elif probe_headroom_proxy(port, host=probe_host):
holder = "by a running Headroom proxy (reuse it, or stop it first)"
else:
holder = f"another process ({error.strerror or error})"
holder = f"by another process ({reason})"
raise click.ClickException(
f"Port {port} on {host} is already in use by {holder}. "
f"Port {port} on {host} is already in use {holder}. "
"Start this proxy on another port with --port N (env: HEADROOM_PORT)."
)
+8 -3
View File
@@ -714,19 +714,24 @@ def _get_proxy_stdio_log_path(port: int | None = None) -> Path:
# error-looking line instead.
_PROXY_STARTUP_ERROR_RE = re.compile(r"\b(?:ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
_PROXY_STARTUP_EXCERPT_CHARS = 500
# The startup error comes before the shutdown summary, so the first 64 KiB of
# this run's output is enough; never read a runaway log whole.
_PROXY_STARTUP_READ_BYTES = 64 * 1024
def _proxy_startup_failure_excerpt(stdio_log_path: Path, start: int) -> str:
"""Return the part of this run's stdio log that best explains a startup exit.
Reads only what was written after byte offset ``start`` (the log is shared
across runs). Prefers the first ERROR/CRITICAL/Traceback line onwards, and
falls back to the last few hundred characters.
across runs), and at most the first ``_PROXY_STARTUP_READ_BYTES`` of it.
Prefers the first ERROR/CRITICAL/Traceback line onwards, and falls back to
the last few hundred characters read.
"""
try:
with open(stdio_log_path, "rb") as fh:
fh.seek(start)
output = fh.read().decode("utf-8", errors="replace").strip()
raw = fh.read(_PROXY_STARTUP_READ_BYTES)
output = raw.decode("utf-8", errors="replace").strip()
except OSError:
return "(no log output)"
if not output:
@@ -6,10 +6,13 @@ error is buried among startup and shutdown log lines.
from __future__ import annotations
import json
import socket
import sys
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, HTTPServer
from unittest.mock import patch
import pytest
@@ -81,3 +84,62 @@ def test_free_port_starts_normally() -> None:
assert result.exit_code == 0, result.output
assert "HEADROOM PROXY" in result.output
assert len(calls) == 1
def _ipv6_loopback_available() -> bool:
try:
with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock:
sock.bind(("::1", 0))
except OSError:
return False
return True
class _HeadroomLivez(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802 - http.server API
body = json.dumps({"service": "headroom-proxy", "alive": True}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args: object) -> None:
pass
@pytest.mark.skipif(not _ipv6_loopback_available(), reason="needs IPv6 loopback")
def test_ipv6_conflict_is_identified_on_the_requested_host() -> None:
"""A Headroom proxy on 127.0.0.1:P must not be reported as the owner of [::1]:P."""
headroom_v4 = HTTPServer(("127.0.0.1", 0), _HeadroomLivez)
port = headroom_v4.server_address[1]
threading.Thread(target=headroom_v4.serve_forever, daemon=True).start()
other_v6 = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
try:
other_v6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
other_v6.bind(("::1", port))
other_v6.listen(1)
with patch("headroom.proxy.server.run_server", lambda config, **kw: None):
v6 = CliRunner().invoke(main, ["proxy", "--host", "::1", "--port", str(port)])
finally:
other_v6.close()
headroom_v4.shutdown()
headroom_v4.server_close()
assert v6.exit_code == 1, v6.output
assert f"Port {port} on ::1 is already in use by another process" in v6.output
assert "Headroom proxy" not in v6.output
def test_unprobeable_host_gets_the_generic_message() -> None:
with _listening() as port:
with (
patch("headroom.proxy.server.run_server", lambda config, **kw: None),
patch("headroom.cli.proxy._probe_host_for", return_value=None),
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=True),
):
result = CliRunner().invoke(main, ["proxy", "--port", str(port)])
assert result.exit_code == 1, result.output
assert f"Port {port} on 127.0.0.1 is already in use (" in result.output
assert "Headroom proxy" not in result.output
+13
View File
@@ -267,6 +267,19 @@ class TestCLIWrapProxyTimeout:
assert "Local telemetry" not in message
assert f"Full log: {stdio_log}" in message
def test_startup_excerpt_reads_at_most_64_kib_of_this_run(self, tmp_path):
"""A runaway stdio log is never read whole: only this run's first 64 KiB."""
stdio_log = tmp_path / "proxy-stdio-8787.log"
earlier = "ERROR: an earlier run\n" * 1000
noise = "INFO noise line\n" * 8000 # ~128 KiB, no error line
stdio_log.write_text(earlier + noise + "END-OF-RUN-MARKER\n")
excerpt = wrap_mod._proxy_startup_failure_excerpt(stdio_log, len(earlier))
assert "earlier run" not in excerpt
assert "END-OF-RUN-MARKER" not in excerpt
assert excerpt.endswith("INFO noise line")
def test_timeout_error_names_configured_timeout_and_env_var(self, monkeypatch, tmp_path):
fake_proc = _FakeProxyProcess()