mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cli): identify the busy-port owner on the requested host; bound the log read
- headroom proxy --host ::1 on a busy port probed 127.0.0.1 for Headroom, so a Headroom proxy on the IPv4 port made an unrelated IPv6 listener read as "a running Headroom proxy". The probe now targets the address that is actually busy (wildcards via loopback of the same family, IPv6 bracketed). When that address cannot be probed (scoped IPv6, unresolvable name) the message is the generic "already in use (<reason>)". - The wrap startup excerpt read this run's whole stdio log. It now reads at most the first 64 KiB written by this run; the startup error precedes the shutdown summary. Refs #4136 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assisted-by: claude-code:claude-opus-5 Assisted-by: claude-code:claude-opus-5-5 Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e Headline-Session: 76c8edeb-416b-4be7-9896-43a71880e563 Headline-Session: 88f3a7f6-986c-4fa8-ab24-0c064dbb438a Headline-AI-Share: 10% strict / 10% weighted
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
03288c0507
commit
c9868420b8
@@ -120,16 +120,19 @@ def candidate_ports(
|
||||
return seen[:_MAX_CANDIDATES]
|
||||
|
||||
|
||||
def probe_headroom_proxy(port: int, timeout: float = _PROBE_TIMEOUT_SECONDS) -> bool:
|
||||
"""True when a Headroom proxy answers ``/livez`` on loopback ``port``.
|
||||
def probe_headroom_proxy(
|
||||
port: int, timeout: float = _PROBE_TIMEOUT_SECONDS, *, host: str = "127.0.0.1"
|
||||
) -> bool:
|
||||
"""True when a Headroom proxy answers ``/livez`` on ``host:port`` (loopback by default).
|
||||
|
||||
Requires the ``service: headroom-proxy`` identity (or the legacy
|
||||
``alive``+``version`` shape) so an unrelated local server on the port is
|
||||
never mistaken for Headroom.
|
||||
never mistaken for Headroom. ``host`` is a numeric IPv4 or IPv6 address.
|
||||
"""
|
||||
from headroom.install.health import probe_json
|
||||
|
||||
return is_headroom_livez(probe_json(f"http://127.0.0.1:{port}/livez", timeout=timeout))
|
||||
netloc = f"[{host}]" if ":" in host else host
|
||||
return is_headroom_livez(probe_json(f"http://{netloc}:{port}/livez", timeout=timeout))
|
||||
|
||||
|
||||
def is_headroom_livez(payload: Mapping[str, Any] | None) -> bool:
|
||||
|
||||
+43
-9
@@ -1,6 +1,7 @@
|
||||
"""Proxy server CLI commands."""
|
||||
|
||||
import errno
|
||||
import ipaddress
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
@@ -207,8 +208,10 @@ def _get_env_float_optional(name: str) -> float | None:
|
||||
raise click.ClickException(f"{name} must be a number, got {val!r}") from None
|
||||
|
||||
|
||||
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
|
||||
"""Return the EADDRINUSE error uvicorn would hit binding ``host:port``, else None.
|
||||
def _tcp_port_in_use(host: str, port: int, *, workers: int) -> tuple[OSError, Any] | None:
|
||||
"""Return ``(EADDRINUSE error, busy address)`` uvicorn would hit binding ``host:port``.
|
||||
|
||||
Returns None when the bind would succeed.
|
||||
|
||||
Mirrors uvicorn's own bind so the probe never refuses a bind uvicorn would
|
||||
accept: ``SO_REUSEADDR`` (both uvicorn paths set it on POSIX, so a port in
|
||||
@@ -242,27 +245,58 @@ def _tcp_port_in_use(host: str, port: int, *, workers: int) -> OSError | None:
|
||||
probe.bind(address)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EADDRINUSE:
|
||||
return exc
|
||||
return exc, address
|
||||
return None
|
||||
|
||||
|
||||
_WILDCARD_PROBE_HOSTS = {"": "127.0.0.1", "0.0.0.0": "127.0.0.1", "::": "::1"}
|
||||
|
||||
|
||||
def _probe_host_for(address: Any) -> str | None:
|
||||
"""The numeric address to ask "is this Headroom?", or None when it can't be targeted.
|
||||
|
||||
A wildcard bind is reached through loopback of the same family. A
|
||||
hostname (the ``--workers > 1`` path binds it unresolved) is resolved for
|
||||
IPv4. A scoped IPv6 address cannot go in a probe URL.
|
||||
"""
|
||||
host = str(address[0])
|
||||
if host in _WILDCARD_PROBE_HOSTS:
|
||||
return _WILDCARD_PROBE_HOSTS[host]
|
||||
if "%" in host:
|
||||
return None
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM)
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
return str(infos[0][4][0]) if infos else None
|
||||
return host
|
||||
|
||||
|
||||
def _refuse_busy_port(host: str, port: int, *, workers: int) -> None:
|
||||
"""Fail before the banner when ``host:port`` is already taken.
|
||||
|
||||
Without this the banner prints "Press Ctrl+C to stop", then uvicorn's bind
|
||||
error is buried among startup and shutdown log lines.
|
||||
"""
|
||||
error = _tcp_port_in_use(host, port, workers=workers)
|
||||
if error is None:
|
||||
busy = _tcp_port_in_use(host, port, workers=workers)
|
||||
if busy is None:
|
||||
return
|
||||
error, address = busy
|
||||
from headroom.cli.port_discovery import probe_headroom_proxy
|
||||
|
||||
if probe_headroom_proxy(port):
|
||||
holder = "a running Headroom proxy (reuse it, or stop it first)"
|
||||
reason = error.strerror or str(error)
|
||||
probe_host = _probe_host_for(address)
|
||||
if probe_host is None:
|
||||
holder = f"({reason})"
|
||||
elif probe_headroom_proxy(port, host=probe_host):
|
||||
holder = "by a running Headroom proxy (reuse it, or stop it first)"
|
||||
else:
|
||||
holder = f"another process ({error.strerror or error})"
|
||||
holder = f"by another process ({reason})"
|
||||
raise click.ClickException(
|
||||
f"Port {port} on {host} is already in use by {holder}. "
|
||||
f"Port {port} on {host} is already in use {holder}. "
|
||||
"Start this proxy on another port with --port N (env: HEADROOM_PORT)."
|
||||
)
|
||||
|
||||
|
||||
@@ -714,19 +714,24 @@ def _get_proxy_stdio_log_path(port: int | None = None) -> Path:
|
||||
# error-looking line instead.
|
||||
_PROXY_STARTUP_ERROR_RE = re.compile(r"\b(?:ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
|
||||
_PROXY_STARTUP_EXCERPT_CHARS = 500
|
||||
# The startup error comes before the shutdown summary, so the first 64 KiB of
|
||||
# this run's output is enough; never read a runaway log whole.
|
||||
_PROXY_STARTUP_READ_BYTES = 64 * 1024
|
||||
|
||||
|
||||
def _proxy_startup_failure_excerpt(stdio_log_path: Path, start: int) -> str:
|
||||
"""Return the part of this run's stdio log that best explains a startup exit.
|
||||
|
||||
Reads only what was written after byte offset ``start`` (the log is shared
|
||||
across runs). Prefers the first ERROR/CRITICAL/Traceback line onwards, and
|
||||
falls back to the last few hundred characters.
|
||||
across runs), and at most the first ``_PROXY_STARTUP_READ_BYTES`` of it.
|
||||
Prefers the first ERROR/CRITICAL/Traceback line onwards, and falls back to
|
||||
the last few hundred characters read.
|
||||
"""
|
||||
try:
|
||||
with open(stdio_log_path, "rb") as fh:
|
||||
fh.seek(start)
|
||||
output = fh.read().decode("utf-8", errors="replace").strip()
|
||||
raw = fh.read(_PROXY_STARTUP_READ_BYTES)
|
||||
output = raw.decode("utf-8", errors="replace").strip()
|
||||
except OSError:
|
||||
return "(no log output)"
|
||||
if not output:
|
||||
|
||||
@@ -6,10 +6,13 @@ error is buried among startup and shutdown log lines.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -81,3 +84,62 @@ def test_free_port_starts_normally() -> None:
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "HEADROOM PROXY" in result.output
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def _ipv6_loopback_available() -> bool:
|
||||
try:
|
||||
with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("::1", 0))
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class _HeadroomLivez(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802 - http.server API
|
||||
body = json.dumps({"service": "headroom-proxy", "alive": True}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args: object) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _ipv6_loopback_available(), reason="needs IPv6 loopback")
|
||||
def test_ipv6_conflict_is_identified_on_the_requested_host() -> None:
|
||||
"""A Headroom proxy on 127.0.0.1:P must not be reported as the owner of [::1]:P."""
|
||||
headroom_v4 = HTTPServer(("127.0.0.1", 0), _HeadroomLivez)
|
||||
port = headroom_v4.server_address[1]
|
||||
threading.Thread(target=headroom_v4.serve_forever, daemon=True).start()
|
||||
other_v6 = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
|
||||
try:
|
||||
other_v6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
|
||||
other_v6.bind(("::1", port))
|
||||
other_v6.listen(1)
|
||||
with patch("headroom.proxy.server.run_server", lambda config, **kw: None):
|
||||
v6 = CliRunner().invoke(main, ["proxy", "--host", "::1", "--port", str(port)])
|
||||
finally:
|
||||
other_v6.close()
|
||||
headroom_v4.shutdown()
|
||||
headroom_v4.server_close()
|
||||
|
||||
assert v6.exit_code == 1, v6.output
|
||||
assert f"Port {port} on ::1 is already in use by another process" in v6.output
|
||||
assert "Headroom proxy" not in v6.output
|
||||
|
||||
|
||||
def test_unprobeable_host_gets_the_generic_message() -> None:
|
||||
with _listening() as port:
|
||||
with (
|
||||
patch("headroom.proxy.server.run_server", lambda config, **kw: None),
|
||||
patch("headroom.cli.proxy._probe_host_for", return_value=None),
|
||||
patch("headroom.cli.port_discovery.probe_headroom_proxy", return_value=True),
|
||||
):
|
||||
result = CliRunner().invoke(main, ["proxy", "--port", str(port)])
|
||||
|
||||
assert result.exit_code == 1, result.output
|
||||
assert f"Port {port} on 127.0.0.1 is already in use (" in result.output
|
||||
assert "Headroom proxy" not in result.output
|
||||
|
||||
@@ -267,6 +267,19 @@ class TestCLIWrapProxyTimeout:
|
||||
assert "Local telemetry" not in message
|
||||
assert f"Full log: {stdio_log}" in message
|
||||
|
||||
def test_startup_excerpt_reads_at_most_64_kib_of_this_run(self, tmp_path):
|
||||
"""A runaway stdio log is never read whole: only this run's first 64 KiB."""
|
||||
stdio_log = tmp_path / "proxy-stdio-8787.log"
|
||||
earlier = "ERROR: an earlier run\n" * 1000
|
||||
noise = "INFO noise line\n" * 8000 # ~128 KiB, no error line
|
||||
stdio_log.write_text(earlier + noise + "END-OF-RUN-MARKER\n")
|
||||
|
||||
excerpt = wrap_mod._proxy_startup_failure_excerpt(stdio_log, len(earlier))
|
||||
|
||||
assert "earlier run" not in excerpt
|
||||
assert "END-OF-RUN-MARKER" not in excerpt
|
||||
assert excerpt.endswith("INFO noise line")
|
||||
|
||||
def test_timeout_error_names_configured_timeout_and_env_var(self, monkeypatch, tmp_path):
|
||||
fake_proc = _FakeProxyProcess()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user