mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cli): also scan the tail of a long startup log for the error
The bounded read looked only at the first 64 KiB this run wrote, so an error after a long burst of startup output was never quoted. When the head has no error line and the run wrote more, the last 64 KiB is read too (128 KiB at most). An early error still wins. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c9868420b8
commit
062cf01110
+21
-10
@@ -714,8 +714,9 @@ def _get_proxy_stdio_log_path(port: int | None = None) -> Path:
|
||||
# error-looking line instead.
|
||||
_PROXY_STARTUP_ERROR_RE = re.compile(r"\b(?:ERROR|CRITICAL)\b|Traceback \(most recent call last\)")
|
||||
_PROXY_STARTUP_EXCERPT_CHARS = 500
|
||||
# The startup error comes before the shutdown summary, so the first 64 KiB of
|
||||
# this run's output is enough; never read a runaway log whole.
|
||||
# Never read a runaway log whole: look at most at the first and the last 64 KiB
|
||||
# of this run's output. The error usually comes early, before the shutdown
|
||||
# summary; the tail catches one that follows a long burst of startup output.
|
||||
_PROXY_STARTUP_READ_BYTES = 64 * 1024
|
||||
|
||||
|
||||
@@ -723,24 +724,34 @@ def _proxy_startup_failure_excerpt(stdio_log_path: Path, start: int) -> str:
|
||||
"""Return the part of this run's stdio log that best explains a startup exit.
|
||||
|
||||
Reads only what was written after byte offset ``start`` (the log is shared
|
||||
across runs), and at most the first ``_PROXY_STARTUP_READ_BYTES`` of it.
|
||||
across runs): the first ``_PROXY_STARTUP_READ_BYTES`` of it and, when the run
|
||||
wrote more and the head has no error, the last ``_PROXY_STARTUP_READ_BYTES``.
|
||||
Prefers the first ERROR/CRITICAL/Traceback line onwards, and falls back to
|
||||
the last few hundred characters read.
|
||||
"""
|
||||
try:
|
||||
with open(stdio_log_path, "rb") as fh:
|
||||
fh.seek(start)
|
||||
raw = fh.read(_PROXY_STARTUP_READ_BYTES)
|
||||
output = raw.decode("utf-8", errors="replace").strip()
|
||||
windows = [fh.read(_PROXY_STARTUP_READ_BYTES)]
|
||||
end = fh.seek(0, os.SEEK_END)
|
||||
tail_start = max(start + _PROXY_STARTUP_READ_BYTES, end - _PROXY_STARTUP_READ_BYTES)
|
||||
if end > tail_start and not _PROXY_STARTUP_ERROR_RE.search(
|
||||
windows[0].decode("utf-8", errors="replace")
|
||||
):
|
||||
fh.seek(tail_start)
|
||||
windows.append(fh.read(_PROXY_STARTUP_READ_BYTES))
|
||||
except OSError:
|
||||
return "(no log output)"
|
||||
output = ""
|
||||
for raw in windows:
|
||||
output = raw.decode("utf-8", errors="replace").strip()
|
||||
match = _PROXY_STARTUP_ERROR_RE.search(output)
|
||||
if match is not None:
|
||||
line_start = output.rfind("\n", 0, match.start()) + 1
|
||||
return output[line_start : line_start + _PROXY_STARTUP_EXCERPT_CHARS]
|
||||
if not output:
|
||||
return "(no log output)"
|
||||
match = _PROXY_STARTUP_ERROR_RE.search(output)
|
||||
if match is None:
|
||||
return output[-_PROXY_STARTUP_EXCERPT_CHARS:]
|
||||
line_start = output.rfind("\n", 0, match.start()) + 1
|
||||
return output[line_start : line_start + _PROXY_STARTUP_EXCERPT_CHARS]
|
||||
return output[-_PROXY_STARTUP_EXCERPT_CHARS:]
|
||||
|
||||
|
||||
def _start_proxy(
|
||||
|
||||
@@ -267,18 +267,45 @@ class TestCLIWrapProxyTimeout:
|
||||
assert "Local telemetry" not in message
|
||||
assert f"Full log: {stdio_log}" in message
|
||||
|
||||
def test_startup_excerpt_reads_at_most_64_kib_of_this_run(self, tmp_path):
|
||||
"""A runaway stdio log is never read whole: only this run's first 64 KiB."""
|
||||
def test_startup_excerpt_reads_at_most_64_kib_of_this_run(self, monkeypatch, tmp_path):
|
||||
"""A runaway stdio log is never read whole: this run's first and last 64 KiB
|
||||
at most, so an error after a long burst of output is still quoted."""
|
||||
stdio_log = tmp_path / "proxy-stdio-8787.log"
|
||||
earlier = "ERROR: an earlier run\n" * 1000
|
||||
noise = "INFO noise line\n" * 8000 # ~128 KiB, no error line
|
||||
stdio_log.write_text(earlier + noise + "END-OF-RUN-MARKER\n")
|
||||
noise = "INFO noise line\n" * 16000 # ~256 KiB, no error line
|
||||
late = "ERROR: late startup failure\n"
|
||||
stdio_log.write_text(earlier + noise + late + "INFO shutdown summary\n")
|
||||
read_sizes: list[int] = []
|
||||
real_open = open
|
||||
|
||||
def tracking_open(*args, **kwargs):
|
||||
handle = real_open(*args, **kwargs)
|
||||
real_read = handle.read
|
||||
|
||||
def read(size=-1):
|
||||
data = real_read(size)
|
||||
read_sizes.append(len(data))
|
||||
return data
|
||||
|
||||
handle.read = read
|
||||
return handle
|
||||
|
||||
monkeypatch.setattr(wrap_mod, "open", tracking_open, raising=False)
|
||||
|
||||
excerpt = wrap_mod._proxy_startup_failure_excerpt(stdio_log, len(earlier))
|
||||
|
||||
assert "earlier run" not in excerpt
|
||||
assert "END-OF-RUN-MARKER" not in excerpt
|
||||
assert excerpt.endswith("INFO noise line")
|
||||
assert excerpt.startswith("ERROR: late startup failure")
|
||||
assert sum(read_sizes) <= 2 * wrap_mod._PROXY_STARTUP_READ_BYTES
|
||||
|
||||
def test_startup_excerpt_prefers_an_error_in_the_first_64_kib(self, tmp_path):
|
||||
stdio_log = tmp_path / "proxy-stdio-8787.log"
|
||||
noise = "INFO noise line\n" * 16000
|
||||
stdio_log.write_text("ERROR: early failure\n" + noise + "ERROR: later echo\n")
|
||||
|
||||
excerpt = wrap_mod._proxy_startup_failure_excerpt(stdio_log, 0)
|
||||
|
||||
assert excerpt.startswith("ERROR: early failure")
|
||||
|
||||
def test_timeout_error_names_configured_timeout_and_env_var(self, monkeypatch, tmp_path):
|
||||
fake_proc = _FakeProxyProcess()
|
||||
|
||||
Reference in New Issue
Block a user