mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
Merge remote-tracking branch 'origin/cleanup/log-safety-helpers' into cleanup/proxy-core-a-surface-ignored-config
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
"""Payload-safe building blocks for log lines.
|
||||
|
||||
Log records are kept, rotated, attached to bug reports and shipped to log
|
||||
collectors, so no level may carry request content, tool output or
|
||||
credentials. Exception messages are the usual leak: a compressor, hook or
|
||||
storage backend can quote the payload or a connection URL in its error. These
|
||||
helpers describe a failure by what is safe to keep (exception types, code
|
||||
locations, errno, ids, counts) and leave the message out.
|
||||
|
||||
Content is logged only when the operator opts in to it explicitly with
|
||||
``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Hashable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"
|
||||
_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
|
||||
_MAX_FRAMES = 3
|
||||
_MAX_CHAIN = 4
|
||||
_ID_MAX_CHARS = 80
|
||||
_MAX_ERRNO = 2**31 - 1
|
||||
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
|
||||
|
||||
|
||||
def content_logging_enabled() -> bool:
|
||||
"""True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""
|
||||
return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES
|
||||
|
||||
|
||||
def describe_exception(exc: BaseException) -> str:
|
||||
"""Describe ``exc`` for a log line without its message.
|
||||
|
||||
Gives each exception in the cause chain as its type plus the innermost code
|
||||
locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by
|
||||
KeyError at ...``. An ``OSError`` keeps its errno and the system's text for
|
||||
it, never the exception's own strerror. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full
|
||||
traceback, messages included.
|
||||
"""
|
||||
if content_logging_enabled():
|
||||
return "".join(traceback.format_exception(exc)).rstrip()
|
||||
parts: list[str] = []
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = exc
|
||||
while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:
|
||||
seen.add(id(current))
|
||||
parts.append(_describe_one(current))
|
||||
current = current.__cause__ or (
|
||||
None if current.__suppress_context__ else current.__context__
|
||||
)
|
||||
return "; caused by ".join(parts)
|
||||
|
||||
|
||||
def _describe_one(exc: BaseException) -> str:
|
||||
text = type(exc).__qualname__
|
||||
if isinstance(exc, OSError) and isinstance(exc.errno, int):
|
||||
# The system's own text for the errno; exc.strerror is free text in some
|
||||
# subclasses (ssl.SSLError) and can quote a path or a peer's message.
|
||||
# An out-of-range errno is not formatted at all: converting a huge int
|
||||
# to text can itself raise, and a log helper must never raise.
|
||||
if 0 <= exc.errno <= _MAX_ERRNO:
|
||||
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
|
||||
else:
|
||||
text += " [Errno out of range]"
|
||||
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
|
||||
if frames:
|
||||
where = " <- ".join(
|
||||
f"{_short_path(frame.filename)}:{frame.lineno} in {frame.name}"
|
||||
for frame in reversed(frames)
|
||||
)
|
||||
text += f" at {where}"
|
||||
return text
|
||||
|
||||
|
||||
def _errno_text(errno: int) -> str:
|
||||
try:
|
||||
return os.strerror(errno)
|
||||
except (ValueError, OverflowError):
|
||||
return "unknown error"
|
||||
|
||||
|
||||
def _short_path(path: str) -> str:
|
||||
"""``headroom/<module path>`` for Headroom's own files, the file name for anything else."""
|
||||
normalized = path.replace("\\", "/")
|
||||
if normalized.startswith(_PACKAGE_DIR + "/"):
|
||||
return "headroom/" + normalized[len(_PACKAGE_DIR) + 1 :]
|
||||
return normalized.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Return ``url`` as scheme, host and port only, safe for any log line.
|
||||
|
||||
Userinfo, query and fragment are always dropped, and so is the path, since
|
||||
a path segment can carry a credential (``/bot<token>/``, webhook URLs) and
|
||||
no rule tells such a segment from a route name. The path is kept only with
|
||||
the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes
|
||||
back lowercased.
|
||||
"""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
host = parts.hostname or ""
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
return "<unparseable url>"
|
||||
if ":" in host:
|
||||
host = f"[{host}]"
|
||||
netloc = f"{host}:{port}" if port is not None else host
|
||||
if content_logging_enabled():
|
||||
path = parts.path
|
||||
else:
|
||||
path = "/<path>" if parts.path not in ("", "/") else parts.path
|
||||
redacted = urlunsplit((parts.scheme, netloc, path, "", ""))
|
||||
return f"{redacted}?<redacted>" if parts.query else redacted
|
||||
|
||||
|
||||
def safe_id(value: object) -> str:
|
||||
"""Render a client- or model-supplied id as one bounded log token.
|
||||
|
||||
Uses ``repr`` so a newline in the id cannot forge a second log line, and
|
||||
cuts it to a fixed length so an oversized id cannot flood the log.
|
||||
"""
|
||||
text = repr(value)
|
||||
if len(text) <= _ID_MAX_CHARS:
|
||||
return text
|
||||
return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"
|
||||
|
||||
|
||||
class WarnOnce:
|
||||
"""Warn once per key per window, for at most ``limit`` keys per window.
|
||||
|
||||
The first warning opens a window of ``window_seconds`` (an hour by
|
||||
default). Inside it each key warns once; past ``limit`` keys it logs one
|
||||
overflow WARNING and stays quiet, so a failure that hits many distinct keys
|
||||
(paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings
|
||||
per window. When the window ends the guard starts over, so a failure that
|
||||
keeps recurring, or a newly broken key, warns again once an hour. A key is
|
||||
used up only when WARNING is enabled, so a warning suppressed by the log
|
||||
level is still emitted later. ``forget`` re-arms a key once its failure has
|
||||
cleared; it cannot reopen the cap after the overflow notice.
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:
|
||||
if limit < 1:
|
||||
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
|
||||
if window_seconds <= 0:
|
||||
raise ValueError(f"WarnOnce window_seconds must be positive, got {window_seconds}")
|
||||
self._limit = limit
|
||||
self._what = what
|
||||
self._window = window_seconds
|
||||
self._keys: set[Hashable] = set()
|
||||
self._window_start: float | None = None
|
||||
self._overflowed = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def first(self, key: Hashable, log: logging.Logger) -> bool:
|
||||
"""Return True if the caller should log this key's WARNING now."""
|
||||
if not log.isEnabledFor(logging.WARNING):
|
||||
return False
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
if self._window_start is not None and now - self._window_start >= self._window:
|
||||
self._keys.clear()
|
||||
self._overflowed = False
|
||||
self._window_start = None
|
||||
if key in self._keys:
|
||||
return False
|
||||
if not self._overflowed and len(self._keys) < self._limit:
|
||||
self._keys.add(key)
|
||||
if self._window_start is None:
|
||||
self._window_start = now
|
||||
return True
|
||||
report_overflow = not self._overflowed
|
||||
self._overflowed = True
|
||||
if report_overflow:
|
||||
log.warning(
|
||||
"More than %d distinct %s; further ones are logged at debug only "
|
||||
"for the next %d minutes",
|
||||
self._limit,
|
||||
self._what,
|
||||
max(1, round(self._window / 60)),
|
||||
)
|
||||
return False
|
||||
|
||||
def forget(self, key: Hashable) -> None:
|
||||
"""Re-arm ``key`` so its next failure warns again."""
|
||||
with self._lock:
|
||||
self._keys.discard(key)
|
||||
@@ -14,7 +14,8 @@ import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from headroom.log_safety import content_logging_enabled, redact_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:
|
||||
"""
|
||||
if getattr(config, "stateless", False):
|
||||
return "off"
|
||||
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
|
||||
if raw in ("full", "all", "content"):
|
||||
if content_logging_enabled():
|
||||
return "full"
|
||||
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
|
||||
if raw in ("1", "true", "yes", "on", "redacted"):
|
||||
return "redacted"
|
||||
return "off"
|
||||
@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _redact_url(url: str) -> str:
|
||||
"""Drop the query string and fragment from ``url``.
|
||||
|
||||
Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
|
||||
query is never needed to debug a rejected request, so it is elided in every
|
||||
mode — ``full`` opts in to prompt content, not to credentials.
|
||||
"""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
except ValueError:
|
||||
return "<unparseable url>"
|
||||
if not parts.query and not parts.fragment:
|
||||
return url
|
||||
redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
return f"{redacted}?<redacted>" if parts.query else redacted
|
||||
|
||||
|
||||
def _decode_sent_body(body: Any) -> Any:
|
||||
"""Decode the exact bytes sent upstream back to JSON for the dump."""
|
||||
if not isinstance(body, (bytes, bytearray)):
|
||||
@@ -119,7 +103,7 @@ def write_upstream_error_dump(
|
||||
payload = json.dumps(
|
||||
{
|
||||
"request_id": request_id,
|
||||
"url": _redact_url(url),
|
||||
"url": redact_url(url),
|
||||
"status": status,
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
|
||||
@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)
|
||||
assert path is not None
|
||||
text = path.read_text()
|
||||
assert "AIzaSECRET" not in text
|
||||
assert json.loads(text)["url"] == (
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
|
||||
"?<redacted>"
|
||||
)
|
||||
if mode == "full": # the content opt-in keeps the route, still never the key
|
||||
expected = (
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
|
||||
)
|
||||
else: # a path segment can carry a credential, so it is dropped without the opt-in
|
||||
expected = "https://generativelanguage.googleapis.com/<path>"
|
||||
assert json.loads(text)["url"] == expected + "?<redacted>"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],
|
||||
)
|
||||
def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")
|
||||
path = _write(url=url)
|
||||
assert path is not None
|
||||
text = path.read_text()
|
||||
assert "CANARY" not in text and "%43ANARY" not in text
|
||||
|
||||
|
||||
def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom import log_safety
|
||||
from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
|
||||
|
||||
CANARY = "sk-live-CANARY tool output: patient record 42"
|
||||
|
||||
|
||||
def _raise_chain() -> BaseException:
|
||||
try:
|
||||
try:
|
||||
raise KeyError(CANARY)
|
||||
except KeyError as inner:
|
||||
raise ValueError(f"could not parse {CANARY}") from inner
|
||||
except ValueError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def test_exception_description_names_types_and_places_but_not_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
text = describe_exception(_raise_chain())
|
||||
|
||||
assert "CANARY" not in text
|
||||
assert text.startswith("ValueError at ")
|
||||
assert "test_log_safety.py:" in text
|
||||
assert "in _raise_chain" in text
|
||||
assert "; caused by KeyError at " in text
|
||||
|
||||
|
||||
def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))
|
||||
|
||||
assert text == "PermissionError [Errno 13] Permission denied"
|
||||
|
||||
|
||||
def test_oserror_free_text_strerror_is_replaced_by_the_system_text() -> None:
|
||||
text = describe_exception(OSError(2, f"token={CANARY}"))
|
||||
|
||||
assert "CANARY" not in text
|
||||
assert text == f"FileNotFoundError [Errno 2] {os.strerror(2)}"
|
||||
|
||||
|
||||
def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
|
||||
|
||||
text = describe_exception(_raise_chain())
|
||||
|
||||
assert "Traceback" in text
|
||||
assert CANARY in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
|
||||
"https://proxy.example:8443/<path>?<redacted>",
|
||||
),
|
||||
("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),
|
||||
("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),
|
||||
("http://127.0.0.1:8787", "http://127.0.0.1:8787"),
|
||||
("http://[::1]:8787/", "http://[::1]:8787/"),
|
||||
("http://[::1", "<unparseable url>"),
|
||||
],
|
||||
)
|
||||
def test_redact_url_keeps_only_scheme_host_and_port(
|
||||
url: str, expected: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
assert redact_url(url) == expected
|
||||
|
||||
|
||||
def test_content_opt_in_keeps_the_path_but_never_credentials(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
|
||||
|
||||
assert redact_url("https://u:pw@upstream.example/v1/messages?key=abc") == (
|
||||
"https://upstream.example/v1/messages?<redacted>"
|
||||
)
|
||||
|
||||
|
||||
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
|
||||
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
|
||||
long = safe_id("x" * 10_000)
|
||||
assert len(long) < 120
|
||||
assert long.endswith("chars)")
|
||||
|
||||
|
||||
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
||||
|
||||
|
||||
def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.bounded")
|
||||
guard = WarnOnce(limit=64, what="unwritable ledgers")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger=log.name):
|
||||
for _ in range(3):
|
||||
for path in range(65):
|
||||
if guard.first(path, log):
|
||||
log.warning("ledger %d failed", path)
|
||||
|
||||
messages = _warnings(caplog)
|
||||
assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass
|
||||
assert messages[-1] == (
|
||||
"More than 64 distinct unwritable ledgers; further ones are logged at debug only "
|
||||
"for the next 60 minutes"
|
||||
)
|
||||
|
||||
|
||||
def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.level")
|
||||
guard = WarnOnce(limit=4, what="failures")
|
||||
log.setLevel(logging.ERROR)
|
||||
try:
|
||||
assert guard.first("k", log) is False
|
||||
log.setLevel(logging.INFO)
|
||||
assert guard.first("k", log) is True
|
||||
assert guard.first("k", log) is False
|
||||
finally:
|
||||
log.setLevel(logging.NOTSET)
|
||||
|
||||
|
||||
def test_forget_rearms_a_key() -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.forget")
|
||||
guard = WarnOnce(limit=4, what="failures")
|
||||
|
||||
assert guard.first("path", log) is True
|
||||
guard.forget("path")
|
||||
assert guard.first("path", log) is True
|
||||
|
||||
|
||||
def test_frame_paths_are_package_relative_for_headroom_and_bare_otherwise() -> None:
|
||||
package_dir = os.path.dirname(log_safety.__file__)
|
||||
|
||||
assert log_safety._short_path(os.path.join(package_dir, "proxy", "server.py")) == (
|
||||
"headroom/proxy/server.py"
|
||||
)
|
||||
assert log_safety._short_path("/home/runner/work/headroom/headroom/tests/test_x.py") == (
|
||||
"test_x.py"
|
||||
)
|
||||
assert log_safety._short_path("/repo/headroom/.venv/lib/site-packages/httpx/_client.py") == (
|
||||
"_client.py"
|
||||
)
|
||||
|
||||
|
||||
def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
try:
|
||||
try:
|
||||
raise KeyError(CANARY)
|
||||
except KeyError:
|
||||
raise ValueError("bad") from None
|
||||
except ValueError as exc:
|
||||
text = describe_exception(exc)
|
||||
|
||||
assert "KeyError" not in text
|
||||
|
||||
|
||||
def test_a_cause_cycle_is_described_once() -> None:
|
||||
first = ValueError(CANARY)
|
||||
second = KeyError(CANARY)
|
||||
first.__cause__ = second
|
||||
second.__cause__ = first
|
||||
|
||||
assert describe_exception(first) == "ValueError; caused by KeyError"
|
||||
|
||||
|
||||
def test_forgetting_after_overflow_does_not_reopen_the_cap(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.reopen")
|
||||
guard = WarnOnce(limit=2, what="failures")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger=log.name):
|
||||
assert guard.first("a", log) and guard.first("b", log)
|
||||
assert guard.first("c", log) is False # overflow notice
|
||||
for cycle in range(5):
|
||||
guard.forget("a")
|
||||
assert guard.first(f"new-{cycle}", log) is False
|
||||
|
||||
assert len(_warnings(caplog)) == 1
|
||||
|
||||
|
||||
def test_warn_once_starts_over_after_the_window(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.window")
|
||||
guard = WarnOnce(limit=2, what="hook failures", window_seconds=600)
|
||||
clock = [1000.0]
|
||||
monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])
|
||||
|
||||
results = []
|
||||
with caplog.at_level(logging.INFO, logger=log.name):
|
||||
steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))
|
||||
for key, advance in steps:
|
||||
clock[0] += advance
|
||||
results.append(guard.first(key, log))
|
||||
|
||||
# a and b warn and c is the overflow notice; inside the window a stays quiet
|
||||
# and d is suppressed; once the window ends, the still-recurring a and a
|
||||
# newly broken key both warn again.
|
||||
assert results == [True, True, False, False, False, True, True]
|
||||
assert len(_warnings(caplog)) == 1
|
||||
|
||||
|
||||
def test_warn_once_needs_a_positive_window() -> None:
|
||||
with pytest.raises(ValueError, match="window_seconds must be positive"):
|
||||
WarnOnce(limit=1, what="failures", window_seconds=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("digits", [19, 10_001])
|
||||
def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:
|
||||
"""An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""
|
||||
exc = OSError()
|
||||
exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it
|
||||
|
||||
assert describe_exception(exc) == "OSError [Errno out of range]"
|
||||
Reference in New Issue
Block a user