mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(logging): drop URL paths from logs, never format a huge errno, reset WarnOnce hourly
A path segment can carry a credential (/bot<token>/, webhook URLs), so redact_url keeps only scheme, host and port unless HEADROOM_DEBUG_DUMP=full; userinfo and query are always dropped. An errno outside the C int range is not formatted, since int-to-text can raise. WarnOnce now resets every window for all keys, so a recurring failure warns again once an hour. Refs #4136 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
cd50ad35d8
commit
b566a75851
+42
-23
@@ -26,6 +26,7 @@ _CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
|
||||
_MAX_FRAMES = 3
|
||||
_MAX_CHAIN = 4
|
||||
_ID_MAX_CHARS = 80
|
||||
_MAX_ERRNO = 2**31 - 1
|
||||
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
|
||||
|
||||
|
||||
@@ -62,7 +63,12 @@ def _describe_one(exc: BaseException) -> str:
|
||||
if isinstance(exc, OSError) and isinstance(exc.errno, int):
|
||||
# The system's own text for the errno; exc.strerror is free text in some
|
||||
# subclasses (ssl.SSLError) and can quote a path or a peer's message.
|
||||
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
|
||||
# An out-of-range errno is not formatted at all: converting a huge int
|
||||
# to text can itself raise, and a log helper must never raise.
|
||||
if 0 <= exc.errno <= _MAX_ERRNO:
|
||||
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
|
||||
else:
|
||||
text += " [Errno out of range]"
|
||||
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
|
||||
if frames:
|
||||
where = " <- ".join(
|
||||
@@ -89,10 +95,13 @@ def _short_path(path: str) -> str:
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Return ``url`` without userinfo, query or fragment, which can hold credentials.
|
||||
"""Return ``url`` as scheme, host and port only, safe for any log line.
|
||||
|
||||
The path is kept, so do not pass URLs that carry a secret in the path
|
||||
(webhook URLs, ``/bot<token>/``). The host comes back lowercased.
|
||||
Userinfo, query and fragment are always dropped, and so is the path, since
|
||||
a path segment can carry a credential (``/bot<token>/``, webhook URLs) and
|
||||
no rule tells such a segment from a route name. The path is kept only with
|
||||
the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes
|
||||
back lowercased.
|
||||
"""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
@@ -103,7 +112,11 @@ def redact_url(url: str) -> str:
|
||||
if ":" in host:
|
||||
host = f"[{host}]"
|
||||
netloc = f"{host}:{port}" if port is not None else host
|
||||
redacted = urlunsplit((parts.scheme, netloc, parts.path, "", ""))
|
||||
if content_logging_enabled():
|
||||
path = parts.path
|
||||
else:
|
||||
path = "/<path>" if parts.path not in ("", "/") else parts.path
|
||||
redacted = urlunsplit((parts.scheme, netloc, path, "", ""))
|
||||
return f"{redacted}?<redacted>" if parts.query else redacted
|
||||
|
||||
|
||||
@@ -120,26 +133,30 @@ def safe_id(value: object) -> str:
|
||||
|
||||
|
||||
class WarnOnce:
|
||||
"""Warn once per key, for at most ``limit`` keys per ``window_seconds``.
|
||||
"""Warn once per key per window, for at most ``limit`` keys per window.
|
||||
|
||||
Past the limit it logs one overflow WARNING and stays quiet for the rest of
|
||||
the window, so a failure that hits many distinct keys (paths, rows, models)
|
||||
cannot flood the log on every pass: at most ``limit + 1`` warnings per
|
||||
window. After the window the guard starts over, so a newly broken key in a
|
||||
long-lived process still warns. A key is used up only when WARNING is
|
||||
enabled, so a warning suppressed by the log level is still emitted later.
|
||||
``forget`` re-arms a key once its failure has cleared; it cannot reopen the
|
||||
cap during an overflow window.
|
||||
The first warning opens a window of ``window_seconds`` (an hour by
|
||||
default). Inside it each key warns once; past ``limit`` keys it logs one
|
||||
overflow WARNING and stays quiet, so a failure that hits many distinct keys
|
||||
(paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings
|
||||
per window. When the window ends the guard starts over, so a failure that
|
||||
keeps recurring, or a newly broken key, warns again once an hour. A key is
|
||||
used up only when WARNING is enabled, so a warning suppressed by the log
|
||||
level is still emitted later. ``forget`` re-arms a key once its failure has
|
||||
cleared; it cannot reopen the cap after the overflow notice.
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:
|
||||
if limit < 1:
|
||||
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
|
||||
if window_seconds <= 0:
|
||||
raise ValueError(f"WarnOnce window_seconds must be positive, got {window_seconds}")
|
||||
self._limit = limit
|
||||
self._what = what
|
||||
self._window = window_seconds
|
||||
self._keys: set[Hashable] = set()
|
||||
self._overflowed_at: float | None = None
|
||||
self._window_start: float | None = None
|
||||
self._overflowed = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def first(self, key: Hashable, log: logging.Logger) -> bool:
|
||||
@@ -147,18 +164,20 @@ class WarnOnce:
|
||||
if not log.isEnabledFor(logging.WARNING):
|
||||
return False
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
if self._window_start is not None and now - self._window_start >= self._window:
|
||||
self._keys.clear()
|
||||
self._overflowed = False
|
||||
self._window_start = None
|
||||
if key in self._keys:
|
||||
return False
|
||||
now = time.monotonic()
|
||||
if self._overflowed_at is not None and now - self._overflowed_at >= self._window:
|
||||
self._keys.clear()
|
||||
self._overflowed_at = None
|
||||
if self._overflowed_at is None and len(self._keys) < self._limit:
|
||||
if not self._overflowed and len(self._keys) < self._limit:
|
||||
self._keys.add(key)
|
||||
if self._window_start is None:
|
||||
self._window_start = now
|
||||
return True
|
||||
report_overflow = self._overflowed_at is None
|
||||
if report_overflow:
|
||||
self._overflowed_at = now
|
||||
report_overflow = not self._overflowed
|
||||
self._overflowed = True
|
||||
if report_overflow:
|
||||
log.warning(
|
||||
"More than %d distinct %s; further ones are logged at debug only "
|
||||
|
||||
@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)
|
||||
assert path is not None
|
||||
text = path.read_text()
|
||||
assert "AIzaSECRET" not in text
|
||||
assert json.loads(text)["url"] == (
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
|
||||
"?<redacted>"
|
||||
)
|
||||
if mode == "full": # the content opt-in keeps the route, still never the key
|
||||
expected = (
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
|
||||
)
|
||||
else: # a path segment can carry a credential, so it is dropped without the opt-in
|
||||
expected = "https://generativelanguage.googleapis.com/<path>"
|
||||
assert json.loads(text)["url"] == expected + "?<redacted>"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],
|
||||
)
|
||||
def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")
|
||||
path = _write(url=url)
|
||||
assert path is not None
|
||||
text = path.read_text()
|
||||
assert "CANARY" not in text and "%43ANARY" not in text
|
||||
|
||||
|
||||
def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
|
||||
|
||||
+36
-12
@@ -64,17 +64,32 @@ def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch
|
||||
[
|
||||
(
|
||||
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
|
||||
"https://proxy.example:8443/v1?<redacted>",
|
||||
"https://proxy.example:8443/<path>?<redacted>",
|
||||
),
|
||||
("http://127.0.0.1:8787/stats", "http://127.0.0.1:8787/stats"),
|
||||
("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),
|
||||
("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),
|
||||
("http://127.0.0.1:8787", "http://127.0.0.1:8787"),
|
||||
("http://[::1]:8787/", "http://[::1]:8787/"),
|
||||
("http://[::1", "<unparseable url>"),
|
||||
],
|
||||
)
|
||||
def test_redact_url_drops_userinfo_and_query(url: str, expected: str) -> None:
|
||||
def test_redact_url_keeps_only_scheme_host_and_port(
|
||||
url: str, expected: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
assert redact_url(url) == expected
|
||||
|
||||
|
||||
def test_content_opt_in_keeps_the_path_but_never_credentials(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
|
||||
|
||||
assert redact_url("https://u:pw@upstream.example/v1/messages?key=abc") == (
|
||||
"https://upstream.example/v1/messages?<redacted>"
|
||||
)
|
||||
|
||||
|
||||
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
|
||||
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
|
||||
long = safe_id("x" * 10_000)
|
||||
@@ -178,7 +193,7 @@ def test_forgetting_after_overflow_does_not_reopen_the_cap(
|
||||
assert len(_warnings(caplog)) == 1
|
||||
|
||||
|
||||
def test_warn_once_starts_over_after_the_overflow_window(
|
||||
def test_warn_once_starts_over_after_the_window(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.window")
|
||||
@@ -188,18 +203,27 @@ def test_warn_once_starts_over_after_the_overflow_window(
|
||||
|
||||
results = []
|
||||
with caplog.at_level(logging.INFO, logger=log.name):
|
||||
for key, advance in (("a", 0), ("b", 0), ("c", 0), ("d", 599), ("new-hook", 1)):
|
||||
steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))
|
||||
for key, advance in steps:
|
||||
clock[0] += advance
|
||||
results.append(guard.first(key, log))
|
||||
|
||||
# a and b warn, c triggers the overflow notice, d is inside the window,
|
||||
# and a newly broken key after the window warns again.
|
||||
assert results == [True, True, False, False, True]
|
||||
# a and b warn and c is the overflow notice; inside the window a stays quiet
|
||||
# and d is suppressed; once the window ends, the still-recurring a and a
|
||||
# newly broken key both warn again.
|
||||
assert results == [True, True, False, False, False, True, True]
|
||||
assert len(_warnings(caplog)) == 1
|
||||
|
||||
|
||||
def test_huge_errno_does_not_break_the_description() -> None:
|
||||
exc = OSError()
|
||||
exc.errno = 2**63
|
||||
def test_warn_once_needs_a_positive_window() -> None:
|
||||
with pytest.raises(ValueError, match="window_seconds must be positive"):
|
||||
WarnOnce(limit=1, what="failures", window_seconds=0)
|
||||
|
||||
assert describe_exception(exc) == f"OSError [Errno {2**63}] unknown error"
|
||||
|
||||
@pytest.mark.parametrize("digits", [19, 10_001])
|
||||
def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:
|
||||
"""An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""
|
||||
exc = OSError()
|
||||
exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it
|
||||
|
||||
assert describe_exception(exc) == "OSError [Errno out of range]"
|
||||
|
||||
Reference in New Issue
Block a user