fix(logging): drop URL paths from logs, never format a huge errno, reset WarnOnce hourly

A path segment can carry a credential (/bot<token>/, webhook URLs), so
redact_url keeps only scheme, host and port unless HEADROOM_DEBUG_DUMP=full;
userinfo and query are always dropped. An errno outside the C int range is
not formatted, since int-to-text can raise. WarnOnce now resets every window
for all keys, so a recurring failure warns again once an hour.

Refs #4136

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-10 09:10:34 -07:00
co-authored by Claude Opus 5.5
parent cd50ad35d8
commit b566a75851
3 changed files with 97 additions and 39 deletions
+42 -23
View File
@@ -26,6 +26,7 @@ _CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
_MAX_FRAMES = 3
_MAX_CHAIN = 4
_ID_MAX_CHARS = 80
_MAX_ERRNO = 2**31 - 1
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
@@ -62,7 +63,12 @@ def _describe_one(exc: BaseException) -> str:
if isinstance(exc, OSError) and isinstance(exc.errno, int):
# The system's own text for the errno; exc.strerror is free text in some
# subclasses (ssl.SSLError) and can quote a path or a peer's message.
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
# An out-of-range errno is not formatted at all: converting a huge int
# to text can itself raise, and a log helper must never raise.
if 0 <= exc.errno <= _MAX_ERRNO:
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
else:
text += " [Errno out of range]"
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
if frames:
where = " <- ".join(
@@ -89,10 +95,13 @@ def _short_path(path: str) -> str:
def redact_url(url: str) -> str:
"""Return ``url`` without userinfo, query or fragment, which can hold credentials.
"""Return ``url`` as scheme, host and port only, safe for any log line.
The path is kept, so do not pass URLs that carry a secret in the path
(webhook URLs, ``/bot<token>/``). The host comes back lowercased.
Userinfo, query and fragment are always dropped, and so is the path, since
a path segment can carry a credential (``/bot<token>/``, webhook URLs) and
no rule tells such a segment from a route name. The path is kept only with
the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes
back lowercased.
"""
try:
parts = urlsplit(url)
@@ -103,7 +112,11 @@ def redact_url(url: str) -> str:
if ":" in host:
host = f"[{host}]"
netloc = f"{host}:{port}" if port is not None else host
redacted = urlunsplit((parts.scheme, netloc, parts.path, "", ""))
if content_logging_enabled():
path = parts.path
else:
path = "/<path>" if parts.path not in ("", "/") else parts.path
redacted = urlunsplit((parts.scheme, netloc, path, "", ""))
return f"{redacted}?<redacted>" if parts.query else redacted
@@ -120,26 +133,30 @@ def safe_id(value: object) -> str:
class WarnOnce:
"""Warn once per key, for at most ``limit`` keys per ``window_seconds``.
"""Warn once per key per window, for at most ``limit`` keys per window.
Past the limit it logs one overflow WARNING and stays quiet for the rest of
the window, so a failure that hits many distinct keys (paths, rows, models)
cannot flood the log on every pass: at most ``limit + 1`` warnings per
window. After the window the guard starts over, so a newly broken key in a
long-lived process still warns. A key is used up only when WARNING is
enabled, so a warning suppressed by the log level is still emitted later.
``forget`` re-arms a key once its failure has cleared; it cannot reopen the
cap during an overflow window.
The first warning opens a window of ``window_seconds`` (an hour by
default). Inside it each key warns once; past ``limit`` keys it logs one
overflow WARNING and stays quiet, so a failure that hits many distinct keys
(paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings
per window. When the window ends the guard starts over, so a failure that
keeps recurring, or a newly broken key, warns again once an hour. A key is
used up only when WARNING is enabled, so a warning suppressed by the log
level is still emitted later. ``forget`` re-arms a key once its failure has
cleared; it cannot reopen the cap after the overflow notice.
"""
def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:
if limit < 1:
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
if window_seconds <= 0:
raise ValueError(f"WarnOnce window_seconds must be positive, got {window_seconds}")
self._limit = limit
self._what = what
self._window = window_seconds
self._keys: set[Hashable] = set()
self._overflowed_at: float | None = None
self._window_start: float | None = None
self._overflowed = False
self._lock = threading.Lock()
def first(self, key: Hashable, log: logging.Logger) -> bool:
@@ -147,18 +164,20 @@ class WarnOnce:
if not log.isEnabledFor(logging.WARNING):
return False
with self._lock:
now = time.monotonic()
if self._window_start is not None and now - self._window_start >= self._window:
self._keys.clear()
self._overflowed = False
self._window_start = None
if key in self._keys:
return False
now = time.monotonic()
if self._overflowed_at is not None and now - self._overflowed_at >= self._window:
self._keys.clear()
self._overflowed_at = None
if self._overflowed_at is None and len(self._keys) < self._limit:
if not self._overflowed and len(self._keys) < self._limit:
self._keys.add(key)
if self._window_start is None:
self._window_start = now
return True
report_overflow = self._overflowed_at is None
if report_overflow:
self._overflowed_at = now
report_overflow = not self._overflowed
self._overflowed = True
if report_overflow:
log.warning(
"More than %d distinct %s; further ones are logged at debug only "
+19 -4
View File
@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)
assert path is not None
text = path.read_text()
assert "AIzaSECRET" not in text
assert json.loads(text)["url"] == (
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
"?<redacted>"
)
if mode == "full": # the content opt-in keeps the route, still never the key
expected = (
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
)
else: # a path segment can carry a credential, so it is dropped without the opt-in
expected = "https://generativelanguage.googleapis.com/<path>"
assert json.loads(text)["url"] == expected + "?<redacted>"
@pytest.mark.parametrize(
"url",
["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],
)
def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")
path = _write(url=url)
assert path is not None
text = path.read_text()
assert "CANARY" not in text and "%43ANARY" not in text
def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
+36 -12
View File
@@ -64,17 +64,32 @@ def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch
[
(
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
"https://proxy.example:8443/v1?<redacted>",
"https://proxy.example:8443/<path>?<redacted>",
),
("http://127.0.0.1:8787/stats", "http://127.0.0.1:8787/stats"),
("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),
("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),
("http://127.0.0.1:8787", "http://127.0.0.1:8787"),
("http://[::1]:8787/", "http://[::1]:8787/"),
("http://[::1", "<unparseable url>"),
],
)
def test_redact_url_drops_userinfo_and_query(url: str, expected: str) -> None:
def test_redact_url_keeps_only_scheme_host_and_port(
url: str, expected: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
assert redact_url(url) == expected
def test_content_opt_in_keeps_the_path_but_never_credentials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
assert redact_url("https://u:pw@upstream.example/v1/messages?key=abc") == (
"https://upstream.example/v1/messages?<redacted>"
)
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
long = safe_id("x" * 10_000)
@@ -178,7 +193,7 @@ def test_forgetting_after_overflow_does_not_reopen_the_cap(
assert len(_warnings(caplog)) == 1
def test_warn_once_starts_over_after_the_overflow_window(
def test_warn_once_starts_over_after_the_window(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
log = logging.getLogger("headroom.test_log_safety.window")
@@ -188,18 +203,27 @@ def test_warn_once_starts_over_after_the_overflow_window(
results = []
with caplog.at_level(logging.INFO, logger=log.name):
for key, advance in (("a", 0), ("b", 0), ("c", 0), ("d", 599), ("new-hook", 1)):
steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))
for key, advance in steps:
clock[0] += advance
results.append(guard.first(key, log))
# a and b warn, c triggers the overflow notice, d is inside the window,
# and a newly broken key after the window warns again.
assert results == [True, True, False, False, True]
# a and b warn and c is the overflow notice; inside the window a stays quiet
# and d is suppressed; once the window ends, the still-recurring a and a
# newly broken key both warn again.
assert results == [True, True, False, False, False, True, True]
assert len(_warnings(caplog)) == 1
def test_huge_errno_does_not_break_the_description() -> None:
exc = OSError()
exc.errno = 2**63
def test_warn_once_needs_a_positive_window() -> None:
with pytest.raises(ValueError, match="window_seconds must be positive"):
WarnOnce(limit=1, what="failures", window_seconds=0)
assert describe_exception(exc) == f"OSError [Errno {2**63}] unknown error"
@pytest.mark.parametrize("digits", [19, 10_001])
def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:
"""An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""
exc = OSError()
exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it
assert describe_exception(exc) == "OSError [Errno out of range]"