fix(logging): add payload-safe helpers for exceptions, URLs, ids and repeat warnings

Log records are kept and shared, so no level may carry request content, tool
output or credentials. headroom/log_safety.py describes an exception by its
types and code locations without the message (full traceback only with
HEADROOM_DEBUG_DUMP=full), redacts URL userinfo and query, bounds ids, and
gives a warn-once guard that stays bounded instead of resetting.

The request debug dump now uses the shared URL redactor and opt-in check.

Refs #4136

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-09 22:07:22 -07:00
co-authored by Claude Opus 5.5
parent 8c1beb20ae
commit 88f934b51b
3 changed files with 314 additions and 21 deletions
+155
View File
@@ -0,0 +1,155 @@
"""Payload-safe building blocks for log lines.
Log records are kept, rotated, attached to bug reports and shipped to log
collectors, so no level may carry request content, tool output or
credentials. Exception messages are the usual leak: a compressor, hook or
storage backend can quote the payload or a connection URL in its error. These
helpers describe a failure by what is safe to keep (exception types, code
locations, errno, ids, counts) and leave the message out.
Content is logged only when the operator opts in to it explicitly with
``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.
"""
from __future__ import annotations
import logging
import os
import threading
import traceback
from collections.abc import Hashable
from urllib.parse import urlsplit, urlunsplit
CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"
_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
_MAX_FRAMES = 3
_MAX_CHAIN = 4
_ID_MAX_CHARS = 80
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
def content_logging_enabled() -> bool:
"""True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""
return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES
def describe_exception(exc: BaseException) -> str:
"""Describe ``exc`` for a log line without its message.
Gives each exception in the cause chain as its type plus the innermost code
locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by
KeyError at ...``. An ``OSError`` keeps its errno and strerror, which never
carry payload. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full
traceback, messages included.
"""
if content_logging_enabled():
return "".join(traceback.format_exception(exc)).rstrip()
parts: list[str] = []
seen: set[int] = set()
current: BaseException | None = exc
while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:
seen.add(id(current))
parts.append(_describe_one(current))
current = current.__cause__ or (
None if current.__suppress_context__ else current.__context__
)
return "; caused by ".join(parts)
def _describe_one(exc: BaseException) -> str:
text = type(exc).__qualname__
if isinstance(exc, OSError) and exc.errno is not None:
# strerror is free text in some OSError subclasses (ssl.SSLError), so bound it.
text += f" [Errno {exc.errno}] {str(exc.strerror)[:_ID_MAX_CHARS]}"
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
if frames:
where = " <- ".join(
f"{_short_path(frame.filename)}:{frame.lineno} in {frame.name}"
for frame in reversed(frames)
)
text += f" at {where}"
return text
def _short_path(path: str) -> str:
"""``headroom/<module path>`` for Headroom's own files, the file name for anything else."""
normalized = path.replace("\\", "/")
if normalized.startswith(_PACKAGE_DIR + "/"):
return "headroom/" + normalized[len(_PACKAGE_DIR) + 1 :]
return normalized.rsplit("/", 1)[-1]
def redact_url(url: str) -> str:
"""Return ``url`` without userinfo, query or fragment, which can hold credentials.
The path is kept, so do not pass URLs that carry a secret in the path
(webhook URLs, ``/bot<token>/``). The host comes back lowercased.
"""
try:
parts = urlsplit(url)
host = parts.hostname or ""
port = parts.port
except ValueError:
return "<unparseable url>"
if ":" in host:
host = f"[{host}]"
netloc = f"{host}:{port}" if port is not None else host
redacted = urlunsplit((parts.scheme, netloc, parts.path, "", ""))
return f"{redacted}?<redacted>" if parts.query else redacted
def safe_id(value: object) -> str:
"""Render a client- or model-supplied id as one bounded log token.
Uses ``repr`` so a newline in the id cannot forge a second log line, and
cuts it to a fixed length so an oversized id cannot flood the log.
"""
text = repr(value)
if len(text) <= _ID_MAX_CHARS:
return text
return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"
class WarnOnce:
"""Warn once per key, for at most ``limit`` keys over the process lifetime.
Past the limit it logs one overflow WARNING and then stays quiet, so a
failure that hits many distinct keys (paths, rows, models) cannot flood the
log on every pass. A key is used up only when WARNING is enabled, so a
warning suppressed by the log level is still emitted later. ``forget``
re-arms a key once its failure has cleared.
"""
def __init__(self, limit: int, what: str) -> None:
if limit < 1:
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
self._limit = limit
self._what = what
self._keys: set[Hashable] = set()
self._overflowed = False
self._lock = threading.Lock()
def first(self, key: Hashable, log: logging.Logger) -> bool:
"""Return True if the caller should log this key's WARNING now."""
if not log.isEnabledFor(logging.WARNING):
return False
with self._lock:
if key in self._keys:
return False
if len(self._keys) < self._limit:
self._keys.add(key)
return True
report_overflow = not self._overflowed
self._overflowed = True
if report_overflow:
log.warning(
"More than %d distinct %s; further ones are logged at debug only",
self._limit,
self._what,
)
return False
def forget(self, key: Hashable) -> None:
"""Re-arm ``key`` so its next failure warns again."""
with self._lock:
self._keys.discard(key)
+5 -21
View File
@@ -14,7 +14,8 @@ import os
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
from headroom.log_safety import content_logging_enabled, redact_url
logger = logging.getLogger(__name__)
@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:
"""
if getattr(config, "stateless", False):
return "off"
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
if raw in ("full", "all", "content"):
if content_logging_enabled():
return "full"
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
if raw in ("1", "true", "yes", "on", "redacted"):
return "redacted"
return "off"
@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
return value
def _redact_url(url: str) -> str:
"""Drop the query string and fragment from ``url``.
Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
query is never needed to debug a rejected request, so it is elided in every
mode — ``full`` opts in to prompt content, not to credentials.
"""
try:
parts = urlsplit(url)
except ValueError:
return "<unparseable url>"
if not parts.query and not parts.fragment:
return url
redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
return f"{redacted}?<redacted>" if parts.query else redacted
def _decode_sent_body(body: Any) -> Any:
"""Decode the exact bytes sent upstream back to JSON for the dump."""
if not isinstance(body, (bytes, bytearray)):
@@ -119,7 +103,7 @@ def write_upstream_error_dump(
payload = json.dumps(
{
"request_id": request_id,
"url": _redact_url(url),
"url": redact_url(url),
"status": status,
"provider": provider,
"model": model,
+154
View File
@@ -0,0 +1,154 @@
"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""
from __future__ import annotations
import logging
import os
import pytest
from headroom import log_safety
from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
CANARY = "sk-live-CANARY tool output: patient record 42"
def _raise_chain() -> BaseException:
try:
try:
raise KeyError(CANARY)
except KeyError as inner:
raise ValueError(f"could not parse {CANARY}") from inner
except ValueError as exc:
return exc
def test_exception_description_names_types_and_places_but_not_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
text = describe_exception(_raise_chain())
assert "CANARY" not in text
assert text.startswith("ValueError at ")
assert "test_log_safety.py:" in text
assert "in _raise_chain" in text
assert "; caused by KeyError at " in text
def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))
assert text == "PermissionError [Errno 13] Permission denied"
def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
text = describe_exception(_raise_chain())
assert "Traceback" in text
assert CANARY in text
@pytest.mark.parametrize(
("url", "expected"),
[
(
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
"https://proxy.example:8443/v1?<redacted>",
),
("http://127.0.0.1:8787/stats", "http://127.0.0.1:8787/stats"),
("http://[::1]:8787/", "http://[::1]:8787/"),
("http://[::1", "<unparseable url>"),
],
)
def test_redact_url_drops_userinfo_and_query(url: str, expected: str) -> None:
assert redact_url(url) == expected
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
long = safe_id("x" * 10_000)
assert len(long) < 120
assert long.endswith("chars)")
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:
log = logging.getLogger("headroom.test_log_safety.bounded")
guard = WarnOnce(limit=64, what="unwritable ledgers")
with caplog.at_level(logging.INFO, logger=log.name):
for _ in range(3):
for path in range(65):
if guard.first(path, log):
log.warning("ledger %d failed", path)
messages = _warnings(caplog)
assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass
assert messages[-1] == (
"More than 64 distinct unwritable ledgers; further ones are logged at debug only"
)
def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:
log = logging.getLogger("headroom.test_log_safety.level")
guard = WarnOnce(limit=4, what="failures")
log.setLevel(logging.ERROR)
try:
assert guard.first("k", log) is False
log.setLevel(logging.INFO)
assert guard.first("k", log) is True
assert guard.first("k", log) is False
finally:
log.setLevel(logging.NOTSET)
def test_forget_rearms_a_key() -> None:
log = logging.getLogger("headroom.test_log_safety.forget")
guard = WarnOnce(limit=4, what="failures")
assert guard.first("path", log) is True
guard.forget("path")
assert guard.first("path", log) is True
def test_frame_paths_are_package_relative_for_headroom_and_bare_otherwise() -> None:
package_dir = os.path.dirname(log_safety.__file__)
assert log_safety._short_path(os.path.join(package_dir, "proxy", "server.py")) == (
"headroom/proxy/server.py"
)
assert log_safety._short_path("/home/runner/work/headroom/headroom/tests/test_x.py") == (
"test_x.py"
)
assert log_safety._short_path("/repo/headroom/.venv/lib/site-packages/httpx/_client.py") == (
"_client.py"
)
def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
try:
try:
raise KeyError(CANARY)
except KeyError:
raise ValueError("bad") from None
except ValueError as exc:
text = describe_exception(exc)
assert "KeyError" not in text
def test_a_cause_cycle_is_described_once() -> None:
first = ValueError(CANARY)
second = KeyError(CANARY)
first.__cause__ = second
second.__cause__ = first
assert describe_exception(first) == "ValueError; caused by KeyError"