mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(logging): add payload-safe helpers for exceptions, URLs, ids and repeat warnings
Log records are kept and shared, so no level may carry request content, tool output or credentials. headroom/log_safety.py describes an exception by its types and code locations without the message (full traceback only with HEADROOM_DEBUG_DUMP=full), redacts URL userinfo and query, bounds ids, and gives a warn-once guard that stays bounded instead of resetting. The request debug dump now uses the shared URL redactor and opt-in check. Refs #4136 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8c1beb20ae
commit
88f934b51b
@@ -0,0 +1,155 @@
|
||||
"""Payload-safe building blocks for log lines.
|
||||
|
||||
Log records are kept, rotated, attached to bug reports and shipped to log
|
||||
collectors, so no level may carry request content, tool output or
|
||||
credentials. Exception messages are the usual leak: a compressor, hook or
|
||||
storage backend can quote the payload or a connection URL in its error. These
|
||||
helpers describe a failure by what is safe to keep (exception types, code
|
||||
locations, errno, ids, counts) and leave the message out.
|
||||
|
||||
Content is logged only when the operator opts in to it explicitly with
|
||||
``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import traceback
|
||||
from collections.abc import Hashable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"
|
||||
_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
|
||||
_MAX_FRAMES = 3
|
||||
_MAX_CHAIN = 4
|
||||
_ID_MAX_CHARS = 80
|
||||
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
|
||||
|
||||
|
||||
def content_logging_enabled() -> bool:
|
||||
"""True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""
|
||||
return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES
|
||||
|
||||
|
||||
def describe_exception(exc: BaseException) -> str:
|
||||
"""Describe ``exc`` for a log line without its message.
|
||||
|
||||
Gives each exception in the cause chain as its type plus the innermost code
|
||||
locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by
|
||||
KeyError at ...``. An ``OSError`` keeps its errno and strerror, which never
|
||||
carry payload. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full
|
||||
traceback, messages included.
|
||||
"""
|
||||
if content_logging_enabled():
|
||||
return "".join(traceback.format_exception(exc)).rstrip()
|
||||
parts: list[str] = []
|
||||
seen: set[int] = set()
|
||||
current: BaseException | None = exc
|
||||
while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:
|
||||
seen.add(id(current))
|
||||
parts.append(_describe_one(current))
|
||||
current = current.__cause__ or (
|
||||
None if current.__suppress_context__ else current.__context__
|
||||
)
|
||||
return "; caused by ".join(parts)
|
||||
|
||||
|
||||
def _describe_one(exc: BaseException) -> str:
|
||||
text = type(exc).__qualname__
|
||||
if isinstance(exc, OSError) and exc.errno is not None:
|
||||
# strerror is free text in some OSError subclasses (ssl.SSLError), so bound it.
|
||||
text += f" [Errno {exc.errno}] {str(exc.strerror)[:_ID_MAX_CHARS]}"
|
||||
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
|
||||
if frames:
|
||||
where = " <- ".join(
|
||||
f"{_short_path(frame.filename)}:{frame.lineno} in {frame.name}"
|
||||
for frame in reversed(frames)
|
||||
)
|
||||
text += f" at {where}"
|
||||
return text
|
||||
|
||||
|
||||
def _short_path(path: str) -> str:
|
||||
"""``headroom/<module path>`` for Headroom's own files, the file name for anything else."""
|
||||
normalized = path.replace("\\", "/")
|
||||
if normalized.startswith(_PACKAGE_DIR + "/"):
|
||||
return "headroom/" + normalized[len(_PACKAGE_DIR) + 1 :]
|
||||
return normalized.rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def redact_url(url: str) -> str:
|
||||
"""Return ``url`` without userinfo, query or fragment, which can hold credentials.
|
||||
|
||||
The path is kept, so do not pass URLs that carry a secret in the path
|
||||
(webhook URLs, ``/bot<token>/``). The host comes back lowercased.
|
||||
"""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
host = parts.hostname or ""
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
return "<unparseable url>"
|
||||
if ":" in host:
|
||||
host = f"[{host}]"
|
||||
netloc = f"{host}:{port}" if port is not None else host
|
||||
redacted = urlunsplit((parts.scheme, netloc, parts.path, "", ""))
|
||||
return f"{redacted}?<redacted>" if parts.query else redacted
|
||||
|
||||
|
||||
def safe_id(value: object) -> str:
|
||||
"""Render a client- or model-supplied id as one bounded log token.
|
||||
|
||||
Uses ``repr`` so a newline in the id cannot forge a second log line, and
|
||||
cuts it to a fixed length so an oversized id cannot flood the log.
|
||||
"""
|
||||
text = repr(value)
|
||||
if len(text) <= _ID_MAX_CHARS:
|
||||
return text
|
||||
return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"
|
||||
|
||||
|
||||
class WarnOnce:
|
||||
"""Warn once per key, for at most ``limit`` keys over the process lifetime.
|
||||
|
||||
Past the limit it logs one overflow WARNING and then stays quiet, so a
|
||||
failure that hits many distinct keys (paths, rows, models) cannot flood the
|
||||
log on every pass. A key is used up only when WARNING is enabled, so a
|
||||
warning suppressed by the log level is still emitted later. ``forget``
|
||||
re-arms a key once its failure has cleared.
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int, what: str) -> None:
|
||||
if limit < 1:
|
||||
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
|
||||
self._limit = limit
|
||||
self._what = what
|
||||
self._keys: set[Hashable] = set()
|
||||
self._overflowed = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def first(self, key: Hashable, log: logging.Logger) -> bool:
|
||||
"""Return True if the caller should log this key's WARNING now."""
|
||||
if not log.isEnabledFor(logging.WARNING):
|
||||
return False
|
||||
with self._lock:
|
||||
if key in self._keys:
|
||||
return False
|
||||
if len(self._keys) < self._limit:
|
||||
self._keys.add(key)
|
||||
return True
|
||||
report_overflow = not self._overflowed
|
||||
self._overflowed = True
|
||||
if report_overflow:
|
||||
log.warning(
|
||||
"More than %d distinct %s; further ones are logged at debug only",
|
||||
self._limit,
|
||||
self._what,
|
||||
)
|
||||
return False
|
||||
|
||||
def forget(self, key: Hashable) -> None:
|
||||
"""Re-arm ``key`` so its next failure warns again."""
|
||||
with self._lock:
|
||||
self._keys.discard(key)
|
||||
@@ -14,7 +14,8 @@ import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from headroom.log_safety import content_logging_enabled, redact_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:
|
||||
"""
|
||||
if getattr(config, "stateless", False):
|
||||
return "off"
|
||||
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
|
||||
if raw in ("full", "all", "content"):
|
||||
if content_logging_enabled():
|
||||
return "full"
|
||||
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
|
||||
if raw in ("1", "true", "yes", "on", "redacted"):
|
||||
return "redacted"
|
||||
return "off"
|
||||
@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _redact_url(url: str) -> str:
|
||||
"""Drop the query string and fragment from ``url``.
|
||||
|
||||
Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
|
||||
query is never needed to debug a rejected request, so it is elided in every
|
||||
mode — ``full`` opts in to prompt content, not to credentials.
|
||||
"""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
except ValueError:
|
||||
return "<unparseable url>"
|
||||
if not parts.query and not parts.fragment:
|
||||
return url
|
||||
redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
return f"{redacted}?<redacted>" if parts.query else redacted
|
||||
|
||||
|
||||
def _decode_sent_body(body: Any) -> Any:
|
||||
"""Decode the exact bytes sent upstream back to JSON for the dump."""
|
||||
if not isinstance(body, (bytes, bytearray)):
|
||||
@@ -119,7 +103,7 @@ def write_upstream_error_dump(
|
||||
payload = json.dumps(
|
||||
{
|
||||
"request_id": request_id,
|
||||
"url": _redact_url(url),
|
||||
"url": redact_url(url),
|
||||
"status": status,
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom import log_safety
|
||||
from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
|
||||
|
||||
CANARY = "sk-live-CANARY tool output: patient record 42"
|
||||
|
||||
|
||||
def _raise_chain() -> BaseException:
|
||||
try:
|
||||
try:
|
||||
raise KeyError(CANARY)
|
||||
except KeyError as inner:
|
||||
raise ValueError(f"could not parse {CANARY}") from inner
|
||||
except ValueError as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def test_exception_description_names_types_and_places_but_not_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
text = describe_exception(_raise_chain())
|
||||
|
||||
assert "CANARY" not in text
|
||||
assert text.startswith("ValueError at ")
|
||||
assert "test_log_safety.py:" in text
|
||||
assert "in _raise_chain" in text
|
||||
assert "; caused by KeyError at " in text
|
||||
|
||||
|
||||
def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))
|
||||
|
||||
assert text == "PermissionError [Errno 13] Permission denied"
|
||||
|
||||
|
||||
def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
|
||||
|
||||
text = describe_exception(_raise_chain())
|
||||
|
||||
assert "Traceback" in text
|
||||
assert CANARY in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url", "expected"),
|
||||
[
|
||||
(
|
||||
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
|
||||
"https://proxy.example:8443/v1?<redacted>",
|
||||
),
|
||||
("http://127.0.0.1:8787/stats", "http://127.0.0.1:8787/stats"),
|
||||
("http://[::1]:8787/", "http://[::1]:8787/"),
|
||||
("http://[::1", "<unparseable url>"),
|
||||
],
|
||||
)
|
||||
def test_redact_url_drops_userinfo_and_query(url: str, expected: str) -> None:
|
||||
assert redact_url(url) == expected
|
||||
|
||||
|
||||
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
|
||||
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
|
||||
long = safe_id("x" * 10_000)
|
||||
assert len(long) < 120
|
||||
assert long.endswith("chars)")
|
||||
|
||||
|
||||
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
|
||||
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
||||
|
||||
|
||||
def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.bounded")
|
||||
guard = WarnOnce(limit=64, what="unwritable ledgers")
|
||||
|
||||
with caplog.at_level(logging.INFO, logger=log.name):
|
||||
for _ in range(3):
|
||||
for path in range(65):
|
||||
if guard.first(path, log):
|
||||
log.warning("ledger %d failed", path)
|
||||
|
||||
messages = _warnings(caplog)
|
||||
assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass
|
||||
assert messages[-1] == (
|
||||
"More than 64 distinct unwritable ledgers; further ones are logged at debug only"
|
||||
)
|
||||
|
||||
|
||||
def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.level")
|
||||
guard = WarnOnce(limit=4, what="failures")
|
||||
log.setLevel(logging.ERROR)
|
||||
try:
|
||||
assert guard.first("k", log) is False
|
||||
log.setLevel(logging.INFO)
|
||||
assert guard.first("k", log) is True
|
||||
assert guard.first("k", log) is False
|
||||
finally:
|
||||
log.setLevel(logging.NOTSET)
|
||||
|
||||
|
||||
def test_forget_rearms_a_key() -> None:
|
||||
log = logging.getLogger("headroom.test_log_safety.forget")
|
||||
guard = WarnOnce(limit=4, what="failures")
|
||||
|
||||
assert guard.first("path", log) is True
|
||||
guard.forget("path")
|
||||
assert guard.first("path", log) is True
|
||||
|
||||
|
||||
def test_frame_paths_are_package_relative_for_headroom_and_bare_otherwise() -> None:
|
||||
package_dir = os.path.dirname(log_safety.__file__)
|
||||
|
||||
assert log_safety._short_path(os.path.join(package_dir, "proxy", "server.py")) == (
|
||||
"headroom/proxy/server.py"
|
||||
)
|
||||
assert log_safety._short_path("/home/runner/work/headroom/headroom/tests/test_x.py") == (
|
||||
"test_x.py"
|
||||
)
|
||||
assert log_safety._short_path("/repo/headroom/.venv/lib/site-packages/httpx/_client.py") == (
|
||||
"_client.py"
|
||||
)
|
||||
|
||||
|
||||
def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
|
||||
try:
|
||||
try:
|
||||
raise KeyError(CANARY)
|
||||
except KeyError:
|
||||
raise ValueError("bad") from None
|
||||
except ValueError as exc:
|
||||
text = describe_exception(exc)
|
||||
|
||||
assert "KeyError" not in text
|
||||
|
||||
|
||||
def test_a_cause_cycle_is_described_once() -> None:
|
||||
first = ValueError(CANARY)
|
||||
second = KeyError(CANARY)
|
||||
first.__cause__ = second
|
||||
second.__cause__ = first
|
||||
|
||||
assert describe_exception(first) == "ValueError; caused by KeyError"
|
||||
Reference in New Issue
Block a user