Merge remote-tracking branch 'origin/cleanup/log-safety-helpers' into cleanup/proxy-core-a-surface-ignored-config

This commit is contained in:
Tejas Chopra
2026-10-10 09:14:03 -07:00
4 changed files with 447 additions and 25 deletions
+194
View File
@@ -0,0 +1,194 @@
"""Payload-safe building blocks for log lines.
Log records are kept, rotated, attached to bug reports and shipped to log
collectors, so no level may carry request content, tool output or
credentials. Exception messages are the usual leak: a compressor, hook or
storage backend can quote the payload or a connection URL in its error. These
helpers describe a failure by what is safe to keep (exception types, code
locations, errno, ids, counts) and leave the message out.
Content is logged only when the operator opts in to it explicitly with
``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.
"""
from __future__ import annotations
import logging
import os
import threading
import time
import traceback
from collections.abc import Hashable
from urllib.parse import urlsplit, urlunsplit
CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"
_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
_MAX_FRAMES = 3
_MAX_CHAIN = 4
_ID_MAX_CHARS = 80
_MAX_ERRNO = 2**31 - 1
_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)).replace("\\", "/")
def content_logging_enabled() -> bool:
"""True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""
return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES
def describe_exception(exc: BaseException) -> str:
"""Describe ``exc`` for a log line without its message.
Gives each exception in the cause chain as its type plus the innermost code
locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by
KeyError at ...``. An ``OSError`` keeps its errno and the system's text for
it, never the exception's own strerror. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full
traceback, messages included.
"""
if content_logging_enabled():
return "".join(traceback.format_exception(exc)).rstrip()
parts: list[str] = []
seen: set[int] = set()
current: BaseException | None = exc
while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:
seen.add(id(current))
parts.append(_describe_one(current))
current = current.__cause__ or (
None if current.__suppress_context__ else current.__context__
)
return "; caused by ".join(parts)
def _describe_one(exc: BaseException) -> str:
text = type(exc).__qualname__
if isinstance(exc, OSError) and isinstance(exc.errno, int):
# The system's own text for the errno; exc.strerror is free text in some
# subclasses (ssl.SSLError) and can quote a path or a peer's message.
# An out-of-range errno is not formatted at all: converting a huge int
# to text can itself raise, and a log helper must never raise.
if 0 <= exc.errno <= _MAX_ERRNO:
text += f" [Errno {exc.errno}] {_errno_text(exc.errno)}"
else:
text += " [Errno out of range]"
frames = traceback.extract_tb(exc.__traceback__)[-_MAX_FRAMES:]
if frames:
where = " <- ".join(
f"{_short_path(frame.filename)}:{frame.lineno} in {frame.name}"
for frame in reversed(frames)
)
text += f" at {where}"
return text
def _errno_text(errno: int) -> str:
try:
return os.strerror(errno)
except (ValueError, OverflowError):
return "unknown error"
def _short_path(path: str) -> str:
"""``headroom/<module path>`` for Headroom's own files, the file name for anything else."""
normalized = path.replace("\\", "/")
if normalized.startswith(_PACKAGE_DIR + "/"):
return "headroom/" + normalized[len(_PACKAGE_DIR) + 1 :]
return normalized.rsplit("/", 1)[-1]
def redact_url(url: str) -> str:
"""Return ``url`` as scheme, host and port only, safe for any log line.
Userinfo, query and fragment are always dropped, and so is the path, since
a path segment can carry a credential (``/bot<token>/``, webhook URLs) and
no rule tells such a segment from a route name. The path is kept only with
the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes
back lowercased.
"""
try:
parts = urlsplit(url)
host = parts.hostname or ""
port = parts.port
except ValueError:
return "<unparseable url>"
if ":" in host:
host = f"[{host}]"
netloc = f"{host}:{port}" if port is not None else host
if content_logging_enabled():
path = parts.path
else:
path = "/<path>" if parts.path not in ("", "/") else parts.path
redacted = urlunsplit((parts.scheme, netloc, path, "", ""))
return f"{redacted}?<redacted>" if parts.query else redacted
def safe_id(value: object) -> str:
"""Render a client- or model-supplied id as one bounded log token.
Uses ``repr`` so a newline in the id cannot forge a second log line, and
cuts it to a fixed length so an oversized id cannot flood the log.
"""
text = repr(value)
if len(text) <= _ID_MAX_CHARS:
return text
return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"
class WarnOnce:
"""Warn once per key per window, for at most ``limit`` keys per window.
The first warning opens a window of ``window_seconds`` (an hour by
default). Inside it each key warns once; past ``limit`` keys it logs one
overflow WARNING and stays quiet, so a failure that hits many distinct keys
(paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings
per window. When the window ends the guard starts over, so a failure that
keeps recurring, or a newly broken key, warns again once an hour. A key is
used up only when WARNING is enabled, so a warning suppressed by the log
level is still emitted later. ``forget`` re-arms a key once its failure has
cleared; it cannot reopen the cap after the overflow notice.
"""
def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:
if limit < 1:
raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
if window_seconds <= 0:
raise ValueError(f"WarnOnce window_seconds must be positive, got {window_seconds}")
self._limit = limit
self._what = what
self._window = window_seconds
self._keys: set[Hashable] = set()
self._window_start: float | None = None
self._overflowed = False
self._lock = threading.Lock()
def first(self, key: Hashable, log: logging.Logger) -> bool:
"""Return True if the caller should log this key's WARNING now."""
if not log.isEnabledFor(logging.WARNING):
return False
with self._lock:
now = time.monotonic()
if self._window_start is not None and now - self._window_start >= self._window:
self._keys.clear()
self._overflowed = False
self._window_start = None
if key in self._keys:
return False
if not self._overflowed and len(self._keys) < self._limit:
self._keys.add(key)
if self._window_start is None:
self._window_start = now
return True
report_overflow = not self._overflowed
self._overflowed = True
if report_overflow:
log.warning(
"More than %d distinct %s; further ones are logged at debug only "
"for the next %d minutes",
self._limit,
self._what,
max(1, round(self._window / 60)),
)
return False
def forget(self, key: Hashable) -> None:
"""Re-arm ``key`` so its next failure warns again."""
with self._lock:
self._keys.discard(key)
+5 -21
View File
@@ -14,7 +14,8 @@ import os
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
from headroom.log_safety import content_logging_enabled, redact_url
logger = logging.getLogger(__name__)
@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:
"""
if getattr(config, "stateless", False):
return "off"
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
if raw in ("full", "all", "content"):
if content_logging_enabled():
return "full"
raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
if raw in ("1", "true", "yes", "on", "redacted"):
return "redacted"
return "off"
@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
return value
def _redact_url(url: str) -> str:
"""Drop the query string and fragment from ``url``.
Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
query is never needed to debug a rejected request, so it is elided in every
mode — ``full`` opts in to prompt content, not to credentials.
"""
try:
parts = urlsplit(url)
except ValueError:
return "<unparseable url>"
if not parts.query and not parts.fragment:
return url
redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
return f"{redacted}?<redacted>" if parts.query else redacted
def _decode_sent_body(body: Any) -> Any:
"""Decode the exact bytes sent upstream back to JSON for the dump."""
if not isinstance(body, (bytes, bytearray)):
@@ -119,7 +103,7 @@ def write_upstream_error_dump(
payload = json.dumps(
{
"request_id": request_id,
"url": _redact_url(url),
"url": redact_url(url),
"status": status,
"provider": provider,
"model": model,
+19 -4
View File
@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)
assert path is not None
text = path.read_text()
assert "AIzaSECRET" not in text
assert json.loads(text)["url"] == (
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
"?<redacted>"
)
if mode == "full": # the content opt-in keeps the route, still never the key
expected = (
"https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
)
else: # a path segment can carry a credential, so it is dropped without the opt-in
expected = "https://generativelanguage.googleapis.com/<path>"
assert json.loads(text)["url"] == expected + "?<redacted>"
@pytest.mark.parametrize(
"url",
["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],
)
def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")
path = _write(url=url)
assert path is not None
text = path.read_text()
assert "CANARY" not in text and "%43ANARY" not in text
def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
+229
View File
@@ -0,0 +1,229 @@
"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""
from __future__ import annotations
import logging
import os
import pytest
from headroom import log_safety
from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
CANARY = "sk-live-CANARY tool output: patient record 42"
def _raise_chain() -> BaseException:
try:
try:
raise KeyError(CANARY)
except KeyError as inner:
raise ValueError(f"could not parse {CANARY}") from inner
except ValueError as exc:
return exc
def test_exception_description_names_types_and_places_but_not_messages(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
text = describe_exception(_raise_chain())
assert "CANARY" not in text
assert text.startswith("ValueError at ")
assert "test_log_safety.py:" in text
assert "in _raise_chain" in text
assert "; caused by KeyError at " in text
def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))
assert text == "PermissionError [Errno 13] Permission denied"
def test_oserror_free_text_strerror_is_replaced_by_the_system_text() -> None:
text = describe_exception(OSError(2, f"token={CANARY}"))
assert "CANARY" not in text
assert text == f"FileNotFoundError [Errno 2] {os.strerror(2)}"
def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
text = describe_exception(_raise_chain())
assert "Traceback" in text
assert CANARY in text
@pytest.mark.parametrize(
("url", "expected"),
[
(
"https://user:hunter2@proxy.example:8443/v1?key=abc#frag",
"https://proxy.example:8443/<path>?<redacted>",
),
("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),
("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),
("http://127.0.0.1:8787", "http://127.0.0.1:8787"),
("http://[::1]:8787/", "http://[::1]:8787/"),
("http://[::1", "<unparseable url>"),
],
)
def test_redact_url_keeps_only_scheme_host_and_port(
url: str, expected: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
assert redact_url(url) == expected
def test_content_opt_in_keeps_the_path_but_never_credentials(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
assert redact_url("https://u:pw@upstream.example/v1/messages?key=abc") == (
"https://upstream.example/v1/messages?<redacted>"
)
def test_safe_id_escapes_newlines_and_bounds_length() -> None:
assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
long = safe_id("x" * 10_000)
assert len(long) < 120
assert long.endswith("chars)")
def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:
log = logging.getLogger("headroom.test_log_safety.bounded")
guard = WarnOnce(limit=64, what="unwritable ledgers")
with caplog.at_level(logging.INFO, logger=log.name):
for _ in range(3):
for path in range(65):
if guard.first(path, log):
log.warning("ledger %d failed", path)
messages = _warnings(caplog)
assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass
assert messages[-1] == (
"More than 64 distinct unwritable ledgers; further ones are logged at debug only "
"for the next 60 minutes"
)
def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:
log = logging.getLogger("headroom.test_log_safety.level")
guard = WarnOnce(limit=4, what="failures")
log.setLevel(logging.ERROR)
try:
assert guard.first("k", log) is False
log.setLevel(logging.INFO)
assert guard.first("k", log) is True
assert guard.first("k", log) is False
finally:
log.setLevel(logging.NOTSET)
def test_forget_rearms_a_key() -> None:
log = logging.getLogger("headroom.test_log_safety.forget")
guard = WarnOnce(limit=4, what="failures")
assert guard.first("path", log) is True
guard.forget("path")
assert guard.first("path", log) is True
def test_frame_paths_are_package_relative_for_headroom_and_bare_otherwise() -> None:
package_dir = os.path.dirname(log_safety.__file__)
assert log_safety._short_path(os.path.join(package_dir, "proxy", "server.py")) == (
"headroom/proxy/server.py"
)
assert log_safety._short_path("/home/runner/work/headroom/headroom/tests/test_x.py") == (
"test_x.py"
)
assert log_safety._short_path("/repo/headroom/.venv/lib/site-packages/httpx/_client.py") == (
"_client.py"
)
def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
try:
try:
raise KeyError(CANARY)
except KeyError:
raise ValueError("bad") from None
except ValueError as exc:
text = describe_exception(exc)
assert "KeyError" not in text
def test_a_cause_cycle_is_described_once() -> None:
first = ValueError(CANARY)
second = KeyError(CANARY)
first.__cause__ = second
second.__cause__ = first
assert describe_exception(first) == "ValueError; caused by KeyError"
def test_forgetting_after_overflow_does_not_reopen_the_cap(
caplog: pytest.LogCaptureFixture,
) -> None:
log = logging.getLogger("headroom.test_log_safety.reopen")
guard = WarnOnce(limit=2, what="failures")
with caplog.at_level(logging.INFO, logger=log.name):
assert guard.first("a", log) and guard.first("b", log)
assert guard.first("c", log) is False # overflow notice
for cycle in range(5):
guard.forget("a")
assert guard.first(f"new-{cycle}", log) is False
assert len(_warnings(caplog)) == 1
def test_warn_once_starts_over_after_the_window(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
log = logging.getLogger("headroom.test_log_safety.window")
guard = WarnOnce(limit=2, what="hook failures", window_seconds=600)
clock = [1000.0]
monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])
results = []
with caplog.at_level(logging.INFO, logger=log.name):
steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))
for key, advance in steps:
clock[0] += advance
results.append(guard.first(key, log))
# a and b warn and c is the overflow notice; inside the window a stays quiet
# and d is suppressed; once the window ends, the still-recurring a and a
# newly broken key both warn again.
assert results == [True, True, False, False, False, True, True]
assert len(_warnings(caplog)) == 1
def test_warn_once_needs_a_positive_window() -> None:
with pytest.raises(ValueError, match="window_seconds must be positive"):
WarnOnce(limit=1, what="failures", window_seconds=0)
@pytest.mark.parametrize("digits", [19, 10_001])
def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:
"""An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""
exc = OSError()
exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it
assert describe_exception(exc) == "OSError [Errno out of range]"