fix(cache): default CCR payload previews off; harden the log when they are on

Up to 4096 characters of retrieved tool-result content — source code, secrets
the redactor does not recognize, customer data — went into the always-on runtime
log by default, and that log was created at the process umask.

HEADROOM_LOG_PAYLOAD_PREVIEW now has to be turned on explicitly (1/true/yes/on);
unset logs byte counts only. Opting in swaps the proxy log's RotatingFileHandler
for one that creates the file 0600, on the first open and on every rollover.
This commit is contained in:
Tejas Chopra
2026-09-23 05:51:15 +02:00
parent 5ff4ea1ef9
commit ec014d64fc
5 changed files with 142 additions and 11 deletions
+1
View File
@@ -296,6 +296,7 @@ Rewrite the upstream model per request — for example, send small, tool-free ca
| `--telemetry` / `HEADROOM_TELEMETRY` | off | **Local-only** usage stats for your own `/stats`, `/metrics`, and dashboard. Nothing leaves the machine. |
| `--log-file` / `HEADROOM_LOG_FILE` | none | JSONL request/response log. |
| `--log-messages` | `false` | Include full message bodies in the log (may contain sensitive data). |
| `HEADROOM_LOG_PAYLOAD_PREVIEW` | `off` | Include up to 4 KB of the retrieved [CCR](/docs/ccr) payload in each `headroom_retrieve` line of the runtime log (`~/.headroom/logs/proxy-<port>.log`). Off by default: the payload is verbatim tool-result content and can carry source code, credentials the redactor does not recognize, or customer data; the log otherwise records byte counts only. Turning it on (`1`/`true`/`yes`/`on`) also creates the log file `0600`. Do not share a log captured with this on. |
| `HEADROOM_LOG_LEVEL` | `warning` | uvicorn's log level (`critical`, `error`, `warning`, `info`, `debug`, `trace`). Raise to `info` for the per-request access log when diagnosing a deployed proxy. An unrecognized value warns and falls back to `warning`. |
| `HEADROOM_OTEL_METRICS_ENABLED` | `false` | Export OpenTelemetry metrics (`HEADROOM_OTEL_METRICS_ENDPOINT`, …). See [OTLP export](/docs/metrics#opentelemetry-otlp-export). |
| `HEADROOM_LANGFUSE_ENABLED` | `false` | Emit Langfuse traces (`LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY`). |
+9 -5
View File
@@ -53,9 +53,12 @@ DEFAULT_CCR_TTL_SECONDS = 1800 # session-scale; override via HEADROOM_CCR_TTL_S
CCR_TTL_SECONDS_ENV = "HEADROOM_CCR_TTL_SECONDS"
_RETRIEVAL_LOG_PREVIEW_CHARS = 4096
# Previews carry verbatim tool-result content (post-redaction), which makes
# proxy.log too sensitive for users to share in bug reports. Set to
# 0/false/no/off to log byte counts only.
# Previews carry verbatim tool-result content (post-redaction) — source code,
# credentials the redactor does not recognize, customer data. That is not
# something the always-on runtime log should hold, so previews are OFF unless
# an operator turns them on with 1/true/yes/on; the log then records byte
# counts only. Opting in also hardens the log file to 0600 (see
# ``headroom/proxy/helpers.py:_setup_file_logging``).
PAYLOAD_PREVIEW_ENV = "HEADROOM_LOG_PAYLOAD_PREVIEW"
_SECRET_KEY_VALUE_RE = re.compile(
r"(?i)\b([A-Z0-9_-]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH)[A-Z0-9_-]*)"
@@ -114,10 +117,11 @@ def _redact_retrieval_log_payload(payload: str) -> str:
def _payload_preview_enabled() -> bool:
"""True only when an operator has explicitly opted in. Default: off."""
raw = os.environ.get(PAYLOAD_PREVIEW_ENV)
if raw is None:
return True
return raw.strip().lower() not in ("0", "false", "no", "off")
return False
return raw.strip().lower() in ("1", "true", "yes", "on")
def _payload_for_retrieval_log(payload: str) -> dict[str, Any]:
+32 -2
View File
@@ -20,6 +20,7 @@ import time
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, cast
@@ -1723,6 +1724,28 @@ def _headroom_log_dir() -> Path:
_PROXY_LOG_HANDLER_NAME = "headroom.proxy.file"
class _OwnerOnlyRotatingFileHandler(RotatingFileHandler):
"""RotatingFileHandler whose log file is only readable by its owner.
``logging`` opens the stream itself — once at construction and again for
every rollover — so the process umask decides the mode and there is no
hook to pass one. Pre-creating the file with an explicit 0600 closes the
window before the handler's own ``open()`` runs (which does not change the
mode of a file that already exists), and the chmod covers a log left at
0644 by an earlier run.
"""
def _open(self): # type: ignore[no-untyped-def]
try:
os.close(os.open(self.baseFilename, os.O_CREAT | os.O_APPEND | os.O_WRONLY, 0o600))
os.chmod(self.baseFilename, 0o600)
except OSError:
# Non-fatal: an unwritable path fails on the real open() below,
# where the caller already handles OSError.
pass
return super()._open()
def _setup_file_logging(
port: int | None = None,
*,
@@ -1737,8 +1760,15 @@ def _setup_file_logging(
The file is keyed by *port* so concurrent instances rotate separate logs.
Multi-worker callers also pass *process_id* so same-port workers cannot
race during rollover. When *port* is omitted the legacy shared name is used.
When ``HEADROOM_LOG_PAYLOAD_PREVIEW`` is on, the log holds verbatim
tool-result content, so the file is created 0600 instead of at the umask.
"""
from logging.handlers import RotatingFileHandler
from headroom.cache.compression_store import _payload_preview_enabled
handler_cls = (
_OwnerOnlyRotatingFileHandler if _payload_preview_enabled() else RotatingFileHandler
)
try:
log_dir = _headroom_log_dir()
@@ -1763,7 +1793,7 @@ def _setup_file_logging(
]
if any(Path(h.baseFilename) == log_path for h in existing):
return
handler = RotatingFileHandler(
handler = handler_cls(
log_path,
maxBytes=10 * 1024 * 1024, # 10 MB
backupCount=5,
+96 -2
View File
@@ -9,6 +9,11 @@ Covers the three pieces added for the uncached-tail investigation:
from __future__ import annotations
import logging
import stat
from contextlib import contextmanager
from pathlib import Path
import pytest
from headroom.cache.compression_store import _payload_for_retrieval_log
from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
@@ -125,10 +130,26 @@ def test_payload_preview_disabled_omits_content(monkeypatch) -> None:
assert event["payload_truncated"] is True
def test_payload_preview_enabled_by_default(monkeypatch) -> None:
def test_payload_preview_disabled_by_default(monkeypatch) -> None:
"""Unset means off: the log gets byte counts, never the content."""
monkeypatch.delenv("HEADROOM_LOG_PAYLOAD_PREVIEW", raising=False)
event = _payload_for_retrieval_log("hello world")
assert event["payload_preview"] == "hello world"
assert event["payload_preview"] == ""
assert event["payload_preview_chars"] == 0
assert event["payload_chars"] == len("hello world")
@pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "on"])
def test_payload_preview_opt_in_values(monkeypatch, value: str) -> None:
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", value)
assert _payload_for_retrieval_log("hello world")["payload_preview"] == "hello world"
@pytest.mark.parametrize("value", ["", "0", "off", "no", "maybe", " "])
def test_payload_preview_stays_off_for_anything_else(monkeypatch, value: str) -> None:
"""Only an explicit opt-in turns previews on — a typo must not."""
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", value)
assert _payload_for_retrieval_log("hello world")["payload_preview"] == ""
def test_append_context_skips_breakpointed_text_block() -> None:
@@ -178,3 +199,76 @@ def test_count_cache_breakpoints_tolerates_malformed_shapes() -> None:
empty = count_cache_breakpoints(None, None, None)
assert empty["total"] == 0
assert empty["message_count"] == 0
# --- the runtime log file itself -------------------------------------------
#
# _payload_for_retrieval_log decides what goes into the record;
# _setup_file_logging decides who can read the file it lands in. Both halves
# of the default-off guarantee are checked against a real log on disk.
@contextmanager
def _proxy_log(tmp_path, monkeypatch, port: int):
"""Point the workspace at *tmp_path*, install the real proxy log handler."""
from headroom.proxy.helpers import _PROXY_LOG_HANDLER_NAME, _setup_file_logging
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
headroom_logger = logging.getLogger("headroom")
before = list(headroom_logger.handlers)
propagate = headroom_logger.propagate
try:
_setup_file_logging(port)
[handler] = [h for h in headroom_logger.handlers if h.name == _PROXY_LOG_HANDLER_NAME]
yield Path(handler.baseFilename)
finally:
for handler in list(headroom_logger.handlers):
if handler not in before:
headroom_logger.removeHandler(handler)
handler.close()
headroom_logger.propagate = propagate
def test_runtime_log_holds_no_payload_text_at_default_settings(tmp_path, monkeypatch) -> None:
"""A retrieval on default settings leaves byte counts in the log, not content."""
from headroom.cache.compression_store import CompressionStore
monkeypatch.delenv("HEADROOM_LOG_PAYLOAD_PREVIEW", raising=False)
secret = "BEGIN-CUSTOMER-DATA ssn=123-45-6789 def sekrit(): pass END-CUSTOMER-DATA"
with _proxy_log(tmp_path, monkeypatch, 18801) as log_path:
store = CompressionStore(enable_feedback=False)
assert store.retrieve(store.store(original=secret, compressed="[compressed]")) is not None
logging.getLogger("headroom").handlers[-1].flush()
text = log_path.read_text(encoding="utf-8")
assert "event=headroom_retrieve" in text, "the retrieval was not logged at all"
assert secret not in text
assert "123-45-6789" not in text
assert f'"payload_chars":{len(secret)}' in text
assert '"payload_preview":""' in text
def test_runtime_log_is_owner_only_when_preview_enabled(tmp_path, monkeypatch) -> None:
"""Opting in to previews hardens the log the previews land in."""
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
with _proxy_log(tmp_path, monkeypatch, 18802) as log_path:
assert log_path.exists()
assert stat.S_IMODE(log_path.stat().st_mode) == 0o600
def test_runtime_log_hardening_survives_a_pre_existing_world_readable_log(
tmp_path, monkeypatch
) -> None:
"""O_CREAT's mode does not apply to an existing file; the chmod must."""
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
from headroom import paths as _paths
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
stale = _paths.proxy_log_path(18803)
stale.parent.mkdir(parents=True, exist_ok=True)
stale.write_text("from an older, unhardened run\n", encoding="utf-8")
stale.chmod(0o644)
with _proxy_log(tmp_path, monkeypatch, 18803) as log_path:
assert stat.S_IMODE(log_path.stat().st_mode) == 0o600
+4 -2
View File
@@ -60,7 +60,8 @@ def _capture_headroom_retrieve_events():
logger.setLevel(previous_level)
def test_retrieve_logs_payload_preview():
def test_retrieve_logs_payload_preview(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
store = CompressionStore(enable_feedback=False)
hash_key = store.store(
original="secret-ish payload for operator debugging",
@@ -85,7 +86,8 @@ def test_retrieve_logs_payload_preview():
assert events[0]["tool_name"] == "tool_a"
def test_retrieve_log_redacts_secret_payload_values():
def test_retrieve_log_redacts_secret_payload_values(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
store = CompressionStore(enable_feedback=False)
hash_key = store.store(
original="OPENAI_API_KEY=sk-proj-secret1234567890 Authorization: Bearer token123456789",