mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(cache): default CCR payload previews off; harden the log when they are on
Up to 4096 characters of retrieved tool-result content — source code, secrets the redactor does not recognize, customer data — went into the always-on runtime log by default, and that log was created at the process umask. HEADROOM_LOG_PAYLOAD_PREVIEW now has to be turned on explicitly (1/true/yes/on); unset logs byte counts only. Opting in swaps the proxy log's RotatingFileHandler for one that creates the file 0600, on the first open and on every rollover.
This commit is contained in:
@@ -296,6 +296,7 @@ Rewrite the upstream model per request — for example, send small, tool-free ca
|
||||
| `--telemetry` / `HEADROOM_TELEMETRY` | off | **Local-only** usage stats for your own `/stats`, `/metrics`, and dashboard. Nothing leaves the machine. |
|
||||
| `--log-file` / `HEADROOM_LOG_FILE` | none | JSONL request/response log. |
|
||||
| `--log-messages` | `false` | Include full message bodies in the log (may contain sensitive data). |
|
||||
| `HEADROOM_LOG_PAYLOAD_PREVIEW` | `off` | Include up to 4 KB of the retrieved [CCR](/docs/ccr) payload in each `headroom_retrieve` line of the runtime log (`~/.headroom/logs/proxy-<port>.log`). Off by default: the payload is verbatim tool-result content and can carry source code, credentials the redactor does not recognize, or customer data; the log otherwise records byte counts only. Turning it on (`1`/`true`/`yes`/`on`) also creates the log file `0600`. Do not share a log captured with this on. |
|
||||
| `HEADROOM_LOG_LEVEL` | `warning` | uvicorn's log level (`critical`, `error`, `warning`, `info`, `debug`, `trace`). Raise to `info` for the per-request access log when diagnosing a deployed proxy. An unrecognized value warns and falls back to `warning`. |
|
||||
| `HEADROOM_OTEL_METRICS_ENABLED` | `false` | Export OpenTelemetry metrics (`HEADROOM_OTEL_METRICS_ENDPOINT`, …). See [OTLP export](/docs/metrics#opentelemetry-otlp-export). |
|
||||
| `HEADROOM_LANGFUSE_ENABLED` | `false` | Emit Langfuse traces (`LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY`). |
|
||||
|
||||
Vendored
+9
-5
@@ -53,9 +53,12 @@ DEFAULT_CCR_TTL_SECONDS = 1800 # session-scale; override via HEADROOM_CCR_TTL_S
|
||||
CCR_TTL_SECONDS_ENV = "HEADROOM_CCR_TTL_SECONDS"
|
||||
|
||||
_RETRIEVAL_LOG_PREVIEW_CHARS = 4096
|
||||
# Previews carry verbatim tool-result content (post-redaction), which makes
|
||||
# proxy.log too sensitive for users to share in bug reports. Set to
|
||||
# 0/false/no/off to log byte counts only.
|
||||
# Previews carry verbatim tool-result content (post-redaction) — source code,
|
||||
# credentials the redactor does not recognize, customer data. That is not
|
||||
# something the always-on runtime log should hold, so previews are OFF unless
|
||||
# an operator turns them on with 1/true/yes/on; the log then records byte
|
||||
# counts only. Opting in also hardens the log file to 0600 (see
|
||||
# ``headroom/proxy/helpers.py:_setup_file_logging``).
|
||||
PAYLOAD_PREVIEW_ENV = "HEADROOM_LOG_PAYLOAD_PREVIEW"
|
||||
_SECRET_KEY_VALUE_RE = re.compile(
|
||||
r"(?i)\b([A-Z0-9_-]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH)[A-Z0-9_-]*)"
|
||||
@@ -114,10 +117,11 @@ def _redact_retrieval_log_payload(payload: str) -> str:
|
||||
|
||||
|
||||
def _payload_preview_enabled() -> bool:
|
||||
"""True only when an operator has explicitly opted in. Default: off."""
|
||||
raw = os.environ.get(PAYLOAD_PREVIEW_ENV)
|
||||
if raw is None:
|
||||
return True
|
||||
return raw.strip().lower() not in ("0", "false", "no", "off")
|
||||
return False
|
||||
return raw.strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _payload_for_retrieval_log(payload: str) -> dict[str, Any]:
|
||||
|
||||
@@ -20,6 +20,7 @@ import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
@@ -1723,6 +1724,28 @@ def _headroom_log_dir() -> Path:
|
||||
_PROXY_LOG_HANDLER_NAME = "headroom.proxy.file"
|
||||
|
||||
|
||||
class _OwnerOnlyRotatingFileHandler(RotatingFileHandler):
|
||||
"""RotatingFileHandler whose log file is only readable by its owner.
|
||||
|
||||
``logging`` opens the stream itself — once at construction and again for
|
||||
every rollover — so the process umask decides the mode and there is no
|
||||
hook to pass one. Pre-creating the file with an explicit 0600 closes the
|
||||
window before the handler's own ``open()`` runs (which does not change the
|
||||
mode of a file that already exists), and the chmod covers a log left at
|
||||
0644 by an earlier run.
|
||||
"""
|
||||
|
||||
def _open(self): # type: ignore[no-untyped-def]
|
||||
try:
|
||||
os.close(os.open(self.baseFilename, os.O_CREAT | os.O_APPEND | os.O_WRONLY, 0o600))
|
||||
os.chmod(self.baseFilename, 0o600)
|
||||
except OSError:
|
||||
# Non-fatal: an unwritable path fails on the real open() below,
|
||||
# where the caller already handles OSError.
|
||||
pass
|
||||
return super()._open()
|
||||
|
||||
|
||||
def _setup_file_logging(
|
||||
port: int | None = None,
|
||||
*,
|
||||
@@ -1737,8 +1760,15 @@ def _setup_file_logging(
|
||||
The file is keyed by *port* so concurrent instances rotate separate logs.
|
||||
Multi-worker callers also pass *process_id* so same-port workers cannot
|
||||
race during rollover. When *port* is omitted the legacy shared name is used.
|
||||
|
||||
When ``HEADROOM_LOG_PAYLOAD_PREVIEW`` is on, the log holds verbatim
|
||||
tool-result content, so the file is created 0600 instead of at the umask.
|
||||
"""
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from headroom.cache.compression_store import _payload_preview_enabled
|
||||
|
||||
handler_cls = (
|
||||
_OwnerOnlyRotatingFileHandler if _payload_preview_enabled() else RotatingFileHandler
|
||||
)
|
||||
|
||||
try:
|
||||
log_dir = _headroom_log_dir()
|
||||
@@ -1763,7 +1793,7 @@ def _setup_file_logging(
|
||||
]
|
||||
if any(Path(h.baseFilename) == log_path for h in existing):
|
||||
return
|
||||
handler = RotatingFileHandler(
|
||||
handler = handler_cls(
|
||||
log_path,
|
||||
maxBytes=10 * 1024 * 1024, # 10 MB
|
||||
backupCount=5,
|
||||
|
||||
@@ -9,6 +9,11 @@ Covers the three pieces added for the uncached-tail investigation:
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import stat
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.cache.compression_store import _payload_for_retrieval_log
|
||||
from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
|
||||
@@ -125,10 +130,26 @@ def test_payload_preview_disabled_omits_content(monkeypatch) -> None:
|
||||
assert event["payload_truncated"] is True
|
||||
|
||||
|
||||
def test_payload_preview_enabled_by_default(monkeypatch) -> None:
|
||||
def test_payload_preview_disabled_by_default(monkeypatch) -> None:
|
||||
"""Unset means off: the log gets byte counts, never the content."""
|
||||
monkeypatch.delenv("HEADROOM_LOG_PAYLOAD_PREVIEW", raising=False)
|
||||
event = _payload_for_retrieval_log("hello world")
|
||||
assert event["payload_preview"] == "hello world"
|
||||
assert event["payload_preview"] == ""
|
||||
assert event["payload_preview_chars"] == 0
|
||||
assert event["payload_chars"] == len("hello world")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["1", "true", "TRUE", "yes", "on"])
|
||||
def test_payload_preview_opt_in_values(monkeypatch, value: str) -> None:
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", value)
|
||||
assert _payload_for_retrieval_log("hello world")["payload_preview"] == "hello world"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "0", "off", "no", "maybe", " "])
|
||||
def test_payload_preview_stays_off_for_anything_else(monkeypatch, value: str) -> None:
|
||||
"""Only an explicit opt-in turns previews on — a typo must not."""
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", value)
|
||||
assert _payload_for_retrieval_log("hello world")["payload_preview"] == ""
|
||||
|
||||
|
||||
def test_append_context_skips_breakpointed_text_block() -> None:
|
||||
@@ -178,3 +199,76 @@ def test_count_cache_breakpoints_tolerates_malformed_shapes() -> None:
|
||||
empty = count_cache_breakpoints(None, None, None)
|
||||
assert empty["total"] == 0
|
||||
assert empty["message_count"] == 0
|
||||
|
||||
|
||||
# --- the runtime log file itself -------------------------------------------
|
||||
#
|
||||
# _payload_for_retrieval_log decides what goes into the record;
|
||||
# _setup_file_logging decides who can read the file it lands in. Both halves
|
||||
# of the default-off guarantee are checked against a real log on disk.
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _proxy_log(tmp_path, monkeypatch, port: int):
|
||||
"""Point the workspace at *tmp_path*, install the real proxy log handler."""
|
||||
from headroom.proxy.helpers import _PROXY_LOG_HANDLER_NAME, _setup_file_logging
|
||||
|
||||
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
|
||||
headroom_logger = logging.getLogger("headroom")
|
||||
before = list(headroom_logger.handlers)
|
||||
propagate = headroom_logger.propagate
|
||||
try:
|
||||
_setup_file_logging(port)
|
||||
[handler] = [h for h in headroom_logger.handlers if h.name == _PROXY_LOG_HANDLER_NAME]
|
||||
yield Path(handler.baseFilename)
|
||||
finally:
|
||||
for handler in list(headroom_logger.handlers):
|
||||
if handler not in before:
|
||||
headroom_logger.removeHandler(handler)
|
||||
handler.close()
|
||||
headroom_logger.propagate = propagate
|
||||
|
||||
|
||||
def test_runtime_log_holds_no_payload_text_at_default_settings(tmp_path, monkeypatch) -> None:
|
||||
"""A retrieval on default settings leaves byte counts in the log, not content."""
|
||||
from headroom.cache.compression_store import CompressionStore
|
||||
|
||||
monkeypatch.delenv("HEADROOM_LOG_PAYLOAD_PREVIEW", raising=False)
|
||||
secret = "BEGIN-CUSTOMER-DATA ssn=123-45-6789 def sekrit(): pass END-CUSTOMER-DATA"
|
||||
|
||||
with _proxy_log(tmp_path, monkeypatch, 18801) as log_path:
|
||||
store = CompressionStore(enable_feedback=False)
|
||||
assert store.retrieve(store.store(original=secret, compressed="[compressed]")) is not None
|
||||
logging.getLogger("headroom").handlers[-1].flush()
|
||||
text = log_path.read_text(encoding="utf-8")
|
||||
|
||||
assert "event=headroom_retrieve" in text, "the retrieval was not logged at all"
|
||||
assert secret not in text
|
||||
assert "123-45-6789" not in text
|
||||
assert f'"payload_chars":{len(secret)}' in text
|
||||
assert '"payload_preview":""' in text
|
||||
|
||||
|
||||
def test_runtime_log_is_owner_only_when_preview_enabled(tmp_path, monkeypatch) -> None:
|
||||
"""Opting in to previews hardens the log the previews land in."""
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
|
||||
with _proxy_log(tmp_path, monkeypatch, 18802) as log_path:
|
||||
assert log_path.exists()
|
||||
assert stat.S_IMODE(log_path.stat().st_mode) == 0o600
|
||||
|
||||
|
||||
def test_runtime_log_hardening_survives_a_pre_existing_world_readable_log(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
"""O_CREAT's mode does not apply to an existing file; the chmod must."""
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
|
||||
from headroom import paths as _paths
|
||||
|
||||
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
|
||||
stale = _paths.proxy_log_path(18803)
|
||||
stale.parent.mkdir(parents=True, exist_ok=True)
|
||||
stale.write_text("from an older, unhardened run\n", encoding="utf-8")
|
||||
stale.chmod(0o644)
|
||||
|
||||
with _proxy_log(tmp_path, monkeypatch, 18803) as log_path:
|
||||
assert stat.S_IMODE(log_path.stat().st_mode) == 0o600
|
||||
|
||||
@@ -60,7 +60,8 @@ def _capture_headroom_retrieve_events():
|
||||
logger.setLevel(previous_level)
|
||||
|
||||
|
||||
def test_retrieve_logs_payload_preview():
|
||||
def test_retrieve_logs_payload_preview(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
|
||||
store = CompressionStore(enable_feedback=False)
|
||||
hash_key = store.store(
|
||||
original="secret-ish payload for operator debugging",
|
||||
@@ -85,7 +86,8 @@ def test_retrieve_logs_payload_preview():
|
||||
assert events[0]["tool_name"] == "tool_a"
|
||||
|
||||
|
||||
def test_retrieve_log_redacts_secret_payload_values():
|
||||
def test_retrieve_log_redacts_secret_payload_values(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("HEADROOM_LOG_PAYLOAD_PREVIEW", "1")
|
||||
store = CompressionStore(enable_feedback=False)
|
||||
hash_key = store.store(
|
||||
original="OPENAI_API_KEY=sk-proj-secret1234567890 Authorization: Bearer token123456789",
|
||||
|
||||
Reference in New Issue
Block a user