mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-23 02:49:17 +08:00
## Description Credentials reached proxy logs and on-disk debug dumps: - The inbound request log line printed the full query string, so `?key=<api key>` (Google routes) and similar parameters were logged in clear. The streaming upstream-error lines and three passthrough failure lines printed the raw path with its query. - `should_redact_key` did not match `apikey` or headers such as `Ocp-Apim-Subscription-Key`, `X-Functions-Key`, `x-goog-api-key`. - The Anthropic and OpenAI 400 debug dumps kept a prefix of auth header values and were written with the default umask (world-readable on most systems). Closes # ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - `headroom/proxy/query_log_policy.py`: `redact_query_string_for_log` keeps parameter names (percent-encoded, so a `%0A` in a name cannot start a new log line) and drops every value; used by the inbound request line in `server.py`. - `query_log_policy.py` / `server.py`: `AccessLogQueryFilter` on `uvicorn.access`, installed by `create_app` in every process, replaces query values in uvicorn's access log (previously only LiteLLM's filter, when present, masked them). - `handlers/streaming.py`, `handlers/anthropic.py`, `handlers/openai.py`: error and passthrough log lines use the existing `_path_for_log`. - `wire_debug_redaction_policy.py`: `apikey` and any `*_key` / `*-key` name are secret. - `handlers/_debug_dump.py`: `redact_dump_headers` masks secret headers fully, including names configured in `*_extra_headers`; `write_owner_only_json` writes dumps with dir 0700 / file 0600, and the shared dump writer now uses it too. - `tests/test_credential_log_redaction.py`: query values never appear in the request/error log lines; dumped headers are fully masked; dump files are 0600 in a 0700 directory. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text $ pytest -q tests/test_credential_log_redaction.py 16 passed # same file against origin/main 9 failed, 7 passed $ pytest -q tests/test_proxy tests/test_log_safety.py tests/test_image_log_redaction.py tests/test_credential_log_redaction.py 693 passed ruff check / format --check: All checks passed mypy (changed modules): no new errors (same 8 pre-existing errors as origin/main, in prefix_tracker.py and ccr/mcp_server.py) ``` ## Real Behavior Proof - Environment: macOS, Python 3.12, the real `headroom proxy` CLI (`--stateless --no-telemetry`, isolated `HOME`), origin/main and this branch on separate ports. - Exact command / steps: `curl "http://127.0.0.1:<port>/stats?key=AIzaFAKEKEY0123456789&alt=sse"` and `curl "http://127.0.0.1:<port>/stats?x%0AINJECTED%20fake%20log%20line=1"`; grep `proxy_inbound_request` in the proxy's stderr. - Observed result: - origin/main: `... path=/stats query=key=AIzaFAKEKEY0123456789&alt=sse ...` - this branch: `... path=/stats query=key=<redacted>&alt=<redacted> ...` and `... query=x%0AINJECTED%20fake%20log%20line=<redacted> ...` (one line). - uvicorn access log: real uvicorn at `info` and `debug`, LiteLLM's filter removed, `POST ...?key=<canary>`: without this change the access line contains the canary; with it the line shows `key=<redacted>`. `test_uvicorn_access_log_drops_query_values` runs real uvicorn the same way. - 400 dumps: covered by `test_upstream_error_dump_masks_credentials_and_is_owner_only` (real handler, upstream response stubbed); not run against a live provider. ## Runtime Rollout Safety - Rollout-managed feature(s): none - Minimum rollout channel: n/a - Stable/default behavior changed: log text only (query values shown as `<redacted>`, more header names masked); 400 dump files are now owner-only. - Kill switch / disable path: none needed. - Unsafe override required: no - Qualification impact: none - Rollback path: revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
230 lines
8.1 KiB
Python
230 lines
8.1 KiB
Python
"""Credentials stay out of the proxy log and the upstream-error dumps."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import stat
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
pytest.importorskip("fastapi")
|
|
|
|
from fastapi.testclient import TestClient # noqa: E402
|
|
|
|
from headroom.proxy.models import ProxyConfig # noqa: E402
|
|
from headroom.proxy.server import create_app # noqa: E402
|
|
from headroom.proxy.wire_debug_redaction_policy import should_redact_key # noqa: E402
|
|
|
|
FAKE_KEY = "AIzaSyFAKEKEY0123456789"
|
|
|
|
|
|
def _config() -> ProxyConfig:
|
|
return ProxyConfig(
|
|
optimize=False,
|
|
cache_enabled=False,
|
|
rate_limit_enabled=False,
|
|
cost_tracking_enabled=False,
|
|
log_requests=False,
|
|
ccr_inject_tool=False,
|
|
ccr_handle_responses=False,
|
|
ccr_context_tracking=False,
|
|
image_optimize=False,
|
|
)
|
|
|
|
|
|
def test_inbound_request_log_drops_query_values_and_key_headers(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
with caplog.at_level(logging.INFO), TestClient(create_app(_config())) as client:
|
|
client.get(
|
|
f"/stats?key={FAKE_KEY}&alt=sse",
|
|
headers={"Ocp-Apim-Subscription-Key": "FAKE-OCP", "apikey": "FAKE-APIKEY"},
|
|
)
|
|
lines = [r.getMessage() for r in caplog.records if "proxy_inbound_request" in r.getMessage()]
|
|
assert lines
|
|
for line in lines:
|
|
assert FAKE_KEY not in line
|
|
assert "FAKE-OCP" not in line
|
|
assert "FAKE-APIKEY" not in line
|
|
assert any("query=key=<redacted>&alt=<redacted>" in line for line in lines)
|
|
|
|
|
|
def test_streaming_upstream_error_log_drops_query(caplog: pytest.LogCaptureFixture) -> None:
|
|
def upstream(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(400, json={"error": {"message": "bad"}})
|
|
|
|
body = {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}
|
|
with caplog.at_level(logging.WARNING), TestClient(create_app(_config())) as client:
|
|
client.app.state.proxy.http_client = httpx.AsyncClient(
|
|
transport=httpx.MockTransport(upstream)
|
|
)
|
|
client.post(
|
|
f"/v1beta/models/gemini-2.5-flash:streamGenerateContent?key={FAKE_KEY}&alt=sse",
|
|
json=body,
|
|
)
|
|
messages = [r.getMessage() for r in caplog.records]
|
|
assert any("Upstream provider returned an error" in m for m in messages), messages
|
|
assert not any(FAKE_KEY in m for m in messages)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name",
|
|
["Ocp-Apim-Subscription-Key", "X-Functions-Key", "apikey", "x-goog-api-key", "x-api-key"],
|
|
)
|
|
def test_wire_debug_redacts_key_headers(name: str) -> None:
|
|
assert should_redact_key(name)
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["max_tokens", "input_tokens", "keyboard", "monkey", "key_id"])
|
|
def test_wire_debug_key_rule_does_not_overreach(name: str) -> None:
|
|
assert not should_redact_key(name)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "headers"),
|
|
[
|
|
(
|
|
"/v1/messages",
|
|
{"x-api-key": "sk-ant-FAKESECRET-0123456789", "anthropic-version": "2023-06-01"},
|
|
),
|
|
("/v1/chat/completions", {"authorization": "Bearer sk-FAKESECRET-0123456789"}),
|
|
],
|
|
)
|
|
def test_upstream_error_dump_masks_credentials_and_is_owner_only(
|
|
tmp_path, monkeypatch: pytest.MonkeyPatch, path: str, headers: dict[str, str]
|
|
) -> None:
|
|
from headroom import paths
|
|
|
|
dump_dir = tmp_path / "debug_400"
|
|
monkeypatch.setattr(paths, "debug_400_dir", lambda: dump_dir)
|
|
monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "1")
|
|
error = httpx.Response(
|
|
400,
|
|
content=json.dumps({"error": {"type": "invalid_request_error", "message": "bad"}}).encode(),
|
|
headers={"content-type": "application/json"},
|
|
)
|
|
body = {"model": "m", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
|
|
with patch("headroom.proxy.server.AnyLLMBackend"):
|
|
with TestClient(create_app(_config())) as client:
|
|
client.app.state.proxy._retry_request = AsyncMock(return_value=error)
|
|
client.post(
|
|
path, headers={**headers, "Ocp-Apim-Subscription-Key": "FAKE-OCP"}, json=body
|
|
)
|
|
|
|
(dump,) = dump_dir.glob("*.json")
|
|
text = dump.read_text(encoding="utf-8")
|
|
assert "FAKESECRET" not in text
|
|
assert "sk-ant-FAKE" not in text and "Bearer sk-F" not in text
|
|
assert "FAKE-OCP" not in text
|
|
if os.name != "nt": # POSIX permission bits
|
|
assert stat.S_IMODE(dump.stat().st_mode) == 0o600
|
|
|
|
|
|
def test_query_names_cannot_split_the_log_line() -> None:
|
|
from headroom.proxy.query_log_policy import redact_query_string_for_log
|
|
|
|
rendered = redact_query_string_for_log("a%0D%0Aforged%20line=1&key=secret")
|
|
|
|
assert "\n" not in rendered and "\r" not in rendered
|
|
assert "secret" not in rendered
|
|
assert rendered == "a%0D%0Aforged%20line=<redacted>&key=<redacted>"
|
|
|
|
|
|
def test_uvicorn_access_log_drops_query_values() -> None:
|
|
"""Real uvicorn at info level, with no filter but headroom's own on the logger."""
|
|
import asyncio
|
|
import socket
|
|
|
|
import uvicorn
|
|
|
|
app = create_app(_config())
|
|
access = logging.getLogger("uvicorn.access")
|
|
others = [f for f in access.filters if not type(f).__module__.startswith("headroom.")]
|
|
for f in others: # e.g. LiteLLM's, which is only present when litellm is imported
|
|
access.removeFilter(f)
|
|
lines: list[str] = []
|
|
|
|
class _Lines(logging.Handler):
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
lines.append(record.getMessage())
|
|
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
server = uvicorn.Server(
|
|
uvicorn.Config(app, host="127.0.0.1", port=port, log_level="info", lifespan="off")
|
|
)
|
|
|
|
async def run() -> None:
|
|
task = asyncio.create_task(server.serve())
|
|
|
|
async def started() -> None:
|
|
while not server.started:
|
|
if task.done(): # e.g. the port could not be bound
|
|
pytest.fail(f"uvicorn stopped before starting: {task.exception()!r}")
|
|
await asyncio.sleep(0.02)
|
|
|
|
await asyncio.wait_for(started(), timeout=10)
|
|
handler = _Lines()
|
|
access.addHandler(handler)
|
|
try:
|
|
async with httpx.AsyncClient(trust_env=False) as client:
|
|
await client.get(f"http://127.0.0.1:{port}/livez?key={FAKE_KEY}&alt=sse")
|
|
finally:
|
|
access.removeHandler(handler)
|
|
server.should_exit = True
|
|
await task
|
|
|
|
try:
|
|
asyncio.run(run())
|
|
finally:
|
|
for f in others:
|
|
access.addFilter(f)
|
|
|
|
assert any("/livez?" in line for line in lines)
|
|
assert not any(FAKE_KEY in line for line in lines)
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits")
|
|
def test_owner_only_write_tightens_an_existing_directory_and_file(tmp_path) -> None:
|
|
from headroom.proxy.handlers._debug_dump import write_owner_only_json
|
|
|
|
dump_dir = tmp_path / "debug_400"
|
|
dump_dir.mkdir(mode=0o755)
|
|
os.chmod(dump_dir, 0o755)
|
|
dump = dump_dir / "dump.json"
|
|
dump.write_text("{}", encoding="utf-8")
|
|
os.chmod(dump, 0o644)
|
|
|
|
write_owner_only_json(dump, {"headers": {"authorization": "<redacted>"}})
|
|
|
|
assert stat.S_IMODE(dump_dir.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE(dump.stat().st_mode) == 0o600
|
|
assert json.loads(dump.read_text(encoding="utf-8"))["headers"]["authorization"] == "<redacted>"
|
|
|
|
|
|
@pytest.mark.skipif(not os.path.isdir("/dev/fd"), reason="needs /dev/fd to count descriptors")
|
|
def test_owner_only_write_closes_and_keeps_the_file_when_chmod_fails(
|
|
tmp_path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from headroom.proxy.handlers import _debug_dump
|
|
|
|
dump = tmp_path / "dump.json"
|
|
dump.write_text('{"previous": true}', encoding="utf-8")
|
|
|
|
def refuse(fd: int, mode: int) -> None: # e.g. a file owned by another user
|
|
raise PermissionError(1, "Operation not permitted")
|
|
|
|
monkeypatch.setattr(_debug_dump.os, "fchmod", refuse, raising=False)
|
|
open_before = len(os.listdir("/dev/fd"))
|
|
|
|
with pytest.raises(PermissionError):
|
|
_debug_dump.write_owner_only_json(dump, {"x": 1})
|
|
|
|
assert len(os.listdir("/dev/fd")) == open_before
|
|
assert dump.read_text(encoding="utf-8") == '{"previous": true}'
|