fix(proxy): keep exception messages out of handler logs at every level

Addresses the maintainer review on this PR:

- log_hook_failure and log_request_failure log describe_exception (types and code
  locations), never the message or traceback, at warning, error and debug alike.
  Upstream httpx transport text is kept, with URLs passed through redact_url and capped.
  Context values go through safe_id.
- The hook once-guard is WarnOnce: one warning per hook class, stage and exception type,
  never consumed while WARNING is disabled, bounded with one overflow warning.
- The token-count, cache-observation, tool-desc and prefix-dump lines this PR added use
  describe_exception instead of exc_info. So does the Anthropic "Optimization failed"
  warning, which a failing hook stage now reaches.
- Tests render every record with logging.Formatter and assert that a content and a
  credential canary never appear, and that a compute_biases failure still warns after
  an earlier pre_compress failure of the same type.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assisted-by: claude-code:claude-fable-5
Assisted-by: claude-code:claude-fable-5-1
Assisted-by: claude-code:claude-opus-5
Assisted-by: claude-code:claude-opus-5-5
Headline-Session: 05a613a2-4a69-4818-9ae6-2e0e33a4d9f2
Headline-Session: 56d69125-fc5b-4ba0-a637-9be6fe80d456
Headline-Session: 6e77a807-8dd4-4a77-a2fa-ddce6dc2515e
Headline-Session: e7d23c15-6d13-45b7-8cff-6775e20e195b
Headline-AI-Share: 79% strict / 81% weighted
This commit is contained in:
Tejas Chopra
2026-10-09 22:16:47 -07:00
co-authored by Claude Opus 5.5
parent be14ace04c
commit 1531ac59b0
5 changed files with 192 additions and 170 deletions
+36 -38
View File
@@ -2,27 +2,30 @@
Exceptions on this path can carry request content: a compression hook sees the
messages, and a catch-all handler catches whatever a compressor or router raised.
So warning and error lines name the failure (request id, provider, model, hook,
stage, exception type) without the exception text or traceback; those go to a
separate debug line, which an operator turns on explicitly. Upstream transport
errors keep their text: httpx writes it, and it is what diagnoses an outage.
So no line, at any level, carries an exception message or a traceback. A failure
is named by the request id, provider, model, hook, stage and
``describe_exception`` (exception types and code locations). Upstream transport
errors keep their text, which httpx writes and which diagnoses an outage, with
any URL in it redacted.
"""
from __future__ import annotations
import logging
import threading
import re
import httpx
from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
logger = logging.getLogger("headroom.proxy")
# Hook warnings: one per (hook class, stage, exception type), so a hook failing
# on every request warns once. Bounded: cleared when full, after which a hook
# that is still failing warns again.
_MAX_WARNED = 256
_WARNED: set[tuple[str, str, str]] = set()
_WARNED_LOCK = threading.Lock()
# One warning per (hook class, stage, exception type), so a hook failing on
# every request warns once and a different hook or stage still warns.
_HOOK_WARNINGS = WarnOnce(256, "failing compression hooks")
_URL_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://[^\s'\"<>]+")
_TRANSPORT_TEXT_MAX = 200
def _hook_name(hooks: object) -> str:
@@ -42,37 +45,32 @@ def log_hook_failure(
``outcome`` says what the caller does next, e.g. that it skips compression.
"""
hook = _hook_name(hooks)
error_type = type(err).__name__
key = (hook, stage, error_type)
warn = False
# Only consume the once-slot when the warning will actually be emitted.
if logger.isEnabledFor(logging.WARNING):
with _WARNED_LOCK:
if key not in _WARNED:
if len(_WARNED) >= _MAX_WARNED:
_WARNED.clear()
_WARNED.add(key)
warn = True
if warn:
logger.warning(
"[%s] hook %s %s failed; %s: %s "
"(repeats log at debug; enable debug logging for the error text)",
request_id,
hook,
stage,
outcome,
error_type,
)
logger.debug("[%s] hook %s %s failed: %s", request_id, hook, stage, error_type, exc_info=err)
first = _HOOK_WARNINGS.first((hook, stage, type(err).__name__), logger)
logger.log(
logging.WARNING if first else logging.DEBUG,
"[%s] hook %s %s failed; %s: %s%s",
request_id,
hook,
stage,
outcome,
describe_exception(err),
" (repeats log at debug)" if first else "",
)
def _transport_text(err: httpx.TransportError) -> str:
text = _URL_RE.sub(lambda m: redact_url(m.group(0)), str(err))
return text[:_TRANSPORT_TEXT_MAX]
def log_request_failure(request_id: str, what: str, err: BaseException, **context: object) -> None:
"""Log a request-path failure at error, with its detail at debug.
"""Log a request-path failure at error.
``what`` names the operation (``"Request"``, ``"OpenAI request"``); ``context``
adds ``key=value`` fields such as provider and model.
"""
fields = "".join(f"{key}={value} " for key, value in context.items())
detail = f": {err}" if isinstance(err, httpx.TransportError) else ""
logger.error("[%s] %s failed: %s%s%s", request_id, what, fields, type(err).__name__, detail)
logger.debug("[%s] %s failure detail", request_id, what, exc_info=err)
fields = "".join(f"{key}={safe_id(value)} " for key, value in context.items())
detail = f" ({_transport_text(err)})" if isinstance(err, httpx.TransportError) else ""
logger.error(
"[%s] %s failed: %s%s%s", request_id, what, fields, describe_exception(err), detail
)
+13 -12
View File
@@ -34,6 +34,7 @@ from headroom.agent_savings import proxy_pipeline_kwargs
from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary
from headroom.ccr.marker_resolution import resolve_markers_in_response
from headroom.copilot_auth import apply_copilot_api_auth, is_copilot_upstream_url
from headroom.log_safety import describe_exception
from headroom.pipeline import PipelineStage, summarize_routing_markers
from headroom.proxy import public_errors
from headroom.proxy.anthropic_wire import (
@@ -339,16 +340,14 @@ def _dump_prefix_mismatch(request_id: str, current: list, previous: list) -> Non
default=str,
)
except (OSError, TypeError, ValueError, RecursionError) as e: # debug aid; never fail the turn
# An OSError names only the local dump path; other errors could quote
# message content, so their text stays at debug.
# describe_exception keeps an OSError's errno and strerror; other errors
# could quote message content, so no message is logged.
logger.warning(
"[%s] prefix_mismatch_debug dump to %s failed: %s%s",
"[%s] prefix_mismatch_debug dump to %s failed: %s",
request_id,
os.environ.get("HEADROOM_DEBUG_PREFIX_MISMATCH"),
type(e).__name__,
f": {e}" if isinstance(e, OSError) else "",
describe_exception(e),
)
logger.debug("[%s] prefix_mismatch_debug dump failure detail", request_id, exc_info=True)
class AnthropicHandlerMixin:
@@ -2371,9 +2370,11 @@ class AnthropicHandlerMixin:
if result and result.waste_signals:
waste_signals_dict = result.waste_signals.to_dict()
except Exception as e:
# Include type so TimeoutError vs other failures is distinguishable
# in bug reports — str(asyncio.TimeoutError()) is empty otherwise.
logger.warning(f"[{request_id}] Optimization failed: {type(e).__name__}: {e}")
# Types and code locations only: a compressor or hook error can
# quote the messages it was given.
logger.warning(
"[%s] Optimization failed: %s", request_id, describe_exception(e)
)
# Flag compression failure for observability
_compression_failed = True
# Split timeout from other errors: a timeout means the
@@ -3117,11 +3118,11 @@ class AnthropicHandlerMixin:
def _count_tool_tokens(value: object) -> int:
try:
return tokenizer.count_text(json.dumps(value, default=str))
except Exception:
except Exception as e:
logger.debug(
"[%s] tool token count failed; tool savings not counted",
"[%s] tool token count failed; tool savings not counted: %s",
request_id,
exc_info=True,
describe_exception(e),
)
return 0
+21 -21
View File
@@ -25,6 +25,7 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any
from urllib.parse import quote, unquote, urlparse
from headroom.log_safety import describe_exception
from headroom.proxy.conversation_savings import savings_conversation_key
from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure
from headroom.proxy.helpers import (
@@ -3118,11 +3119,11 @@ class OpenAIHandlerMixin:
- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),
)
_add_timing("compression_tool_schema_token_count", tool_token_started)
except Exception:
except Exception as e:
logger.debug(
"[%s] tool schema compaction token count failed; savings not counted",
"[%s] tool schema compaction token count failed; savings not counted: %s",
request_id,
exc_info=True,
describe_exception(e),
)
if debug_enabled:
_log_codex_compression_debug(
@@ -3162,11 +3163,11 @@ class OpenAIHandlerMixin:
tokenizer.count_text(_json_debug_dumps(payload.get("tools")))
- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),
)
except Exception:
except Exception as e:
logger.debug(
"[%s] tool desc compaction token count failed; savings not counted",
"[%s] tool desc compaction token count failed; savings not counted: %s",
request_id,
exc_info=True,
describe_exception(e),
)
if debug_enabled:
_log_codex_compression_debug(
@@ -3180,11 +3181,10 @@ class OpenAIHandlerMixin:
tools_bytes_saved=desc_before - desc_after,
)
except Exception as e:
# Tool schemas are client content: the error text stays at debug.
# Tool schemas are client content: log the failure, never its message.
logger.warning(
"[%s] tool desc compaction step failed: %s", request_id, type(e).__name__
"[%s] tool desc compaction step failed: %s", request_id, describe_exception(e)
)
logger.debug("[%s] tool desc compaction failure detail", request_id, exc_info=True)
# Server-side Tool Search deferral (OpenAI Responses, gpt-5.4+): mark
# non-core function/MCP tools defer_loading + inject {"type": "tool_search"}
@@ -3306,11 +3306,11 @@ class OpenAIHandlerMixin:
)
if _mt_after < _mt_before:
tokens_saved += _mt_before - _mt_after
except Exception:
except Exception as e:
logger.debug(
"[%s] turn-hook message fold token count failed; savings not counted",
"[%s] turn-hook message fold token count failed; savings not counted: %s",
request_id,
exc_info=True,
describe_exception(e),
)
modified = True
transforms.append("openai:responses:turn_hook")
@@ -3361,11 +3361,11 @@ class OpenAIHandlerMixin:
"compression_tool_schema_attempted_token_count",
attempted_token_started,
)
except Exception:
except Exception as e:
logger.debug(
"[%s] tool schema attempted-token count failed",
"[%s] tool schema attempted-token count failed: %s",
request_id,
exc_info=True,
describe_exception(e),
)
dedupe_started = time.perf_counter()
@@ -5283,11 +5283,11 @@ class OpenAIHandlerMixin:
current_forwarded_messages=optimized_messages,
),
)
except Exception:
except Exception as e:
logger.debug(
"[%s] cache observation (openai chat, backend) failed",
"[%s] cache observation (openai chat, backend) failed: %s",
request_id,
exc_info=True,
describe_exception(e),
)
openai_prefix_tracker.update_from_response(
@@ -5787,11 +5787,11 @@ class OpenAIHandlerMixin:
current_forwarded_messages=optimized_messages,
),
)
except Exception:
except Exception as e:
logger.debug(
"[%s] cache observation (openai chat, direct) failed",
"[%s] cache observation (openai chat, direct) failed: %s",
request_id,
exc_info=True,
describe_exception(e),
)
openai_prefix_tracker.update_from_response(
+121 -98
View File
@@ -1,4 +1,4 @@
"""Request-path failure logging: visible at warning/error, content only at debug."""
"""Request-path failure logging: named at warning/error, no payload at any level."""
from __future__ import annotations
@@ -8,10 +8,12 @@ import httpx
import pytest
from headroom.hooks import CompressionHooks
from headroom.log_safety import WarnOnce
from headroom.proxy.handlers import _failure_logging
from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure
SECRET = "SECRET-USER-CONTENT"
CONTENT = "CANARY-USER-CONTENT"
CREDENTIAL = "sk-CANARY-CREDENTIAL"
class _Capture(logging.Handler):
@@ -25,10 +27,23 @@ class _Capture(logging.Handler):
def at(self, level: int) -> list[logging.LogRecord]:
return [r for r in self.records if r.levelno == level]
def hook_warnings(self) -> list[str]:
return [r.getMessage() for r in self.at(logging.WARNING) if "] hook " in r.getMessage()]
def assert_no_canaries(self) -> None:
"""Every record, rendered the way a log file renders it (traceback included)."""
formatter = logging.Formatter()
assert self.records
for record in self.records:
text = formatter.format(record)
assert CONTENT not in text
assert CREDENTIAL not in text
@pytest.fixture
def capture(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(_failure_logging, "_WARNED", set())
monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
monkeypatch.setattr(_failure_logging, "_HOOK_WARNINGS", WarnOnce(256, "failing hooks"))
log = logging.getLogger("headroom.proxy")
handler = _Capture()
old_level = log.level
@@ -49,20 +64,25 @@ class _HooksB(CompressionHooks):
pass
def test_hook_failure_warns_once_without_the_error_text(capture) -> None:
for request_id in ("req-1", "req-2"):
log_hook_failure(request_id, "compute_biases", _HooksA(), ValueError(SECRET))
def _raised(exc: BaseException) -> BaseException:
"""``exc`` with a traceback, as a caught exception has."""
try:
raise exc
except BaseException as caught: # noqa: BLE001 - test helper
return caught
(warning,) = capture.at(logging.WARNING)
assert "[req-1] hook" in warning.getMessage()
assert "_HooksA compute_biases failed" in warning.getMessage()
assert "ValueError" in warning.getMessage()
assert SECRET not in warning.getMessage()
assert warning.exc_info is None
# The detail, text and traceback included, is at debug for both failures.
debug = capture.at(logging.DEBUG)
assert len(debug) == 2
assert all(r.exc_info and SECRET in str(r.exc_info[1]) for r in debug)
def test_hook_failure_warns_once_and_no_record_carries_the_payload(capture) -> None:
for request_id in ("req-1", "req-2"):
err = _raised(ValueError(f"bad message {CONTENT} key={CREDENTIAL}"))
log_hook_failure(request_id, "compute_biases", _HooksA(), err)
(warning,) = capture.hook_warnings()
assert warning.startswith("[req-1] hook ")
assert "_HooksA compute_biases failed; continuing without it: ValueError at " in warning
(repeat,) = capture.at(logging.DEBUG)
assert repeat.getMessage().startswith("[req-2] hook ")
capture.assert_no_canaries()
def test_hook_identity_stage_and_error_type_each_get_their_own_warning(capture) -> None:
@@ -72,107 +92,116 @@ def test_hook_identity_stage_and_error_type_each_get_their_own_warning(capture)
log_hook_failure("r4", "pre_compress", _HooksA(), KeyError("x")) # other type
log_hook_failure("r5", "pre_compress", _HooksA(), ValueError("y")) # repeat
warned = [r.getMessage().split()[0] for r in capture.at(logging.WARNING)]
assert warned == ["[r1]", "[r2]", "[r3]", "[r4]"]
assert [w.split()[0] for w in capture.hook_warnings()] == ["[r1]", "[r2]", "[r3]", "[r4]"]
def test_a_suppressed_warning_does_not_use_up_the_once_slot(capture) -> None:
def test_a_suppressed_warning_is_still_emitted_later(capture) -> None:
log = logging.getLogger("headroom.proxy")
log.setLevel(logging.ERROR)
log_hook_failure("r1", "post_compress", _HooksA(), ValueError("x"))
log.setLevel(logging.DEBUG)
log.setLevel(logging.INFO)
log_hook_failure("r2", "post_compress", _HooksA(), ValueError("x"))
assert [r.getMessage().split()[0] for r in capture.at(logging.WARNING)] == ["[r2]"]
assert [w.split()[0] for w in capture.hook_warnings()] == ["[r2]"]
def test_the_once_set_is_bounded(capture, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(_failure_logging, "_MAX_WARNED", 2)
for stage in ("a", "b", "c"):
log_hook_failure("r", stage, _HooksA(), ValueError("x"))
assert len(_failure_logging._WARNED) <= 2
def test_request_failure_names_the_failure_without_payload(capture) -> None:
log_request_failure(
"r1", "Request", _raised(ValueError(CONTENT)), provider="anthropic", model="m"
)
transport = httpx.ConnectError(
f"cannot reach https://user:{CREDENTIAL}@upstream.example/v1?key={CREDENTIAL}"
)
log_request_failure("r2", "Request", transport, model="m")
first, second = (r.getMessage() for r in capture.at(logging.ERROR))
assert first.startswith("[r1] Request failed: provider='anthropic' model='m' ValueError at ")
assert second == (
"[r2] Request failed: model='m' ConnectError"
" (cannot reach https://upstream.example/v1?<redacted>)"
)
capture.assert_no_canaries()
def test_request_failure_keeps_transport_text_and_hides_other_text(capture) -> None:
log_request_failure("r1", "Request", ValueError(SECRET), provider="anthropic", model="m")
log_request_failure("r2", "Request", httpx.ConnectError("Connection refused"), model="m")
first, second = capture.at(logging.ERROR)
assert first.getMessage() == "[r1] Request failed: provider=anthropic model=m ValueError"
assert second.getMessage() == "[r2] Request failed: model=m ConnectError: Connection refused"
assert first.exc_info is None and second.exc_info is None
assert any(r.exc_info and SECRET in str(r.exc_info[1]) for r in capture.at(logging.DEBUG))
def test_openai_chat_names_the_failing_hook_stage(capture) -> None:
"""Through the real handler: the stage that raised is the one logged."""
fastapi = pytest.importorskip("fastapi") # noqa: F841
from fastapi.testclient import TestClient
def _proxy_app(hooks: CompressionHooks, fake_retry, *, optimize: bool): # noqa: ANN001, ANN202
pytest.importorskip("fastapi")
from headroom.proxy.server import ProxyConfig, create_app
class BrokenBiases(CompressionHooks):
app = create_app(
ProxyConfig(optimize=optimize, cache_enabled=False, rate_limit_enabled=False, hooks=hooks)
)
return app
def _chat_response() -> httpx.Response:
return httpx.Response(
200,
json={
"id": "c",
"object": "chat.completion",
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
},
)
def test_openai_chat_warns_for_each_failing_stage(capture) -> None:
"""Through the real handler: after a pre_compress failure, a later compute_biases
failure of the same type still warns, under its own stage."""
from fastapi.testclient import TestClient
class Flaky(CompressionHooks):
failing = "pre_compress"
def pre_compress(self, messages, ctx): # noqa: ANN001, ANN201
if self.failing == "pre_compress":
raise ValueError(CONTENT)
return messages
def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201
raise ValueError(SECRET)
raise ValueError(CONTENT)
async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202
return httpx.Response(
200,
json={
"id": "c",
"object": "chat.completion",
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
},
)
return _chat_response()
config = ProxyConfig(
optimize=False, cache_enabled=False, rate_limit_enabled=False, hooks=BrokenBiases()
)
app = create_app(config)
with TestClient(app) as client:
hooks = Flaky()
with TestClient(_proxy_app(hooks, fake_retry, optimize=False)) as client:
# Proxy startup may reset logger levels; re-assert capture afterwards.
logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)
client.app.state.proxy._retry_request = fake_retry
resp = client.post(
"/v1/chat/completions",
json={"model": "gpt-4o", "messages": [{"role": "user", "content": SECRET}]},
headers={"Authorization": "Bearer test-key"},
)
for failing in ("pre_compress", "compute_biases"):
hooks.failing = failing
resp = client.post(
"/v1/chat/completions",
json={"model": "gpt-4o", "messages": [{"role": "user", "content": CONTENT}]},
headers={"Authorization": f"Bearer {CREDENTIAL}"},
)
assert resp.status_code == 200, resp.text
assert resp.status_code == 200, resp.text
hook_warnings = [r for r in capture.at(logging.WARNING) if "hook" in r.getMessage()]
assert len(hook_warnings) == 1
assert "BrokenBiases compute_biases failed" in hook_warnings[0].getMessage()
assert "pre_compress" not in hook_warnings[0].getMessage()
assert all(
SECRET not in r.getMessage() for r in capture.records if r.levelno >= logging.WARNING
)
warnings = capture.hook_warnings()
assert len(warnings) == 2
assert "Flaky pre_compress failed" in warnings[0]
assert "Flaky compute_biases failed" in warnings[1]
capture.assert_no_canaries()
def test_anthropic_bias_failure_says_compression_is_skipped(capture) -> None:
"""Through the real handler: Anthropic aborts optimization when a bias hook fails,
so the warning must say compression is skipped, not that it continues."""
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
from headroom.proxy.server import ProxyConfig, create_app
class BrokenBiases(CompressionHooks):
def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201
raise ValueError(SECRET)
sent: list[dict] = []
raise ValueError(CONTENT)
async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202
sent.append(body)
return httpx.Response(
200,
json={
@@ -186,12 +215,7 @@ def test_anthropic_bias_failure_says_compression_is_skipped(capture) -> None:
},
)
app = create_app(
ProxyConfig(
optimize=True, cache_enabled=False, rate_limit_enabled=False, hooks=BrokenBiases()
)
)
with TestClient(app) as client:
with TestClient(_proxy_app(BrokenBiases(), fake_retry, optimize=True)) as client:
logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)
client.app.state.proxy._retry_request = fake_retry
resp = client.post(
@@ -199,14 +223,13 @@ def test_anthropic_bias_failure_says_compression_is_skipped(capture) -> None:
json={
"model": "claude-sonnet-4-5",
"max_tokens": 16,
"messages": [{"role": "user", "content": SECRET}],
"messages": [{"role": "user", "content": CONTENT}],
},
headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
headers={"x-api-key": CREDENTIAL, "anthropic-version": "2023-06-01"},
)
assert resp.status_code == 200, resp.text
(warning,) = [r for r in capture.at(logging.WARNING) if "hook" in r.getMessage()]
message = warning.getMessage()
assert "BrokenBiases compute_biases failed; skipping compression for this turn" in message
assert "continuing without it" not in message
assert SECRET not in message
(warning,) = capture.hook_warnings()
assert "BrokenBiases compute_biases failed; skipping compression for this turn" in warning
assert "continuing without it" not in warning
capture.assert_no_canaries()
@@ -142,7 +142,7 @@ def test_optimization_failure_logs_exception_type() -> None:
assert response.status_code == 200, response.text
warning_msgs = [
call.args[0]
str(call.args[0]) % call.args[1:]
for call in mock_warning.call_args_list
if call.args and "Optimization failed" in str(call.args[0])
]