fix(transforms): log failed transforms with tracebacks (#3981)

## Description

Closes #3974

When transform eligibility or application fails, the compression
pipeline now logs the transform name, supplied request correlation,
exception type, and stack frame locations before propagating the
original exception. This gives operators diagnostics while retaining
proxy fail-open behavior. New pipeline logs omit exception messages,
notes, cause/context chains, locals, and source lines to avoid adding
request content to logs. Both failure paths count toward the existing
circuit breaker.

## Validation

Rendered-log regressions cover both phases, sensitive exception
chains/notes, a literal sensitive source line, correlation, and
circuit-breaker behavior. A real TCP uvicorn proxy smoke with a
deliberately failing custom transform returned HTTP 200 and forwarded
original messages unchanged to a separate real loopback HTTP upstream.

### Test Output

```text
49 passed, 2 warnings in 9.23s
Ruff lint and format: pass
Whitespace: pass
Linux-target mypy: Success: no issues found in 566 source files
REAL TCP: HTTP 200; original messages reached loopback upstream unchanged after transform failure.
```

Verified head: `f4cb6e2dc14635f63d29f262a00d53789dbcdc59`. Hosted CI and
final review remain pending.

Python 3.10 compatibility follow-up: exception notes are tested only
when available; the 17 safety-rail tests pass after this adjustment.

---------

Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
lorenzozane
2026-10-08 10:32:10 -05:00
committed by GitHub
co-authored by JD Davis
parent 159daa25a0
commit a65455804c
2 changed files with 98 additions and 2 deletions
+29 -2
View File
@@ -6,6 +6,7 @@ import logging
import os
import threading
import time
import traceback
from collections.abc import Callable
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, TypeVar
@@ -30,6 +31,25 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _log_transform_failure(transform_name: str, log_prefix: str, exc: Exception) -> None:
"""Keep stack diagnostics without logging payload-bearing exception text."""
# Log frame locations only: source lines can embed literal payloads too.
# Exception messages, notes, cause/context chains, and locals stay out.
frames = "".join(
f' File "{frame.filename}", line {frame.lineno}, in {frame.name}\n'
for frame in traceback.extract_tb(exc.__traceback__)
)
logger.error(
"%sTransform %s failed; aborting compression pipeline\n"
"Traceback (most recent call last):\n%s%s: exception message redacted",
log_prefix,
transform_name,
frames,
type(exc).__name__,
)
# Waste-signal detection re-parses the *original* messages for telemetry only
# (it never changes the compression result). On very large transcripts that
# extra parse can take tens of seconds and blow the Anthropic compression
@@ -346,7 +366,13 @@ class TransformPipeline:
for transform in self.transforms:
# Check if transform should run
if not transform.should_apply(current_messages, tokenizer, **kwargs):
try:
eligible = transform.should_apply(current_messages, tokenizer, **kwargs)
except Exception as exc:
_log_transform_failure(transform.name, log_prefix, exc)
self._breaker_record_failure()
raise
if not eligible:
continue
transform_span_context = (
@@ -367,7 +393,8 @@ class TransformPipeline:
t0 = time.perf_counter()
try:
result = transform.apply(current_messages, tokenizer, **kwargs)
except Exception:
except Exception as exc:
_log_transform_failure(transform.name, log_prefix, exc)
self._breaker_record_failure()
raise
duration_ms = (time.perf_counter() - t0) * 1000
+69
View File
@@ -18,6 +18,7 @@ Three rails, each of which only ever makes compression LESS aggressive:
from __future__ import annotations
import importlib
import logging
import time
from typing import Any
@@ -339,3 +340,71 @@ class TestLibraryInflationGuard:
assert result.transforms_applied == ["inflation_guard:reverted"]
assert result.tokens_saved == 0
assert result.compression_ratio == 0.0
def test_transform_failure_logs_name_and_traceback(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
class _Tokenizer:
def count_messages(self, messages: list[dict[str, Any]]) -> int:
return len(messages)
pipeline = TransformPipeline(HeadroomConfig(), transforms=[_FailingTransform()])
monkeypatch.setattr(pipeline, "_get_tokenizer", lambda model: _Tokenizer())
caplog.set_level(logging.ERROR, logger="headroom.transforms.pipeline")
with pytest.raises(RuntimeError, match="boom"):
pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024)
failure_records = [
record
for record in caplog.records
if record.name == "headroom.transforms.pipeline" and "always_fails" in record.getMessage()
]
assert len(failure_records) == 1
rendered = logging.Formatter().format(failure_records[0])
assert "Traceback (most recent call last)" in rendered
assert "RuntimeError: exception message redacted" in rendered
assert "test_compression_safety_rails.py" in rendered
@pytest.mark.parametrize("phase", ["should_apply", "apply"])
def test_transform_failure_rendered_log_redacts_payload_and_correlates_request(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, phase: str
) -> None:
secret = "private-request-payload-3974"
class _Tokenizer:
def count_messages(self, messages: list[dict[str, Any]]) -> int:
return len(messages)
class _SensitiveFailure(_FailingTransform):
def should_apply(self, *args: Any, **kwargs: Any) -> bool:
if phase == "should_apply":
raise ValueError("private-request-payload-3974")
return True
def apply(self, *args: Any, **kwargs: Any) -> TransformResult:
try:
raise ValueError(secret)
except ValueError as cause:
error = RuntimeError(secret)
if hasattr(error, "add_note"):
error.add_note(secret)
raise error from cause
monkeypatch.setenv("HEADROOM_PIPELINE_BREAKER_THRESHOLD", "1")
pipeline = TransformPipeline(HeadroomConfig(), transforms=[_SensitiveFailure()])
monkeypatch.setattr(pipeline, "_get_tokenizer", lambda model: _Tokenizer())
caplog.set_level(logging.ERROR, logger="headroom.transforms.pipeline")
with pytest.raises((ValueError, RuntimeError), match=secret):
pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024, request_id="request-3974")
rendered = "\n".join(logging.Formatter().format(record) for record in caplog.records)
assert "request-3974" in rendered
assert "always_fails" in rendered
assert "Traceback (most recent call last)" in rendered
assert "test_compression_safety_rails.py" in rendered
assert secret not in rendered
# Eligibility and application failures both open the existing breaker.
result = pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024)
assert result.messages == _MESSAGES