mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(transforms): log failed transforms with tracebacks (#3981)
## Description Closes #3974 When transform eligibility or application fails, the compression pipeline now logs the transform name, supplied request correlation, exception type, and stack frame locations before propagating the original exception. This gives operators diagnostics while retaining proxy fail-open behavior. New pipeline logs omit exception messages, notes, cause/context chains, locals, and source lines to avoid adding request content to logs. Both failure paths count toward the existing circuit breaker. ## Validation Rendered-log regressions cover both phases, sensitive exception chains/notes, a literal sensitive source line, correlation, and circuit-breaker behavior. A real TCP uvicorn proxy smoke with a deliberately failing custom transform returned HTTP 200 and forwarded original messages unchanged to a separate real loopback HTTP upstream. ### Test Output ```text 49 passed, 2 warnings in 9.23s Ruff lint and format: pass Whitespace: pass Linux-target mypy: Success: no issues found in 566 source files REAL TCP: HTTP 200; original messages reached loopback upstream unchanged after transform failure. ``` Verified head: `f4cb6e2dc14635f63d29f262a00d53789dbcdc59`. Hosted CI and final review remain pending. Python 3.10 compatibility follow-up: exception notes are tested only when available; the 17 safety-rail tests pass after this adjustment. --------- Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
@@ -30,6 +31,25 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _log_transform_failure(transform_name: str, log_prefix: str, exc: Exception) -> None:
|
||||
"""Keep stack diagnostics without logging payload-bearing exception text."""
|
||||
# Log frame locations only: source lines can embed literal payloads too.
|
||||
# Exception messages, notes, cause/context chains, and locals stay out.
|
||||
frames = "".join(
|
||||
f' File "{frame.filename}", line {frame.lineno}, in {frame.name}\n'
|
||||
for frame in traceback.extract_tb(exc.__traceback__)
|
||||
)
|
||||
logger.error(
|
||||
"%sTransform %s failed; aborting compression pipeline\n"
|
||||
"Traceback (most recent call last):\n%s%s: exception message redacted",
|
||||
log_prefix,
|
||||
transform_name,
|
||||
frames,
|
||||
type(exc).__name__,
|
||||
)
|
||||
|
||||
|
||||
# Waste-signal detection re-parses the *original* messages for telemetry only
|
||||
# (it never changes the compression result). On very large transcripts that
|
||||
# extra parse can take tens of seconds and blow the Anthropic compression
|
||||
@@ -346,7 +366,13 @@ class TransformPipeline:
|
||||
|
||||
for transform in self.transforms:
|
||||
# Check if transform should run
|
||||
if not transform.should_apply(current_messages, tokenizer, **kwargs):
|
||||
try:
|
||||
eligible = transform.should_apply(current_messages, tokenizer, **kwargs)
|
||||
except Exception as exc:
|
||||
_log_transform_failure(transform.name, log_prefix, exc)
|
||||
self._breaker_record_failure()
|
||||
raise
|
||||
if not eligible:
|
||||
continue
|
||||
|
||||
transform_span_context = (
|
||||
@@ -367,7 +393,8 @@ class TransformPipeline:
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
result = transform.apply(current_messages, tokenizer, **kwargs)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
_log_transform_failure(transform.name, log_prefix, exc)
|
||||
self._breaker_record_failure()
|
||||
raise
|
||||
duration_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
@@ -18,6 +18,7 @@ Three rails, each of which only ever makes compression LESS aggressive:
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
@@ -339,3 +340,71 @@ class TestLibraryInflationGuard:
|
||||
assert result.transforms_applied == ["inflation_guard:reverted"]
|
||||
assert result.tokens_saved == 0
|
||||
assert result.compression_ratio == 0.0
|
||||
|
||||
|
||||
def test_transform_failure_logs_name_and_traceback(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
class _Tokenizer:
|
||||
def count_messages(self, messages: list[dict[str, Any]]) -> int:
|
||||
return len(messages)
|
||||
|
||||
pipeline = TransformPipeline(HeadroomConfig(), transforms=[_FailingTransform()])
|
||||
monkeypatch.setattr(pipeline, "_get_tokenizer", lambda model: _Tokenizer())
|
||||
caplog.set_level(logging.ERROR, logger="headroom.transforms.pipeline")
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024)
|
||||
|
||||
failure_records = [
|
||||
record
|
||||
for record in caplog.records
|
||||
if record.name == "headroom.transforms.pipeline" and "always_fails" in record.getMessage()
|
||||
]
|
||||
assert len(failure_records) == 1
|
||||
rendered = logging.Formatter().format(failure_records[0])
|
||||
assert "Traceback (most recent call last)" in rendered
|
||||
assert "RuntimeError: exception message redacted" in rendered
|
||||
assert "test_compression_safety_rails.py" in rendered
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["should_apply", "apply"])
|
||||
def test_transform_failure_rendered_log_redacts_payload_and_correlates_request(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, phase: str
|
||||
) -> None:
|
||||
secret = "private-request-payload-3974"
|
||||
|
||||
class _Tokenizer:
|
||||
def count_messages(self, messages: list[dict[str, Any]]) -> int:
|
||||
return len(messages)
|
||||
|
||||
class _SensitiveFailure(_FailingTransform):
|
||||
def should_apply(self, *args: Any, **kwargs: Any) -> bool:
|
||||
if phase == "should_apply":
|
||||
raise ValueError("private-request-payload-3974")
|
||||
return True
|
||||
|
||||
def apply(self, *args: Any, **kwargs: Any) -> TransformResult:
|
||||
try:
|
||||
raise ValueError(secret)
|
||||
except ValueError as cause:
|
||||
error = RuntimeError(secret)
|
||||
if hasattr(error, "add_note"):
|
||||
error.add_note(secret)
|
||||
raise error from cause
|
||||
|
||||
monkeypatch.setenv("HEADROOM_PIPELINE_BREAKER_THRESHOLD", "1")
|
||||
pipeline = TransformPipeline(HeadroomConfig(), transforms=[_SensitiveFailure()])
|
||||
monkeypatch.setattr(pipeline, "_get_tokenizer", lambda model: _Tokenizer())
|
||||
caplog.set_level(logging.ERROR, logger="headroom.transforms.pipeline")
|
||||
with pytest.raises((ValueError, RuntimeError), match=secret):
|
||||
pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024, request_id="request-3974")
|
||||
rendered = "\n".join(logging.Formatter().format(record) for record in caplog.records)
|
||||
assert "request-3974" in rendered
|
||||
assert "always_fails" in rendered
|
||||
assert "Traceback (most recent call last)" in rendered
|
||||
assert "test_compression_safety_rails.py" in rendered
|
||||
assert secret not in rendered
|
||||
# Eligibility and application failures both open the existing breaker.
|
||||
result = pipeline.apply(_MESSAGES, model="gpt-4o", model_limit=1024)
|
||||
assert result.messages == _MESSAGES
|
||||
|
||||
Reference in New Issue
Block a user