fix(savings): key the ledger append warning by path and re-arm it on success

The once-guard was one process-wide bool. After any failed append, a later failure on a
different ledger path, or after the ledger had recovered, logged at DEBUG only. Now:
- the first failure per ledger path is a WARNING; repeats on that path go to DEBUG;
- a successful append re-arms its path, so a new failure episode warns again;
- the set of warned paths is capped at 64 and cleared when full;
- a key is marked only when WARNING is enabled.

Above DEBUG the append and read warnings now log the error text only for OSError, which
names a local path; for anything else they log the exception type. The traceback stays at
DEBUG. Three new tests fail on the previous commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tejas Chopra
2026-10-09 18:27:54 -07:00
co-authored by Claude Opus 5.5
parent 1c3de00c44
commit 3e0054b934
2 changed files with 80 additions and 22 deletions
+31 -15
View File
@@ -71,9 +71,33 @@ except ImportError:
logger = logging.getLogger(__name__)
# The first failed append per process is a WARNING so an unwritable ledger is
# visible; later ones drop to DEBUG so a persistent failure cannot flood logs.
_append_failure_warned = False
# The first failed append to each ledger path is a WARNING so an unwritable
# ledger is visible; repeats drop to DEBUG so a persistent failure cannot flood
# logs. A successful append re-arms the path. Capped: cleared when full.
_APPEND_WARNED_PATHS_MAX = 64
_append_warned_paths: set[str] = set()
def _describe_error(exc: BaseException) -> str:
# An OSError names a local path; any other exception text may echo event
# data, so only its type is logged above DEBUG.
return str(exc) if isinstance(exc, OSError) else type(exc).__name__
def _warn_append_failure(target: Path, exc: BaseException) -> None:
key = str(target)
if key not in _append_warned_paths and logger.isEnabledFor(logging.WARNING):
if len(_append_warned_paths) >= _APPEND_WARNED_PATHS_MAX:
_append_warned_paths.clear()
_append_warned_paths.add(key)
logger.warning(
"savings ledger: append to %s failed: %s; savings will not be recorded. "
"Make the directory writable or set HEADROOM_SAVINGS_EVENTS_PATH.",
target,
_describe_error(exc),
)
logger.debug("savings ledger: append to %s failed", target, exc_info=True)
SCHEMA_VERSION = 2
UNKNOWN = "unknown"
@@ -422,19 +446,10 @@ def record_savings_event(
if _HAS_FCNTL and fcntl is not None:
fcntl.flock(handle, fcntl.LOCK_UN)
except Exception as exc:
global _append_failure_warned
if not _append_failure_warned:
_append_failure_warned = True
logger.warning(
"savings ledger: append to %s failed: %s; savings will not be recorded. "
"Make the directory writable or set HEADROOM_SAVINGS_EVENTS_PATH.",
target,
exc,
)
else:
logger.debug("savings ledger: append to %s failed", target, exc_info=True)
_warn_append_failure(target, exc)
return False
_append_warned_paths.discard(str(target))
_maybe_compact(target)
return True
@@ -479,8 +494,9 @@ def _read_events(
"savings ledger: could not read %s: %s. Check that it is a readable file, "
"or set HEADROOM_SAVINGS_EVENTS_PATH.",
target,
exc,
_describe_error(exc),
)
logger.debug("savings ledger: reading %s failed", target, exc_info=True)
return []
return events
+49 -7
View File
@@ -581,15 +581,57 @@ def test_unreadable_ledger_warns_instead_of_reporting_zero_silently(tmp_path, le
assert str(tmp_path) in warnings[0]
def test_first_failed_append_warns_once_per_process(monkeypatch, tmp_path, ledger_records):
monkeypatch.setattr(L, "_append_failure_warned", False, raising=False)
def test_failed_append_warns_once_per_ledger_path(monkeypatch, tmp_path, ledger_records):
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
first = tmp_path / "a"
second = tmp_path / "b"
first.mkdir()
second.mkdir()
# A directory where the ledger file should be: every append fails.
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=tmp_path)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=tmp_path)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=second)
warnings = _warnings(ledger_records)
assert len(warnings) == 1 # one WARNING; the second failure goes to DEBUG
assert "append to" in warnings[0]
assert str(tmp_path) in warnings[0]
# One WARNING per path; the repeat on `first` goes to DEBUG only.
assert len(warnings) == 2
assert str(first) in warnings[0] and "append to" in warnings[0]
assert str(second) in warnings[1]
assert any(r.levelno == logging.DEBUG and "append to" in r.getMessage() for r in ledger_records)
def test_successful_append_rearms_the_warning(monkeypatch, tmp_path, ledger_records):
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
ledger = tmp_path / "savings_events.jsonl"
ledger.mkdir()
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
ledger.rmdir()
assert L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
ledger.unlink()
ledger.mkdir()
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
# The failure after a recovery is a new episode and warns again.
assert len(_warnings(ledger_records)) == 2
def test_append_warning_names_only_the_type_of_a_non_os_error(
monkeypatch, tmp_path, ledger_records
):
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
def fail(*_args, **_kwargs):
raise TypeError("secret-looking event payload")
# Shadow open() inside the module only, so nothing else is affected.
monkeypatch.setattr(L, "open", fail, raising=False)
assert not L.record_savings_event(
tokens_before=1000, tokens_after=400, path=tmp_path / "savings_events.jsonl"
)
warnings = _warnings(ledger_records)
assert len(warnings) == 1
assert "TypeError" in warnings[0]
assert "secret-looking" not in warnings[0]