mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(savings): key the ledger append warning by path and re-arm it on success
The once-guard was one process-wide bool. After any failed append, a later failure on a different ledger path, or after the ledger had recovered, logged at DEBUG only. Now: - the first failure per ledger path is a WARNING; repeats on that path go to DEBUG; - a successful append re-arms its path, so a new failure episode warns again; - the set of warned paths is capped at 64 and cleared when full; - a key is marked only when WARNING is enabled. Above DEBUG the append and read warnings now log the error text only for OSError, which names a local path; for anything else they log the exception type. The traceback stays at DEBUG. Three new tests fail on the previous commit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1c3de00c44
commit
3e0054b934
+31
-15
@@ -71,9 +71,33 @@ except ImportError:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The first failed append per process is a WARNING so an unwritable ledger is
|
||||
# visible; later ones drop to DEBUG so a persistent failure cannot flood logs.
|
||||
_append_failure_warned = False
|
||||
# The first failed append to each ledger path is a WARNING so an unwritable
|
||||
# ledger is visible; repeats drop to DEBUG so a persistent failure cannot flood
|
||||
# logs. A successful append re-arms the path. Capped: cleared when full.
|
||||
_APPEND_WARNED_PATHS_MAX = 64
|
||||
_append_warned_paths: set[str] = set()
|
||||
|
||||
|
||||
def _describe_error(exc: BaseException) -> str:
|
||||
# An OSError names a local path; any other exception text may echo event
|
||||
# data, so only its type is logged above DEBUG.
|
||||
return str(exc) if isinstance(exc, OSError) else type(exc).__name__
|
||||
|
||||
|
||||
def _warn_append_failure(target: Path, exc: BaseException) -> None:
|
||||
key = str(target)
|
||||
if key not in _append_warned_paths and logger.isEnabledFor(logging.WARNING):
|
||||
if len(_append_warned_paths) >= _APPEND_WARNED_PATHS_MAX:
|
||||
_append_warned_paths.clear()
|
||||
_append_warned_paths.add(key)
|
||||
logger.warning(
|
||||
"savings ledger: append to %s failed: %s; savings will not be recorded. "
|
||||
"Make the directory writable or set HEADROOM_SAVINGS_EVENTS_PATH.",
|
||||
target,
|
||||
_describe_error(exc),
|
||||
)
|
||||
logger.debug("savings ledger: append to %s failed", target, exc_info=True)
|
||||
|
||||
|
||||
SCHEMA_VERSION = 2
|
||||
UNKNOWN = "unknown"
|
||||
@@ -422,19 +446,10 @@ def record_savings_event(
|
||||
if _HAS_FCNTL and fcntl is not None:
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
except Exception as exc:
|
||||
global _append_failure_warned
|
||||
if not _append_failure_warned:
|
||||
_append_failure_warned = True
|
||||
logger.warning(
|
||||
"savings ledger: append to %s failed: %s; savings will not be recorded. "
|
||||
"Make the directory writable or set HEADROOM_SAVINGS_EVENTS_PATH.",
|
||||
target,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
logger.debug("savings ledger: append to %s failed", target, exc_info=True)
|
||||
_warn_append_failure(target, exc)
|
||||
return False
|
||||
|
||||
_append_warned_paths.discard(str(target))
|
||||
_maybe_compact(target)
|
||||
return True
|
||||
|
||||
@@ -479,8 +494,9 @@ def _read_events(
|
||||
"savings ledger: could not read %s: %s. Check that it is a readable file, "
|
||||
"or set HEADROOM_SAVINGS_EVENTS_PATH.",
|
||||
target,
|
||||
exc,
|
||||
_describe_error(exc),
|
||||
)
|
||||
logger.debug("savings ledger: reading %s failed", target, exc_info=True)
|
||||
return []
|
||||
return events
|
||||
|
||||
|
||||
@@ -581,15 +581,57 @@ def test_unreadable_ledger_warns_instead_of_reporting_zero_silently(tmp_path, le
|
||||
assert str(tmp_path) in warnings[0]
|
||||
|
||||
|
||||
def test_first_failed_append_warns_once_per_process(monkeypatch, tmp_path, ledger_records):
|
||||
monkeypatch.setattr(L, "_append_failure_warned", False, raising=False)
|
||||
def test_failed_append_warns_once_per_ledger_path(monkeypatch, tmp_path, ledger_records):
|
||||
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
|
||||
first = tmp_path / "a"
|
||||
second = tmp_path / "b"
|
||||
first.mkdir()
|
||||
second.mkdir()
|
||||
|
||||
# A directory where the ledger file should be: every append fails.
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=tmp_path)
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=tmp_path)
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=first)
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=second)
|
||||
|
||||
warnings = _warnings(ledger_records)
|
||||
assert len(warnings) == 1 # one WARNING; the second failure goes to DEBUG
|
||||
assert "append to" in warnings[0]
|
||||
assert str(tmp_path) in warnings[0]
|
||||
# One WARNING per path; the repeat on `first` goes to DEBUG only.
|
||||
assert len(warnings) == 2
|
||||
assert str(first) in warnings[0] and "append to" in warnings[0]
|
||||
assert str(second) in warnings[1]
|
||||
assert any(r.levelno == logging.DEBUG and "append to" in r.getMessage() for r in ledger_records)
|
||||
|
||||
|
||||
def test_successful_append_rearms_the_warning(monkeypatch, tmp_path, ledger_records):
|
||||
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
|
||||
ledger = tmp_path / "savings_events.jsonl"
|
||||
ledger.mkdir()
|
||||
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
|
||||
ledger.rmdir()
|
||||
assert L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
|
||||
ledger.unlink()
|
||||
ledger.mkdir()
|
||||
assert not L.record_savings_event(tokens_before=1000, tokens_after=400, path=ledger)
|
||||
|
||||
# The failure after a recovery is a new episode and warns again.
|
||||
assert len(_warnings(ledger_records)) == 2
|
||||
|
||||
|
||||
def test_append_warning_names_only_the_type_of_a_non_os_error(
|
||||
monkeypatch, tmp_path, ledger_records
|
||||
):
|
||||
monkeypatch.setattr(L, "_append_warned_paths", set(), raising=False)
|
||||
|
||||
def fail(*_args, **_kwargs):
|
||||
raise TypeError("secret-looking event payload")
|
||||
|
||||
# Shadow open() inside the module only, so nothing else is affected.
|
||||
monkeypatch.setattr(L, "open", fail, raising=False)
|
||||
assert not L.record_savings_event(
|
||||
tokens_before=1000, tokens_after=400, path=tmp_path / "savings_events.jsonl"
|
||||
)
|
||||
|
||||
warnings = _warnings(ledger_records)
|
||||
assert len(warnings) == 1
|
||||
assert "TypeError" in warnings[0]
|
||||
assert "secret-looking" not in warnings[0]
|
||||
|
||||
Reference in New Issue
Block a user