mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(toin): retain pending changes after filesystem save failures (#4568)
## Description
`FileSystemTOINBackend.save()` currently catches `OSError` and returns
normally. Its caller, `ToolIntelligenceNetwork.save()`, consequently
clears `_dirty` and advances `_last_save_time` even though nothing was
persisted. When storage becomes writable again, subsequent auto-saves
skip pending observations because the failed write was mistakenly
treated as successful.
This PR propagates the original filesystem exception after logging it.
The existing TOIN-level failure handler then preserves pending state
along with the previous save timestamp, emits `event=toin_save_failed`,
and allows subsequent automatic saves to retry.
## Type of Change
- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature
- [ ] Breaking change
- [ ] Documentation update
## Changes Made
- Propagate the backend's original `OSError` to the caller-level failure
handler.
- Document the filesystem backend's failure contract.
- Add regression coverage for directory creation failure, failed atomic
replacement, original-file preservation, temporary-file cleanup, and
recovery through TOIN's automatic save path.
## Testing
- [x] Related TOIN test suites: **86 passed, 1 skipped** (the 3 added
regressions fail on unchanged main).
- [x] Code quality: Ruff check and format pass cleanly across modified
files.
- [x] Pre-commit hooks pass, including commit-message linting.
- [x] Full pre-push suite (`make ci-precheck`): **1,671 Rust tests
passed, 3 ignored**; native extension rebuilt and imported cleanly;
**178 Python tests passed, 5 skipped**; commitlint passed.
```text
Before: 3 failed, 18 deselected
After: 86 passed, 1 skipped
```
## Real Behavior Proof
- Environment: Windows x86-64, Python 3.13.9, `main` at
`a2d4fe72381e186651466ee94e92ce9d8d1396cc`, local filesystem.
- Exact command / steps: Run the Python reproduction script below. A
regular file temporarily occupies the parent-directory path, causing a
real filesystem error. Removing that blocker and triggering an auto-save
check validates recovery without waiting for the default interval.
- Observed result: Before the fix, a failed save clears `_dirty` and
advances the save timestamp, causing the auto-save retry to create no
store. After the fix, pending state and timestamp survive the error;
once the blocker is removed, the store is created and a fresh TOIN
instance successfully loads the observation.
- Not tested: Disk-full hardware conditions, network filesystems,
non-filesystem TOIN backends, or live proxy/model traffic.
```python
import json
import tempfile
from pathlib import Path
from headroom.telemetry.models import ToolSignature
from headroom.telemetry.toin import TOINConfig, ToolIntelligenceNetwork
with tempfile.TemporaryDirectory() as directory:
parent = Path(directory) / "state"
path = parent / "toin.json"
signature = ToolSignature.from_items([{"job_id": "x"}])
toin = ToolIntelligenceNetwork(TOINConfig(
storage_path=str(path), auto_save_interval=3600,
))
toin.record_compression(
tool_signature=signature, original_count=10, compressed_count=5,
original_tokens=100, compressed_tokens=50, strategy="test",
)
toin._last_save_time = 0
parent.write_text("not a directory", encoding="utf-8", newline="\n")
toin.save()
print("after failure:", toin._dirty, toin._last_save_time)
parent.unlink()
toin._maybe_auto_save()
restored = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path)))
pattern = restored.get_pattern(signature.structure_hash)
print("recovered:", path.exists(),
pattern.total_compressions if pattern else 0)
```
```text
Before: after failure: False <new timestamp>; recovered: False 0
After: after failure: True 0; recovered: True 1
```
## Runtime Rollout Safety
- Rollout-managed feature(s): none.
- Minimum rollout channel: stable.
- Stable/default behavior changed: failed filesystem TOIN saves retain
pending state instead of being treated as successful. Successful saves
remain unchanged.
- Kill switch / disable path: `HEADROOM_TOIN_BACKEND=none` disables
filesystem persistence.
- Unsafe override required: no.
- Qualification impact: no schema or file-format changes; the existing
TOIN-level exception handler prevents save errors from escaping into
request flows.
- Rollback path: revert this patch; no migration.
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
## Additional Notes
I implemented and verified this with Codex assistance for drafting and
local tests.
<!-- devin-review-badge-begin -->
---
<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4568"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->
---------
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: JerrettDavis <2610199+JerrettDavis@users.noreply.github.com>
Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Copilot
JerrettDavis
JD Davis
parent
d9cee45268
commit
8b15a15799
@@ -68,6 +68,9 @@ class FileSystemTOINBackend:
|
||||
|
||||
Args:
|
||||
data: Serialized TOIN data to persist.
|
||||
|
||||
Raises:
|
||||
OSError: If persistence fails, so the caller can retain pending changes.
|
||||
"""
|
||||
try:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -89,3 +92,4 @@ class FileSystemTOINBackend:
|
||||
|
||||
except OSError as e:
|
||||
logger.warning("Failed to save TOIN data to %s: %s", self._path, e)
|
||||
raise
|
||||
|
||||
@@ -578,6 +578,7 @@ class ToolIntelligenceNetwork:
|
||||
|
||||
# Tracking
|
||||
self._last_save_time = time.time()
|
||||
self._last_auto_save_attempt = 0.0
|
||||
self._dirty = False
|
||||
|
||||
# Load existing data from backend
|
||||
@@ -1760,8 +1761,12 @@ class ToolIntelligenceNetwork:
|
||||
if not self._dirty:
|
||||
return
|
||||
|
||||
elapsed = time.time() - self._last_save_time
|
||||
now = time.time()
|
||||
elapsed = now - max(self._last_save_time, self._last_auto_save_attempt)
|
||||
if elapsed >= self._config.auto_save_interval:
|
||||
# Failed saves retain dirty state, but must not turn every
|
||||
# observation into synchronous disk I/O and another warning.
|
||||
self._last_auto_save_attempt = now
|
||||
# save() uses the same RLock, so this is safe
|
||||
self.save()
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -12,6 +13,125 @@ from headroom.telemetry.models import ToolSignature
|
||||
from headroom.telemetry.toin import TOINConfig, ToolIntelligenceNetwork
|
||||
|
||||
|
||||
def test_failed_auto_saves_respect_interval_and_retry_pending_data(tmp_path, caplog, monkeypatch):
|
||||
import headroom.telemetry.toin as toin_module
|
||||
|
||||
clock = [10000.0]
|
||||
monkeypatch.setattr(toin_module, "time", SimpleNamespace(time=lambda: clock[0]))
|
||||
parent = tmp_path / "state"
|
||||
parent.write_text("blocked", encoding="utf-8", newline="\n")
|
||||
path = parent / "toin.json"
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=60))
|
||||
clock[0] += 60
|
||||
with caplog.at_level(logging.WARNING):
|
||||
for _ in range(4):
|
||||
toin.record_compression(
|
||||
tool_signature=_signature(456),
|
||||
original_count=10,
|
||||
compressed_count=5,
|
||||
original_tokens=100,
|
||||
compressed_tokens=50,
|
||||
strategy="test",
|
||||
)
|
||||
failures = [
|
||||
record for record in caplog.records if getattr(record, "event", None) == "toin_save_failed"
|
||||
]
|
||||
assert len(failures) == 1
|
||||
assert toin._dirty is True
|
||||
|
||||
parent.unlink()
|
||||
clock[0] += 60
|
||||
toin._maybe_auto_save()
|
||||
restored = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path)))
|
||||
assert restored.get_pattern(_signature(456).structure_hash).total_compressions == 4
|
||||
assert toin._dirty is False
|
||||
|
||||
|
||||
def test_manual_save_can_retry_before_auto_save_interval(tmp_path):
|
||||
parent = tmp_path / "state"
|
||||
parent.write_text("blocked", encoding="utf-8", newline="\n")
|
||||
path = parent / "toin.json"
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=3600))
|
||||
toin.record_compression(
|
||||
tool_signature=_signature(789),
|
||||
original_count=10,
|
||||
compressed_count=5,
|
||||
original_tokens=100,
|
||||
compressed_tokens=50,
|
||||
strategy="test",
|
||||
)
|
||||
toin._last_save_time = 0
|
||||
toin._maybe_auto_save()
|
||||
assert toin._dirty is True
|
||||
parent.unlink()
|
||||
|
||||
toin.save()
|
||||
|
||||
assert path.is_file()
|
||||
assert toin._dirty is False
|
||||
|
||||
|
||||
def test_filesystem_save_reports_parent_directory_failure(tmp_path) -> None:
|
||||
blocked = tmp_path / "blocked"
|
||||
blocked.write_text("not a directory", encoding="utf-8", newline="\n")
|
||||
backend = FileSystemTOINBackend(str(blocked / "toin.json"))
|
||||
|
||||
with pytest.raises(OSError):
|
||||
backend.save({"patterns": {}})
|
||||
|
||||
|
||||
def test_failed_replace_preserves_store_and_cleans_temporary_file(tmp_path, monkeypatch) -> None:
|
||||
from pathlib import Path
|
||||
|
||||
path = tmp_path / "toin.json"
|
||||
backend = FileSystemTOINBackend(str(path))
|
||||
backend.save({"patterns": {"original": {}}})
|
||||
original = path.read_bytes()
|
||||
|
||||
def fail_replace(self, target):
|
||||
raise OSError("replace unavailable")
|
||||
|
||||
monkeypatch.setattr(Path, "replace", fail_replace)
|
||||
with pytest.raises(OSError, match="replace unavailable"):
|
||||
backend.save({"patterns": {"new": {}}})
|
||||
|
||||
assert path.read_bytes() == original
|
||||
assert not list(tmp_path.glob(".toin_*.tmp"))
|
||||
|
||||
|
||||
def test_filesystem_failure_keeps_toin_dirty_for_automatic_retry(tmp_path, caplog) -> None:
|
||||
parent = tmp_path / "state"
|
||||
path = parent / "toin.json"
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=3600))
|
||||
toin.record_compression(
|
||||
tool_signature=_signature(123),
|
||||
original_count=10,
|
||||
compressed_count=5,
|
||||
original_tokens=100,
|
||||
compressed_tokens=50,
|
||||
strategy="test",
|
||||
)
|
||||
toin._last_save_time = 0
|
||||
parent.write_text("temporarily blocked", encoding="utf-8", newline="\n")
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
toin.save()
|
||||
|
||||
assert toin._dirty is True
|
||||
assert toin._last_save_time == 0
|
||||
assert any(getattr(record, "event", None) == "toin_save_failed" for record in caplog.records)
|
||||
|
||||
parent.unlink()
|
||||
toin._maybe_auto_save()
|
||||
|
||||
assert toin._dirty is False
|
||||
assert toin._last_save_time > 0
|
||||
restored = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path)))
|
||||
pattern = restored.get_pattern(_signature(123).structure_hash)
|
||||
assert pattern is not None
|
||||
assert pattern.total_compressions == 1
|
||||
|
||||
|
||||
def _signature(index: int) -> ToolSignature:
|
||||
return ToolSignature.from_items([{f"field_{index}": "x"}])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user