mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(toin): preserve stores that exceed the byte limit (#4589)
## Description When TOIN skips an oversized JSON store at startup, the next save currently replaces it with partial in-memory data. Saves can also create a file that the next startup refuses to load. Enforce the existing byte limit on persistence and preserve the original file on rejection. Ref #4476, part (c) only. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Refuse to overwrite an existing file above the configured limit; reject oversized serialized output before atomic replacement and remove its temporary file - Raise `ValueError` for size rejection so the existing `TOIN.save()` handler reports `toin_save_failed` and keeps the dirty flag and last-save timestamp - Keep streamed compact JSON, pin LF on the existing writer, and document that `max_load_bytes` also guards saves - Cover exact boundaries, zero/disabled limits, new/existing files, cleanup, real TOIN save state, and the LF writer argument with a marked `windows_newline` test ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text # Before production edits: pytest tests/test_toin_retention.py -q 6 failed, 20 passed # Patched checkout; HEADROOM_OFFLINE=1 TIKTOKEN_CACHE_DIR=/private/tmp/headroom-tiktoken .venv/bin/python -m pytest tests/test_toin*.py tests/test_stateless_toin.py tests/test_telemetry.py tests/test_compression_policy_toin_gate.py tests/test_smart_crusher_toin_attachment.py -q 197 passed, 1 skipped in 1.34s # LF regression: removing newline="\n" from the writer makes the new test fail # AssertionError: assert [None] == ['\n']; restoring the pin passes .venv/bin/python -m pytest tests/test_install/test_state.py tests/test_usage_reporter_snapshot.py tests/test_learn/test_section_lifecycle.py tests/test_learn/test_writer.py tests/test_memory/test_writers.py tests/test_cli_learn.py tests/test_toin_retention.py tests/test_memory_sync.py tests/test_providers_opencode_install.py -m windows_newline --tb=short -q 13 passed, 245 deselected, 1 warning in 0.79s # Existing headroom.memory.writers deprecation warning during collection .venv/bin/python -m ruff check . All checks passed! .venv/bin/python -m ruff format --check . 1860 files already formatted git diff --check # exit 0 ``` ## Real Behavior Proof - Environment: macOS 26.4 arm64, Python 3.12.14, local filesystem, no provider/model or network calls. The native integration tests use the installed official Headroom 0.42 wheel's `_core.abi3.so` via an ignored checkout symlink; Python source is from this checkout - Exact command / steps: Ran the executable reproduction below from the checkout, with real TOIN instances writing actual files and small byte limits. Also ran the same two scenarios against Python source archived from baseline `fc1ba891` - Observed result: Baseline replaced the rejected three-pattern store with one new pattern and grew a valid 958-byte file to 1816 bytes despite a 1086-byte limit; both saves cleared dirty state. After the patch both original files remain byte-for-byte identical, dirty state and last-save timestamp remain unchanged, temporary files are removed, and `TOIN.save()` returns normally after logging `toin_save_failed` with `error_type=ValueError` - Not tested: Windows runtime, concurrent external writers, multi-gigabyte stores, full repository pytest/mypy, or a fresh Rust build. Cargo is unavailable, so `make ci-precheck` stops at `cargo fmt`; the optional Hypothesis property test is skipped <details> <summary>Executable disk reproduction and observed output</summary> ```sh PYTHONPATH=. HEADROOM_OFFLINE=1 TIKTOKEN_CACHE_DIR=/private/tmp/headroom-tiktoken .venv/bin/python - <<'PY' import json from pathlib import Path from tempfile import TemporaryDirectory from headroom.telemetry.models import ToolSignature from headroom.telemetry.toin import TOINConfig, ToolIntelligenceNetwork def record(toin, index): toin.record_compression( tool_signature=ToolSignature.from_items([{f"field_{index}": "x"}]), original_count=10, compressed_count=5, original_tokens=100, compressed_tokens=50, strategy="test", ) with TemporaryDirectory(prefix="toin-limit-") as directory: for count, case in [(3, "load_rejected"), (1, "store_grows")]: path = Path(directory) / f"{case}.json" seed = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=0)) for index in range(count): record(seed, index) seed.save() before = path.read_bytes() limit = len(before) - 1 if count == 3 else len(before) + 128 toin = ToolIntelligenceNetwork(TOINConfig( storage_path=str(path), auto_save_interval=0, max_storage_bytes=limit, )) record(toin, 99) last_save = toin._last_save_time toin.save() after = path.read_bytes() print(case, "preserved=", before == after, "dirty=", toin._dirty, "patterns_on_disk=", len(json.loads(after)["patterns"]), "temporary_files=", len(list(path.parent.glob(".toin_*.tmp")))) assert before == after and toin._dirty and toin._last_save_time == last_save assert not list(path.parent.glob(".toin_*.tmp")) PY ``` ```text TOIN storage save failed TOIN storage save failed load_rejected preserved= True dirty= True patterns_on_disk= 3 temporary_files= 0 store_grows preserved= True dirty= True patterns_on_disk= 1 temporary_files= 0 ``` </details> ## Runtime Rollout Safety - Rollout-managed feature(s): None; filesystem persistence guard - Minimum rollout channel: N/A; no rollout gate - Stable/default behavior changed: Filesystem saves above the existing configured limit are refused rather than replacing the store; below-limit saves are unchanged - Kill switch / disable path: Existing `HEADROOM_TOIN_BACKEND=none` disables persistence; direct backend users can still pass `max_load_bytes=None` - Unsafe override required: No - Qualification impact: No routing, compression policy, or provider behavior changes - Rollback path: Revert this patch ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Additional Notes AI assistance: Codex assisted with investigation, implementation, tests, and this PR description. The change was self-reviewed and independently reviewed against the code and actual disk behavior; no actionable findings remained. All configured commit hooks, including scoped mypy and Commitlint, passed. Full-project mypy was not run. The follow-up newline review finding is covered by a marked spy test that asserts the actual `open` keyword argument while writing a real file. Removing the LF pin fails this test on POSIX as well. The current Windows job is packaging-gated and selects only the learn/memory paths; its scope is unchanged. The new test runs in the normal Python suite and was explicitly included in the local marked-test run above. This adds no pruning, backup, or recovery policy. An oversized original remains on disk until explicitly recovered or reloaded under an adequate limit. Serialization still streams into the temporary file before its size is checked; this bounds the committed file, not temporary disk usage. Existing `OSError` handling and multi-writer semantics are unchanged. Documentation changes are limited to the backend docstring and configuration comment. Expected size-rejection warnings are covered by the regression tests. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4589" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> --------- Co-authored-by: JD Davis <mxjerrett@gmail.com>
This commit is contained in:
@@ -26,6 +26,8 @@ class FileSystemTOINBackend:
|
||||
|
||||
Args:
|
||||
path: Path to the JSON storage file.
|
||||
max_load_bytes: Maximum file size in bytes for loading and saving.
|
||||
None disables the limit.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str, *, max_load_bytes: int | None = None) -> None:
|
||||
@@ -48,7 +50,7 @@ class FileSystemTOINBackend:
|
||||
):
|
||||
logger.warning(
|
||||
"TOIN data file %s exceeds the configured load limit (%d bytes); "
|
||||
"ignoring it until the store is rebuilt",
|
||||
"leaving it unchanged",
|
||||
self._path,
|
||||
self._max_load_bytes,
|
||||
)
|
||||
@@ -70,9 +72,20 @@ class FileSystemTOINBackend:
|
||||
data: Serialized TOIN data to persist.
|
||||
|
||||
Raises:
|
||||
ValueError: The existing file or serialized data exceeds the size limit.
|
||||
OSError: If persistence fails, so the caller can retain pending changes.
|
||||
"""
|
||||
try:
|
||||
# A file skipped by load() must not be replaced with partial state.
|
||||
if (
|
||||
self._max_load_bytes is not None
|
||||
and self._path.exists()
|
||||
and self._path.stat().st_size > self._max_load_bytes
|
||||
):
|
||||
raise ValueError(
|
||||
f"Existing TOIN data file {self._path} exceeds the configured "
|
||||
f"size limit ({self._max_load_bytes} bytes); refusing to overwrite it"
|
||||
)
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fd, tmp_path = tempfile.mkstemp(dir=self._path.parent, prefix=".toin_", suffix=".tmp")
|
||||
@@ -80,8 +93,17 @@ class FileSystemTOINBackend:
|
||||
# Stream compact JSON directly to the temp file. Building a
|
||||
# second multi-gigabyte string was the primary autosave RSS
|
||||
# spike reported in #2886.
|
||||
with open(fd, "w") as f:
|
||||
with open(fd, "w", newline="\n") as f:
|
||||
json.dump(data, f, separators=(",", ":"))
|
||||
# Check actual bytes after flushing, before replacing a valid store.
|
||||
if (
|
||||
self._max_load_bytes is not None
|
||||
and Path(tmp_path).stat().st_size > self._max_load_bytes
|
||||
):
|
||||
raise ValueError(
|
||||
f"Serialized TOIN data exceeds the configured size limit "
|
||||
f"({self._max_load_bytes} bytes)"
|
||||
)
|
||||
Path(tmp_path).replace(self._path)
|
||||
except Exception:
|
||||
try:
|
||||
|
||||
@@ -489,7 +489,7 @@ class TOINConfig:
|
||||
storage_path: str = field(default_factory=get_default_toin_storage_path)
|
||||
auto_save_interval: int = 600 # Auto-save every 10 minutes
|
||||
max_patterns: int = DEFAULT_MAX_PATTERNS
|
||||
max_storage_bytes: int = DEFAULT_MAX_STORAGE_BYTES
|
||||
max_storage_bytes: int = DEFAULT_MAX_STORAGE_BYTES # Filesystem load and save limit
|
||||
|
||||
# Network learning thresholds
|
||||
min_samples_for_recommendation: int = 10
|
||||
|
||||
@@ -358,6 +358,24 @@ def test_filesystem_backend_writes_compact_json(tmp_path) -> None:
|
||||
assert " " not in path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.windows_newline
|
||||
def test_filesystem_backend_writes_pin_lf(monkeypatch, tmp_path) -> None:
|
||||
import headroom.telemetry.backends.filesystem as filesystem
|
||||
|
||||
calls: list[dict] = []
|
||||
real_open = open
|
||||
|
||||
def recording_open(fd, *args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return real_open(fd, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(filesystem, "open", recording_open, raising=False)
|
||||
FileSystemTOINBackend(str(tmp_path / "toin.json")).save({"patterns": {}})
|
||||
|
||||
# Artifact bytes alone cannot detect a missing newline pin on POSIX.
|
||||
assert [call.get("newline") for call in calls] == ["\n"]
|
||||
|
||||
|
||||
def test_filesystem_backend_skips_oversized_store(tmp_path) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
path.write_text("x" * 100, encoding="utf-8")
|
||||
@@ -365,3 +383,99 @@ def test_filesystem_backend_skips_oversized_store(tmp_path) -> None:
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=10)
|
||||
assert backend.load() == {}
|
||||
assert path.exists()
|
||||
|
||||
|
||||
def test_filesystem_backend_preserves_oversized_store_on_save(tmp_path) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
original = b'{"patterns":{"existing":{"value":1}}}'
|
||||
path.write_bytes(original)
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=len(original) - 1)
|
||||
|
||||
assert backend.load() == {}
|
||||
with pytest.raises(ValueError, match="limit"):
|
||||
backend.save({})
|
||||
|
||||
assert path.read_bytes() == original
|
||||
assert list(tmp_path.glob(".toin_*.tmp")) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("existing", [False, True], ids=["new_store", "existing_store"])
|
||||
def test_filesystem_backend_rejects_oversized_output(tmp_path, existing) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
original = b"{}"
|
||||
if existing:
|
||||
path.write_bytes(original)
|
||||
data = {"value": "x" * 100}
|
||||
limit = len(json.dumps(data, separators=(",", ":")).encode("utf-8")) - 1
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=limit)
|
||||
|
||||
with pytest.raises(ValueError, match="limit"):
|
||||
backend.save(data)
|
||||
|
||||
if existing:
|
||||
assert path.read_bytes() == original
|
||||
else:
|
||||
assert not path.exists()
|
||||
assert list(tmp_path.glob(".toin_*.tmp")) == []
|
||||
|
||||
|
||||
def test_filesystem_backend_accepts_exact_byte_limit(tmp_path) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
data = {"value": "雪"}
|
||||
serialized = json.dumps(data, separators=(",", ":")).encode("utf-8")
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=len(serialized))
|
||||
|
||||
backend.save(data)
|
||||
assert path.read_bytes() == serialized
|
||||
assert backend.load() == data
|
||||
# An existing file at the boundary can still be replaced.
|
||||
backend.save({})
|
||||
assert backend.load() == {}
|
||||
|
||||
|
||||
def test_filesystem_backend_zero_byte_limit_rejects_output(tmp_path) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=0)
|
||||
|
||||
with pytest.raises(ValueError, match="limit"):
|
||||
backend.save({})
|
||||
|
||||
assert not path.exists()
|
||||
assert list(tmp_path.glob(".toin_*.tmp")) == []
|
||||
|
||||
|
||||
def test_filesystem_backend_without_byte_limit_allows_growth(tmp_path) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
backend = FileSystemTOINBackend(str(path), max_load_bytes=None)
|
||||
backend.save({"value": "x" * 100})
|
||||
data = {"value": "x" * 200}
|
||||
|
||||
backend.save(data)
|
||||
|
||||
assert backend.load() == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("oversized_on_load", [True, False], ids=["load_rejected", "store_grows"])
|
||||
def test_toin_size_rejection_preserves_store_and_dirty_state(
|
||||
tmp_path, caplog, oversized_on_load
|
||||
) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
_saved_store(path, 3 if oversized_on_load else 1)
|
||||
original = path.read_bytes()
|
||||
limit = len(original) - 1 if oversized_on_load else len(original) + 128
|
||||
toin = ToolIntelligenceNetwork(
|
||||
TOINConfig(storage_path=str(path), max_storage_bytes=limit, auto_save_interval=0)
|
||||
)
|
||||
assert len(toin._patterns) == (0 if oversized_on_load else 1)
|
||||
_record(toin, 99)
|
||||
last_save_time = toin._last_save_time
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.telemetry.toin"):
|
||||
toin.save()
|
||||
|
||||
assert path.read_bytes() == original
|
||||
assert toin._dirty
|
||||
assert toin._last_save_time == last_save_time
|
||||
assert list(tmp_path.glob(".toin_*.tmp")) == []
|
||||
[warning] = [r for r in caplog.records if getattr(r, "event", None) == "toin_save_failed"]
|
||||
assert warning.error_type == "ValueError"
|
||||
|
||||
Reference in New Issue
Block a user