Files
headroom/tests/test_memory_sync_full_store.py
Yuchuan HeandJerrettDavis a6a825c55d fix(memory): include older memories in agent sync (#4832)
## Description

The memory sync engine builds its deduplication and export sets from
only the latest 500 user memories. Older rows are omitted from agent
files, and an existing older fact can be imported again as a duplicate.

Use the user's full snapshot for synchronization, including the refetch
after an import and the standalone import/export entry points. Carry the
existing MemoryFilter limit=None semantics through LocalBackend and
HierarchicalMemory; ordinary queries keep their default limit of 100.

Refs #4362, item 9 only.

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- Request the full user-scoped snapshot at each sync entry point.
- Document and type the existing unbounded query option without changing
default query limits.
- Cover a 501-row store, older-fact deduplication, import-triggered
refetch, forced/ordinary sync, idempotence and user isolation.

## Testing

- [x] Relevant unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)
- [x] Formatting passes (`ruff format --check .`)
- [x] Type checking passes (original full-package mypy hook)
- [x] Regression tests added
- [x] Standalone runtime validation completed

### Test Output

```text
Final regressions against original production code: 5 failed, 2 passed
pytest tests/test_memory_sync_full_store.py tests/test_memory_sync.py tests/test_memory/test_core_operations.py tests/test_memory/test_clear_scope_consistency.py -q:
  57 passed, 48 skipped
Original pre-commit and commit-msg hooks: passed
Whole-tree Ruff check and format --check: passed
Original pre-push make ci-precheck: passed
  Rust workspace: 1749 passed, 3 ignored
  Native extension: rebuilt, installed and import-verified
  Critical Python: 177 passed, 8 skipped
  commitlint: passed
```

The owning-suite skips require Hugging Face model files unavailable
offline. Existing datetime/optional-model deprecation warnings remain
visible in the logs.

## Real Behavior Proof

- Environment: Windows 11, CPython 3.13.9, existing dev venv; clean
production baseline `9f2cc017863f540c61f8747b3c744a97776ebaf2`, patch
`64e537fa2459d3540522bf934a32ee2e8e089ea7`. Explicit checkout on
PYTHONPATH; no paid/provider endpoint. The loopback service returns
controlled vectors, not trained-model inference.
- Exact command / steps: Save the script below as `proof.py`. From each
checkout run `python proof.py sync <fresh-output-directory> before` on
main, or `python proof.py sync <fresh-output-directory> after` on the
patch. Identical scripts ran outside pytest. The script initializes a
real LocalBackend and SQLite store, seeds 501 chronologically ordered
Alice rows plus a Bob row, and runs sync into a real CodexAdapter
AGENTS.md file. It then presents the oldest existing fact from a second
agent file and calls sync_import. The unpatched duplicate path makes an
actual Ollama HTTP request to a loopback embedding service; the patched
deduplication path makes none.

<details>
<summary>Standalone reproduction script</summary>

```python
"""Standalone real SQLite/vector/FTS/HTTP and agent-file boundary reproductions.

Usage: python memory_http_proof.py {list,vectors,sync} FRESH_OUTPUT {before,after}
The loopback embedding service returns controlled vectors, not model inference.
"""

import asyncio
from datetime import datetime, timedelta, timezone
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import platform
import sqlite3
import sys
import threading

import headroom
import headroom._core
from headroom.memory.backends.local import LocalBackend, LocalBackendConfig
from headroom.memory.config import EmbedderBackend, MemoryConfig, VectorBackend
from headroom.memory.core import HierarchicalMemory
from headroom.memory.models import Memory
from headroom.memory.sync import sync, sync_import
from headroom.memory.sync_adapters.codex_agent import CodexAdapter
from headroom.memory.system import MemorySystem

sys.stdout.reconfigure(encoding="utf-8")
case, directory, expected = sys.argv[1:]
assert case == "sync"
assert expected in ("before", "after")
out = Path(directory).resolve()
out.mkdir(parents=True, exist_ok=False)
os.environ["HEADROOM_STATELESS"] = "0"
os.environ["HEADROOM_WORKSPACE_DIR"] = str(out / "workspace")
os.environ["HEADROOM_CCR_BACKEND"] = "memory"
os.environ["HEADROOM_OFFLINE"] = "0"  # Only the ephemeral loopback service is called.


class Endpoint(BaseHTTPRequestHandler):
    dimension = 384
    requests = []

    def log_message(self, *_):
        pass

    def do_POST(self):
        payload = json.loads(self.rfile.read(int(self.headers["content-length"])))
        Endpoint.requests.append({"path": self.path, "body": payload, "dimension": self.dimension})
        assert self.path == "/api/embeddings", self.path
        body = json.dumps({"embedding": [1.0] + [0.0] * (self.dimension - 1)}).encode()
        self.send_response(200)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)


server = ThreadingHTTPServer(("127.0.0.1", 0), Endpoint)
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
url = f"http://127.0.0.1:{server.server_port}"


def backend_config(name):
    return LocalBackendConfig(
        db_path=str(out / (name + ".db")), embedder_backend="ollama",
        embedder_model="probe-fixed", ollama_base_url=url, vector_dimension=384,
    )


async def sync_case():
    backend = LocalBackend(backend_config("sync"))
    try:
        await backend._ensure_initialized()
        now = datetime.now(timezone.utc)
        memories = [Memory(
            id=f"alice-{i:04}", content=f"Distinct memory {i}", user_id="alice",
            created_at=now - timedelta(seconds=i),
        ) for i in range(501)]
        memories.append(Memory(content="Bob's private fact", user_id="bob"))
        await backend._hierarchical_memory._store.save_batch(memories)
        adapter = CodexAdapter(out / "AGENTS.md")
        first = await sync(backend, adapter, "alice", state_path=out / "sync-state.json", force=True)
        exported = {m.content for m in await adapter.read_memories()}
        older_adapter = CodexAdapter(out / "older-agent.md")
        await older_adapter.write_memories([{"content": "Distinct memory 500"}])
        imported = await sync_import(backend, older_adapter, "alice")
        result = {
            "first_exported": first.exported,
            "oldest_exported": "Distinct memory 500" in exported,
            "other_user_exported": "Bob's private fact" in exported,
            "reimported_existing_oldest": imported,
            "agent_file_sha256": hashlib.sha256((out / "AGENTS.md").read_bytes()).hexdigest(),
        }
        with sqlite3.connect(out / "sync.db") as db:
            result["alice_primary_rows"] = db.execute("SELECT COUNT(*) FROM memories WHERE user_id = 'alice'").fetchone()[0]
        assert result["first_exported"] == (501 if expected == "after" else 500), result
        assert result["oldest_exported"] == (expected == "after"), result
        assert result["reimported_existing_oldest"] == (0 if expected == "after" else 1), result
        assert result["alice_primary_rows"] == (501 if expected == "after" else 502), result
        assert result["other_user_exported"] is False
        return result
    finally:
        await backend.close()


try:
    result = asyncio.run(sync_case())
    result.update({
        "case": case, "expected": expected, "python": platform.python_version(),
        "source": str(Path(headroom.__file__).resolve()),
        "native": str(Path(headroom._core.__file__).resolve()),
        "http_requests": Endpoint.requests,
        "not_tested": "Trained model inference, paid APIs, provider answer quality, non-Windows runtime, non-local memory backends.",
    })
    (out / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8", newline="\n")
    print(json.dumps(result, indent=2))
finally:
    server.shutdown()
    server.server_close()
    worker.join(timeout=5)
```

</details>

- Observed result: Main exports 500 memories, omits the oldest and
imports that existing fact as a new row (502 Alice rows). The patch
exports all 501, includes the oldest, imports zero duplicates and keeps
501 Alice rows. Bob's row is excluded in both runs.
- Not tested: Very large-store throughput or peak memory, concurrent
sync snapshot consistency, changes to the existing fingerprint sampling
policy, remote memory backends, trained model inference, paid APIs,
non-Windows runtime, or the complete Python suite.

## Runtime Rollout Safety

- Rollout-managed feature(s): Existing memory behavior; no new feature
flag or channel expansion.
- Minimum rollout channel: Normal release after maintainer review and CI
qualification.
- Stable/default behavior changed: Agent sync covers all user rows;
ordinary get_user_memories calls keep the default 100-row limit and
explicit limits still apply.
- Kill switch / disable path: Use the existing memory-disable path, or
revert this commit.
- Unsafe override required: No.
- Qualification impact: Related tests and original local gates passed;
hosted CI is tracked separately.
- Rollback path: Revert this commit; no dependency or storage-schema
change.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have added tests that prove the fix
- [x] Relevant new and existing tests pass locally
- [x] I did **not** edit `CHANGELOG.md`

## Additional Notes

Synchronization already materializes a snapshot and hash set. Covering
the full store makes their size scale with the user's retained memories;
this patch does not introduce a paging protocol or alter the existing
no-op fingerprint algorithm.

Developed and validated with Codex assistance. The author reviewed the
changes and before/after evidence.

<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4832"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->

Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
2026-11-16 23:08:11 -06:00

108 lines
4.2 KiB
Python

"""Agent synchronization must include memories older than the newest 500."""
from datetime import datetime, timedelta
from unittest.mock import AsyncMock
import numpy as np
import pytest
from headroom.memory.adapters.graph import InMemoryGraphStore
from headroom.memory.adapters.hnsw import HNSWVectorIndex
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
from headroom.memory.backends.local import LocalBackend, LocalBackendConfig
from headroom.memory.config import MemoryConfig
from headroom.memory.core import HierarchicalMemory
from headroom.memory.models import Memory
from headroom.memory.ports import MemoryFilter
from headroom.memory.sync import sync, sync_export, sync_import
from headroom.memory.sync_adapters.codex_agent import CodexAdapter
@pytest.fixture
async def backend(tmp_path):
store = SQLiteMemoryStore(tmp_path / "memories.db")
embedder = AsyncMock()
embedder.embed.return_value = np.ones(4)
memory = HierarchicalMemory(
store,
HNSWVectorIndex(dimension=4),
AsyncMock(),
embedder,
config=MemoryConfig(auto_bubble=False),
)
backend = LocalBackend(LocalBackendConfig(db_path=str(tmp_path / "memories.db")))
backend._hierarchical_memory = memory
backend._graph = InMemoryGraphStore()
backend._initialized = True
now = datetime.now()
memories = [
Memory(
id=f"alice-{i:04}",
content=f"Distinct memory {i}",
user_id="alice",
created_at=now - timedelta(seconds=i),
)
for i in range(501)
]
memories.append(Memory(content="Other user's private fact", user_id="bob"))
await store.save_batch(memories)
return backend
@pytest.fixture
def adapter(tmp_path):
return CodexAdapter(tmp_path / "AGENTS.md")
@pytest.mark.asyncio
async def test_import_deduplicates_a_memory_older_than_the_latest_500(backend, adapter):
await adapter.write_memories([{"content": "Distinct memory 500", "headroom_id": "alice-0500"}])
assert await sync_import(backend, adapter, "alice") == 0
rows = await backend._hierarchical_memory._store.query(MemoryFilter(user_id="alice"))
assert len(rows) == 501
@pytest.mark.asyncio
async def test_export_includes_the_oldest_memory_and_keeps_user_scope(backend, adapter):
assert await sync_export(backend, adapter, "alice") == 501
exported = await adapter.read_memories()
assert "Distinct memory 500" in {m.content for m in exported}
assert "Other user's private fact" not in {m.content for m in exported}
@pytest.mark.asyncio
@pytest.mark.parametrize("force", [False, True])
async def test_full_sync_exports_every_memory_and_is_idempotent(backend, adapter, tmp_path, force):
state = tmp_path / "state.json"
first = await sync(backend, adapter, "alice", state_path=state, force=force)
assert first.imported == 0
assert first.exported == 501
second = await sync(backend, adapter, "alice", state_path=state, force=True)
assert second.imported == 0
assert second.exported == 0
@pytest.mark.asyncio
async def test_sync_refetch_after_import_does_not_truncate_exports(backend, adapter, tmp_path):
await adapter.write_memories([{"content": "New agent fact", "headroom_id": "new-agent-fact"}])
result = await sync(backend, adapter, "alice", state_path=tmp_path / "state.json", force=True)
assert result.imported == 1
assert result.exported == 501
assert len(await adapter.read_memories()) == 502
@pytest.mark.asyncio
async def test_explicit_existing_snapshot_is_not_requeried(backend, adapter):
snapshot = [Memory(content="Explicit snapshot", user_id="alice")]
backend.get_user_memories = AsyncMock(side_effect=AssertionError("snapshot already provided"))
assert await sync_export(backend, adapter, "alice", existing_memories=snapshot) == 1
assert await sync_import(backend, adapter, "alice", existing_memories=snapshot) == 0
backend.get_user_memories.assert_not_awaited()
@pytest.mark.asyncio
async def test_optional_unbounded_query_preserves_the_default_limit(backend):
assert len(await backend.get_user_memories("alice")) == 100
assert len(await backend.get_user_memories("alice", limit=3)) == 3
assert len(await backend.get_user_memories("alice", limit=None)) == 501