mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
## Description Incremental sync hashes only the row count and the first eight characters of the newest five IDs. An in-place content edit keeps those values unchanged, so sync returns early and never exports the corrected fact. Hash the content, complete IDs and fields used for export/lineage across the already-fetched snapshot. Unchanged snapshots still take the no-op path. The persisted fingerprint is opaque; existing states receive one normal sync after this change. Related to #4832's full-store sync work; this fixes a separate contract on the current main snapshot. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Include full IDs, content and export/lineage fields from every returned row; exclude volatile access counters. - Add regressions against the unmodified base `5023c84e527e28281dd73d1ba04184fe738aac0f` and real file/component coverage. ## Testing - [x] Relevant unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Formatting passes (`ruff format --check .`) - [x] Type checking passes (original full-package mypy hook with httpx installed) - [x] New tests added - [ ] Manual testing performed by the human author ### Test Output 6 new regressions; focused sync suite: 41 passed. Broader memory run: 675 passed, 82 skipped, 8 failed. The same 8 failures reproduce on the exact unmodified base (275 passed in the two affected files): six Windows SQLite temporary-directory cleanup failures and two POSIX-path assumptions. No old fixture changed. Original pre-commit and commit-msg hooks passed. Original full pre-push `make ci-precheck` passed; native extension rebuilt/import-verified. Full gate summaries: ```text cargo fmt / clippy --workspace: passed Rust workspace: 1755 passed, 3 ignored Native extension: rebuilt and import-verified Critical Python: 177 passed, 8 skipped commitlint: passed ``` ## Real Behavior Proof - Environment: Windows 11, CPython 3.13.9, existing development environment; base `5023c84e527e28281dd73d1ba04184fe738aac0f`, patch `c4ffa08665c26491b2379af0f4d953ceb203ee6b`. Real LocalBackend factory, SQLite/FTS/vector store, agent files and Ollama HTTP adapter; the loopback service returns controlled 384D vectors. - Exact command / steps: save the following script as `proof.py`. Run `python proof.py fingerprint <fresh-output-directory> before` from the base checkout, then `python proof.py fingerprint <another-fresh-directory> after` from the patch. The identical script ran outside pytest. - Observed result: first_exported=1; unchanged_exported=0; edited_exported: main=0, patch=1; corrected_in_file: main=false, patch=true. The existing additive bidirectional semantics also reimport the old file fact (edited_imported=1 on the patch); this change does not implement deletion propagation or conflict resolution. - Not tested: trained-model inference or answer quality, paid/provider APIs, non-local backends, production workloads, other operating systems or the entire Python suite. Local focused, broader and original-gate results are reported separately. <details> <summary>Standalone runtime reproduction</summary> ```python """Real LocalBackend/Ollama HTTP/SQLite/agent-file before-after replay.""" import asyncio import json import os from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import sys import threading from headroom.memory.backends.local import LocalBackend, LocalBackendConfig from headroom.memory.sync import sync from headroom.memory.sync_adapters.codex_agent import CodexAdapter from headroom.memory.sync_adapters.claude_code import ClaudeCodeAdapter case, folder, expected = sys.argv[1:] out = Path(folder).resolve() out.mkdir(parents=True, exist_ok=False) os.environ.update(HEADROOM_STATELESS="0", HEADROOM_WORKSPACE_DIR=str(out/'workspace'), HEADROOM_OFFLINE="0") class Endpoint(BaseHTTPRequestHandler): requests = [] def log_message(self, *_): pass def do_POST(self): payload=json.loads(self.rfile.read(int(self.headers['content-length']))) self.requests.append({'path':self.path,'body':payload}) assert self.path=='/api/embeddings' body=json.dumps({'embedding':[1.0]+[0.0]*383}).encode() self.send_response(200) self.send_header('Content-Type','application/json') self.send_header('Content-Length',str(len(body))) self.end_headers() self.wfile.write(body) server=ThreadingHTTPServer(('127.0.0.1',0),Endpoint) threading.Thread(target=server.serve_forever,daemon=True).start() async def run(): backend=LocalBackend(LocalBackendConfig(db_path=str(out/'memory.db'),embedder_backend='ollama',embedder_model='controlled-vector',vector_dimension=384,ollama_base_url=f'http://127.0.0.1:{server.server_port}')) state=out/'sync-state.json' try: if case=='fingerprint': saved=await backend.save_memory('Original orchard fact',user_id='alice') adapter=CodexAdapter(out/'AGENTS.md') first=await sync(backend,adapter,'alice',state) unchanged=await sync(backend,adapter,'alice',state) await backend._hierarchical_memory.update(saved.id,content='Corrected orchard fact') result=await sync(backend,adapter,'alice',state) text=adapter._path.read_text(encoding='utf-8') data={'first_exported':first.exported,'unchanged_exported':unchanged.exported,'edited_exported':result.exported,'corrected_in_file':'Corrected orchard fact' in text,'edited_imported':result.imported} assert data['first_exported']==1 and data['unchanged_exported']==0 assert data['corrected_in_file']==(expected=='after'),data assert result.exported==(1 if expected=='after' else 0),data else: content='Use foo---bar as the separator\nKeep the second line intact.' saved=await backend.save_memory(content,user_id='alice') adapter=ClaudeCodeAdapter(out/'claude-memory') first=await sync(backend,adapter,'alice',state,force=True) readback=await adapter.read_memories() repeated=await sync(backend,adapter,'alice',state,force=True) rows=await backend.get_user_memories('alice') data={'first_exported':first.exported,'round_trip_equal':readback[0].content==content,'reread_content':readback[0].content,'repeat_imported':repeated.imported,'repeat_exported':repeated.exported,'primary_count':len(rows)} assert data['round_trip_equal']==(expected=='after'),data assert data['repeat_imported']==(0 if expected=='after' else 1),data data['requests']=Endpoint.requests data['source_file']=__import__('headroom').__file__ (out/'result.json').write_text(json.dumps(data,indent=2),encoding='utf-8',newline='\n') print(json.dumps(data,ensure_ascii=False)) finally: await backend.close() try: asyncio.run(run()) finally: server.shutdown();server.server_close() ``` </details> ## Runtime Rollout Safety - Rollout-managed feature(s): none; existing memory sync. - Minimum rollout channel: existing memory sync users. - Stable/default behavior changed: edited sync snapshots trigger comparison/export instead of a false no-op. - Kill switch / disable path: stop invoking memory sync; no new toggle. - Unsafe override required: no. - Qualification impact: memory synchronization correctness; no dependency, native or workflow change. - Rollback path: revert this commit. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented the non-obvious behavior - [x] I have added regression tests - [x] I did not edit CHANGELOG.md ## Additional Notes Implementation and local test execution were completed with Codex assistance. The author personally reviewed the two patches and the before/after evidence; this does not claim the author personally ran the tests. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4885" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> Co-authored-by: JD Davis <mxjerrett@gmail.com>
81 lines
3.4 KiB
Python
81 lines
3.4 KiB
Python
"""Incremental sync must observe edits without new rows or IDs."""
|
|
|
|
from dataclasses import replace
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from headroom.memory.adapters.fts5 import FTS5TextIndex
|
|
from headroom.memory.adapters.graph import InMemoryGraphStore
|
|
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
|
|
from headroom.memory.adapters.sqlite_vector import SQLiteVectorIndex, is_sqlite_vec_available
|
|
from headroom.memory.backends.local import LocalBackend
|
|
from headroom.memory.config import MemoryConfig
|
|
from headroom.memory.core import HierarchicalMemory
|
|
from headroom.memory.models import Memory
|
|
from headroom.memory.sync import _db_fingerprint, sync
|
|
from headroom.memory.sync_adapters.codex_agent import CodexAdapter
|
|
|
|
|
|
class FixedEmbedder:
|
|
async def embed(self, text):
|
|
return np.array([1.0, 0.0, 0.0], dtype=np.float32)
|
|
|
|
|
|
@pytest.mark.parametrize("position", [0, 6])
|
|
def test_fingerprint_observes_content_edits_at_every_position(position):
|
|
rows = [Memory(id=f"memory-{i}", content=f"fact {i}", user_id="alice") for i in range(7)]
|
|
changed = rows.copy()
|
|
changed[position] = replace(rows[position], content="corrected fact")
|
|
assert _db_fingerprint(rows) != _db_fingerprint(changed)
|
|
|
|
|
|
def test_fingerprint_uses_complete_ids():
|
|
original = Memory(id="12345678-old", content="fact", user_id="alice")
|
|
assert _db_fingerprint([original]) != _db_fingerprint([replace(original, id="12345678-new")])
|
|
|
|
|
|
def test_fingerprint_observes_export_eligibility():
|
|
original = Memory(content="fact", user_id="alice")
|
|
changed = replace(original, metadata={"source_agent": "codex", "sync_direction": "import"})
|
|
assert _db_fingerprint([original]) != _db_fingerprint([changed])
|
|
|
|
|
|
def test_fingerprint_stable_for_equivalent_rows():
|
|
rows = [Memory(content="fact", user_id="alice")]
|
|
assert _db_fingerprint(rows) == _db_fingerprint([replace(rows[0])])
|
|
assert _db_fingerprint([]) == _db_fingerprint([])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.skipif(not is_sqlite_vec_available(), reason="sqlite-vec unavailable")
|
|
async def test_incremental_sync_exports_in_place_sqlite_edit(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HEADROOM_STATELESS", "0")
|
|
memory = HierarchicalMemory(
|
|
SQLiteMemoryStore(tmp_path / "memory.db"),
|
|
SQLiteVectorIndex(dimension=3, db_path=tmp_path / "vectors.db"),
|
|
FTS5TextIndex(tmp_path / "fts.db"),
|
|
FixedEmbedder(),
|
|
config=MemoryConfig(auto_bubble=False, vector_dimension=3),
|
|
)
|
|
backend = LocalBackend()
|
|
backend._hierarchical_memory = memory
|
|
backend._graph = InMemoryGraphStore()
|
|
backend._initialized = True
|
|
adapter = CodexAdapter(tmp_path / "AGENTS.md")
|
|
state_path = tmp_path / "state.json"
|
|
try:
|
|
saved = await memory.add("Original fact", user_id="alice")
|
|
first = await sync(backend, adapter, "alice", state_path)
|
|
assert first.exported == 1
|
|
unchanged = await sync(backend, adapter, "alice", state_path)
|
|
assert unchanged.imported == unchanged.exported == 0
|
|
await memory.update(saved.id, content="Corrected fact")
|
|
updated = await sync(backend, adapter, "alice", state_path)
|
|
assert updated.exported == 1
|
|
assert "Corrected fact" in adapter._path.read_text(encoding="utf-8")
|
|
repeated = await sync(backend, adapter, "alice", state_path)
|
|
assert repeated.imported == repeated.exported == 0
|
|
finally:
|
|
await memory.close()
|