mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
## Description A mismatched embedding reaches the vector index after the primary memory row has already been saved. A rejected add leaves a half-written row; a rejected supersession also ends the old version and removes its index entry before failing. This occurs with a vector_dimension that differs from the embedder's actual output. Validate each newly produced embedding's shape before the first primary-store mutation. Apply this to add, the whole add_batch, content updates and supersession. An invalid replacement leaves the old row, cached object and version chain intact. Refs #4362, item 2's half-write boundary. The configured vector dimension remains explicit; this does not automatically resize or migrate an index. ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) ## Changes Made - Reject non-vector and wrong-width embeddings before saving rows. - Validate an entire batch before saving any member. - Validate a content update before mutating a store-returned object, preserving update()'s existing object identity and metadata-only refresh behavior. - Validate supersession before ending the old version or removing its indexes. ## Testing - [x] Relevant unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [x] Formatting passes (`ruff format --check .`) - [x] Type checking passes (original full-package mypy hook) - [x] Regression tests added - [x] Standalone runtime validation completed ### Test Output ```text Final regressions against original production code: 7 failed, 1 passed pytest tests/test_memory_embedding_validation.py tests/test_memory/test_core_operations.py tests/test_memory/test_hierarchical.py tests/test_memory/test_core_metadata_index_sync.py tests/test_memory/test_clear_scope_consistency.py -q: 65 passed, 51 skipped Original pre-commit and commit-msg hooks: passed Whole-tree Ruff check and format --check: passed Original pre-push make ci-precheck: passed Rust workspace: 1749 passed, 3 ignored Native extension: rebuilt, installed and import-verified Critical Python: 177 passed, 8 skipped commitlint: passed ``` The owning-suite skips require Hugging Face model files unavailable offline. Existing datetime/optional-model deprecation warnings remain visible in the logs. ## Real Behavior Proof - Environment: Windows 11, CPython 3.13.9, existing dev venv; clean production baseline `9f2cc017863f540c61f8747b3c744a97776ebaf2`, patch `50d126b1b8379390e72d3e47216519f9d297b4a1`. Explicit checkout on PYTHONPATH; no paid/provider endpoint. The loopback service returns controlled vectors, not trained-model inference. - Exact command / steps: Save the script below as `proof.py`. From each checkout run `python proof.py vectors <fresh-output-directory> before` on main, or `python proof.py vectors <fresh-output-directory> after` on the patch. Identical scripts ran outside pytest. The script creates real HierarchicalMemory with SQLite, FTS5, HNSW(dimension=384) and the Ollama HTTP adapter. A bound loopback server first returns a 768-dimensional embedding for a rejected add, then 384 dimensions for a valid original, then 768 dimensions for a rejected supersession. It inspects primary rows, the original version and the live vector index. <details> <summary>Standalone reproduction script</summary> ```python """Standalone real SQLite/vector/FTS/HTTP and agent-file boundary reproductions. Usage: python memory_http_proof.py {list,vectors,sync} FRESH_OUTPUT {before,after} The loopback embedding service returns controlled vectors, not model inference. """ import asyncio from datetime import datetime, timedelta, timezone import hashlib from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os from pathlib import Path import platform import sqlite3 import sys import threading import headroom import headroom._core from headroom.memory.backends.local import LocalBackend, LocalBackendConfig from headroom.memory.config import EmbedderBackend, MemoryConfig, VectorBackend from headroom.memory.core import HierarchicalMemory from headroom.memory.models import Memory from headroom.memory.sync import sync, sync_import from headroom.memory.sync_adapters.codex_agent import CodexAdapter from headroom.memory.system import MemorySystem sys.stdout.reconfigure(encoding="utf-8") case, directory, expected = sys.argv[1:] assert case == "vectors" assert expected in ("before", "after") out = Path(directory).resolve() out.mkdir(parents=True, exist_ok=False) os.environ["HEADROOM_STATELESS"] = "0" os.environ["HEADROOM_WORKSPACE_DIR"] = str(out / "workspace") os.environ["HEADROOM_CCR_BACKEND"] = "memory" os.environ["HEADROOM_OFFLINE"] = "0" # Only the ephemeral loopback service is called. class Endpoint(BaseHTTPRequestHandler): dimension = 384 requests = [] def log_message(self, *_): pass def do_POST(self): payload = json.loads(self.rfile.read(int(self.headers["content-length"]))) Endpoint.requests.append({"path": self.path, "body": payload, "dimension": self.dimension}) assert self.path == "/api/embeddings", self.path body = json.dumps({"embedding": [1.0] + [0.0] * (self.dimension - 1)}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) server = ThreadingHTTPServer(("127.0.0.1", 0), Endpoint) worker = threading.Thread(target=server.serve_forever, daemon=True) worker.start() url = f"http://127.0.0.1:{server.server_port}" async def new_memory(name): return await HierarchicalMemory.create(MemoryConfig( db_path=out / (name + ".db"), embedder_backend=EmbedderBackend.OLLAMA, embedder_model="nomic-embed-text", ollama_base_url=url, vector_backend=VectorBackend.HNSW, vector_dimension=384, auto_bubble=False, )) async def vectors_case(): memory = await new_memory("vectors") Endpoint.dimension = 768 result = {} try: try: await memory.add("Rejected incompatible embedding", user_id="alice") raise AssertionError("mismatch was accepted") except ValueError as exc: result["add_error"] = str(exc) result["rows_after_rejected_add"] = len(await memory.get_user_memories("alice")) Endpoint.dimension = 384 original = await memory.add("Valid original fact", user_id="alice") Endpoint.dimension = 768 try: await memory.supersede(original.id, "Rejected replacement fact") raise AssertionError("replacement mismatch was accepted") except ValueError as exc: result["supersede_error"] = str(exc) stored = await memory._store.get(original.id) result["original_still_active"] = stored.valid_until is None and stored.superseded_by is None result["vector_entries_after_rejected_supersede"] = memory._vector_index.size with sqlite3.connect(out / "vectors.db") as db: result["all_primary_rows"] = db.execute("SELECT COUNT(*) FROM memories WHERE user_id = 'alice'").fetchone()[0] assert result["rows_after_rejected_add"] == (0 if expected == "after" else 1), result assert result["original_still_active"] == (expected == "after"), result assert result["vector_entries_after_rejected_supersede"] == (1 if expected == "after" else 0), result assert result["all_primary_rows"] == (1 if expected == "after" else 3), result return result finally: await memory.close() try: result = asyncio.run(vectors_case()) result.update({ "case": case, "expected": expected, "python": platform.python_version(), "source": str(Path(headroom.__file__).resolve()), "native": str(Path(headroom._core.__file__).resolve()), "http_requests": Endpoint.requests, "not_tested": "Trained model inference, paid APIs, provider answer quality, non-Windows runtime, non-local memory backends.", }) (out / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8", newline="\n") print(json.dumps(result, indent=2)) finally: server.shutdown() server.server_close() worker.join(timeout=5) ``` </details> - Observed result: Main leaves one row after the rejected add; after the rejected replacement it has three primary rows, the original is no longer active and its vector was removed. The patch leaves zero rows after the rejected add, then retains exactly the valid original row and its one vector after the rejected replacement. - Not tested: General cross-component transactions for arbitrary storage/index I/O failures, automatic dimension discovery or index migration, remote vector backends, trained model inference, paid APIs, non-Windows runtime, or the complete Python suite. ## Runtime Rollout Safety - Rollout-managed feature(s): Existing memory behavior; no new feature flag or channel expansion. - Minimum rollout channel: Normal release after maintainer review and CI qualification. - Stable/default behavior changed: Embedding shape errors are reported before primary writes. Matching vectors and explicit auto_embed=False operations retain their existing behavior. - Kill switch / disable path: Use the existing memory-disable path, or revert this commit. - Unsafe override required: No. - Qualification impact: Related tests and original local gates passed; hosted CI is tracked separately. - Rollback path: Revert this commit; no dependency or storage-schema change. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have added tests that prove the fix - [x] Relevant new and existing tests pass locally - [x] I did **not** edit `CHANGELOG.md` ## Additional Notes Broader regression exposed update()'s object-identity contract; the final fix preserves it without changing the existing fixture. The author's review covered the initial patch and evidence; this compatibility adjustment was additionally reviewed and tested with Codex. Developed and validated with Codex assistance. The author reviewed the changes and before/after evidence. <!-- devin-review-badge-begin --> --- <a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4831" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4" alt="Devin Review"></picture></a> <!-- devin-review-badge-end --> Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
105 lines
4.2 KiB
Python
105 lines
4.2 KiB
Python
"""Reject incompatible embeddings before changing primary or cached memories."""
|
|
|
|
from unittest.mock import AsyncMock
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from headroom.memory.adapters.cache import LRUMemoryCache
|
|
from headroom.memory.adapters.hnsw import HNSWVectorIndex
|
|
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
|
|
from headroom.memory.config import MemoryConfig
|
|
from headroom.memory.core import HierarchicalMemory
|
|
from headroom.memory.models import Memory
|
|
from headroom.memory.ports import MemoryFilter
|
|
|
|
|
|
@pytest.fixture
|
|
def components(tmp_path):
|
|
store = SQLiteMemoryStore(tmp_path / "memories.db")
|
|
vector = HNSWVectorIndex(dimension=4)
|
|
text = AsyncMock()
|
|
embedder = AsyncMock()
|
|
cache = LRUMemoryCache()
|
|
memory = HierarchicalMemory(
|
|
store, vector, text, embedder, cache, MemoryConfig(auto_bubble=False)
|
|
)
|
|
return memory, store, vector, text, embedder, cache
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("shape", [(8,), (2, 2), ()])
|
|
async def test_add_rejects_incompatible_shape_without_persisting(components, shape):
|
|
memory, store, vector, text, embedder, cache = components
|
|
embedder.embed.return_value = np.zeros(shape, dtype=np.float32)
|
|
with pytest.raises(ValueError):
|
|
await memory.add("Must not be half-written", user_id="alice")
|
|
assert await store.query(MemoryFilter(user_id="alice")) == []
|
|
assert vector.size == 0
|
|
text.index_memory.assert_not_called()
|
|
assert cache.size == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_batch_validates_every_embedding_before_saving_any_row(components):
|
|
memory, store, vector, _, embedder, _ = components
|
|
embedder.embed_batch.return_value = [np.ones(4), np.ones(8)]
|
|
with pytest.raises(ValueError):
|
|
await memory.add_batch(
|
|
[
|
|
{"content": "Valid first vector", "user_id": "alice"},
|
|
{"content": "Invalid second vector", "user_id": "alice"},
|
|
]
|
|
)
|
|
assert await store.query(MemoryFilter(user_id="alice")) == []
|
|
assert vector.size == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("operation", ["update", "supersede"])
|
|
async def test_rejected_replacement_keeps_the_existing_memory_and_lineage(components, operation):
|
|
memory, store, vector, _, embedder, cache = components
|
|
embedder.embed.return_value = np.ones(4)
|
|
original = await memory.add("Original fact", user_id="alice")
|
|
embedder.embed.return_value = np.ones(8)
|
|
with pytest.raises(ValueError):
|
|
if operation == "update":
|
|
await memory.update(
|
|
original.id, content="Invalid replacement", metadata={"changed": True}
|
|
)
|
|
else:
|
|
await memory.supersede(original.id, "Invalid replacement")
|
|
stored = await store.get(original.id)
|
|
assert stored.content == "Original fact"
|
|
assert stored.metadata == {}
|
|
assert stored.valid_until is None
|
|
assert stored.superseded_by is None
|
|
assert len(await store.query(MemoryFilter(user_id="alice", include_superseded=True))) == 1
|
|
assert (await cache.get(original.id)).content == "Original fact"
|
|
assert vector.size == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_update_does_not_mutate_a_reference_returning_store(components):
|
|
memory, _, _, _, embedder, _ = components
|
|
original = Memory(content="Original", user_id="alice", embedding=np.ones(4))
|
|
memory._store = AsyncMock()
|
|
memory._store.get.return_value = original
|
|
embedder.embed.return_value = np.ones(8)
|
|
with pytest.raises(ValueError):
|
|
await memory.update(original.id, content="Replacement", metadata={"changed": True})
|
|
assert original.content == "Original"
|
|
assert original.metadata == {}
|
|
memory._store.save.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matching_embeddings_and_explicit_no_embedding_still_work(components):
|
|
memory, store, vector, _, embedder, _ = components
|
|
embedder.embed.return_value = np.ones(4)
|
|
with_vector = await memory.add("Indexed fact", user_id="alice")
|
|
without_vector = await memory.add("Text-only fact", user_id="alice", auto_embed=False)
|
|
assert (await store.get(with_vector.id)).embedding.shape == (4,)
|
|
assert (await store.get(without_vector.id)).embedding is None
|
|
assert vector.size == 1
|