Files
headroom/tests/test_memory_embedding_validation.py
Yuchuan HeandJerrettDavis cd6f438fec fix(memory): validate embedding dimensions before writes (#4831)
## Description

A mismatched embedding reaches the vector index after the primary memory
row has already been saved. A rejected add leaves a half-written row; a
rejected supersession also ends the old version and removes its index
entry before failing. This occurs with a vector_dimension that differs
from the embedder's actual output.

Validate each newly produced embedding's shape before the first
primary-store mutation. Apply this to add, the whole add_batch, content
updates and supersession. An invalid replacement leaves the old row,
cached object and version chain intact.

Refs #4362, item 2's half-write boundary. The configured vector
dimension remains explicit; this does not automatically resize or
migrate an index.

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)

## Changes Made

- Reject non-vector and wrong-width embeddings before saving rows.
- Validate an entire batch before saving any member.
- Validate a content update before mutating a store-returned object,
preserving update()'s existing object identity and metadata-only refresh
behavior.
- Validate supersession before ending the old version or removing its
indexes.

## Testing

- [x] Relevant unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)
- [x] Formatting passes (`ruff format --check .`)
- [x] Type checking passes (original full-package mypy hook)
- [x] Regression tests added
- [x] Standalone runtime validation completed

### Test Output

```text
Final regressions against original production code: 7 failed, 1 passed
pytest tests/test_memory_embedding_validation.py tests/test_memory/test_core_operations.py tests/test_memory/test_hierarchical.py tests/test_memory/test_core_metadata_index_sync.py tests/test_memory/test_clear_scope_consistency.py -q:
  65 passed, 51 skipped
Original pre-commit and commit-msg hooks: passed
Whole-tree Ruff check and format --check: passed
Original pre-push make ci-precheck: passed
  Rust workspace: 1749 passed, 3 ignored
  Native extension: rebuilt, installed and import-verified
  Critical Python: 177 passed, 8 skipped
  commitlint: passed
```

The owning-suite skips require Hugging Face model files unavailable
offline. Existing datetime/optional-model deprecation warnings remain
visible in the logs.

## Real Behavior Proof

- Environment: Windows 11, CPython 3.13.9, existing dev venv; clean
production baseline `9f2cc017863f540c61f8747b3c744a97776ebaf2`, patch
`50d126b1b8379390e72d3e47216519f9d297b4a1`. Explicit checkout on
PYTHONPATH; no paid/provider endpoint. The loopback service returns
controlled vectors, not trained-model inference.
- Exact command / steps: Save the script below as `proof.py`. From each
checkout run `python proof.py vectors <fresh-output-directory> before`
on main, or `python proof.py vectors <fresh-output-directory> after` on
the patch. Identical scripts ran outside pytest. The script creates real
HierarchicalMemory with SQLite, FTS5, HNSW(dimension=384) and the Ollama
HTTP adapter. A bound loopback server first returns a 768-dimensional
embedding for a rejected add, then 384 dimensions for a valid original,
then 768 dimensions for a rejected supersession. It inspects primary
rows, the original version and the live vector index.

<details>
<summary>Standalone reproduction script</summary>

```python
"""Standalone real SQLite/vector/FTS/HTTP and agent-file boundary reproductions.

Usage: python memory_http_proof.py {list,vectors,sync} FRESH_OUTPUT {before,after}
The loopback embedding service returns controlled vectors, not model inference.
"""

import asyncio
from datetime import datetime, timedelta, timezone
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import platform
import sqlite3
import sys
import threading

import headroom
import headroom._core
from headroom.memory.backends.local import LocalBackend, LocalBackendConfig
from headroom.memory.config import EmbedderBackend, MemoryConfig, VectorBackend
from headroom.memory.core import HierarchicalMemory
from headroom.memory.models import Memory
from headroom.memory.sync import sync, sync_import
from headroom.memory.sync_adapters.codex_agent import CodexAdapter
from headroom.memory.system import MemorySystem

sys.stdout.reconfigure(encoding="utf-8")
case, directory, expected = sys.argv[1:]
assert case == "vectors"
assert expected in ("before", "after")
out = Path(directory).resolve()
out.mkdir(parents=True, exist_ok=False)
os.environ["HEADROOM_STATELESS"] = "0"
os.environ["HEADROOM_WORKSPACE_DIR"] = str(out / "workspace")
os.environ["HEADROOM_CCR_BACKEND"] = "memory"
os.environ["HEADROOM_OFFLINE"] = "0"  # Only the ephemeral loopback service is called.


class Endpoint(BaseHTTPRequestHandler):
    dimension = 384
    requests = []

    def log_message(self, *_):
        pass

    def do_POST(self):
        payload = json.loads(self.rfile.read(int(self.headers["content-length"])))
        Endpoint.requests.append({"path": self.path, "body": payload, "dimension": self.dimension})
        assert self.path == "/api/embeddings", self.path
        body = json.dumps({"embedding": [1.0] + [0.0] * (self.dimension - 1)}).encode()
        self.send_response(200)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)


server = ThreadingHTTPServer(("127.0.0.1", 0), Endpoint)
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
url = f"http://127.0.0.1:{server.server_port}"


async def new_memory(name):
    return await HierarchicalMemory.create(MemoryConfig(
        db_path=out / (name + ".db"), embedder_backend=EmbedderBackend.OLLAMA,
        embedder_model="nomic-embed-text", ollama_base_url=url,
        vector_backend=VectorBackend.HNSW, vector_dimension=384, auto_bubble=False,
    ))


async def vectors_case():
    memory = await new_memory("vectors")
    Endpoint.dimension = 768
    result = {}
    try:
        try:
            await memory.add("Rejected incompatible embedding", user_id="alice")
            raise AssertionError("mismatch was accepted")
        except ValueError as exc:
            result["add_error"] = str(exc)
        result["rows_after_rejected_add"] = len(await memory.get_user_memories("alice"))
        Endpoint.dimension = 384
        original = await memory.add("Valid original fact", user_id="alice")
        Endpoint.dimension = 768
        try:
            await memory.supersede(original.id, "Rejected replacement fact")
            raise AssertionError("replacement mismatch was accepted")
        except ValueError as exc:
            result["supersede_error"] = str(exc)
        stored = await memory._store.get(original.id)
        result["original_still_active"] = stored.valid_until is None and stored.superseded_by is None
        result["vector_entries_after_rejected_supersede"] = memory._vector_index.size
        with sqlite3.connect(out / "vectors.db") as db:
            result["all_primary_rows"] = db.execute("SELECT COUNT(*) FROM memories WHERE user_id = 'alice'").fetchone()[0]
        assert result["rows_after_rejected_add"] == (0 if expected == "after" else 1), result
        assert result["original_still_active"] == (expected == "after"), result
        assert result["vector_entries_after_rejected_supersede"] == (1 if expected == "after" else 0), result
        assert result["all_primary_rows"] == (1 if expected == "after" else 3), result
        return result
    finally:
        await memory.close()


try:
    result = asyncio.run(vectors_case())
    result.update({
        "case": case, "expected": expected, "python": platform.python_version(),
        "source": str(Path(headroom.__file__).resolve()),
        "native": str(Path(headroom._core.__file__).resolve()),
        "http_requests": Endpoint.requests,
        "not_tested": "Trained model inference, paid APIs, provider answer quality, non-Windows runtime, non-local memory backends.",
    })
    (out / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8", newline="\n")
    print(json.dumps(result, indent=2))
finally:
    server.shutdown()
    server.server_close()
    worker.join(timeout=5)
```

</details>

- Observed result: Main leaves one row after the rejected add; after the
rejected replacement it has three primary rows, the original is no
longer active and its vector was removed. The patch leaves zero rows
after the rejected add, then retains exactly the valid original row and
its one vector after the rejected replacement.
- Not tested: General cross-component transactions for arbitrary
storage/index I/O failures, automatic dimension discovery or index
migration, remote vector backends, trained model inference, paid APIs,
non-Windows runtime, or the complete Python suite.

## Runtime Rollout Safety

- Rollout-managed feature(s): Existing memory behavior; no new feature
flag or channel expansion.
- Minimum rollout channel: Normal release after maintainer review and CI
qualification.
- Stable/default behavior changed: Embedding shape errors are reported
before primary writes. Matching vectors and explicit auto_embed=False
operations retain their existing behavior.
- Kill switch / disable path: Use the existing memory-disable path, or
revert this commit.
- Unsafe override required: No.
- Qualification impact: Related tests and original local gates passed;
hosted CI is tracked separately.
- Rollback path: Revert this commit; no dependency or storage-schema
change.

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have added tests that prove the fix
- [x] Relevant new and existing tests pass locally
- [x] I did **not** edit `CHANGELOG.md`

## Additional Notes

Broader regression exposed update()'s object-identity contract; the
final fix preserves it without changing the existing fixture. The
author's review covered the initial patch and evidence; this
compatibility adjustment was additionally reviewed and tested with
Codex.

Developed and validated with Codex assistance. The author reviewed the
changes and before/after evidence.

<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4831"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->

Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
2026-11-16 23:08:59 -06:00

105 lines
4.2 KiB
Python

"""Reject incompatible embeddings before changing primary or cached memories."""
from unittest.mock import AsyncMock
import numpy as np
import pytest
from headroom.memory.adapters.cache import LRUMemoryCache
from headroom.memory.adapters.hnsw import HNSWVectorIndex
from headroom.memory.adapters.sqlite import SQLiteMemoryStore
from headroom.memory.config import MemoryConfig
from headroom.memory.core import HierarchicalMemory
from headroom.memory.models import Memory
from headroom.memory.ports import MemoryFilter
@pytest.fixture
def components(tmp_path):
store = SQLiteMemoryStore(tmp_path / "memories.db")
vector = HNSWVectorIndex(dimension=4)
text = AsyncMock()
embedder = AsyncMock()
cache = LRUMemoryCache()
memory = HierarchicalMemory(
store, vector, text, embedder, cache, MemoryConfig(auto_bubble=False)
)
return memory, store, vector, text, embedder, cache
@pytest.mark.asyncio
@pytest.mark.parametrize("shape", [(8,), (2, 2), ()])
async def test_add_rejects_incompatible_shape_without_persisting(components, shape):
memory, store, vector, text, embedder, cache = components
embedder.embed.return_value = np.zeros(shape, dtype=np.float32)
with pytest.raises(ValueError):
await memory.add("Must not be half-written", user_id="alice")
assert await store.query(MemoryFilter(user_id="alice")) == []
assert vector.size == 0
text.index_memory.assert_not_called()
assert cache.size == 0
@pytest.mark.asyncio
async def test_batch_validates_every_embedding_before_saving_any_row(components):
memory, store, vector, _, embedder, _ = components
embedder.embed_batch.return_value = [np.ones(4), np.ones(8)]
with pytest.raises(ValueError):
await memory.add_batch(
[
{"content": "Valid first vector", "user_id": "alice"},
{"content": "Invalid second vector", "user_id": "alice"},
]
)
assert await store.query(MemoryFilter(user_id="alice")) == []
assert vector.size == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["update", "supersede"])
async def test_rejected_replacement_keeps_the_existing_memory_and_lineage(components, operation):
memory, store, vector, _, embedder, cache = components
embedder.embed.return_value = np.ones(4)
original = await memory.add("Original fact", user_id="alice")
embedder.embed.return_value = np.ones(8)
with pytest.raises(ValueError):
if operation == "update":
await memory.update(
original.id, content="Invalid replacement", metadata={"changed": True}
)
else:
await memory.supersede(original.id, "Invalid replacement")
stored = await store.get(original.id)
assert stored.content == "Original fact"
assert stored.metadata == {}
assert stored.valid_until is None
assert stored.superseded_by is None
assert len(await store.query(MemoryFilter(user_id="alice", include_superseded=True))) == 1
assert (await cache.get(original.id)).content == "Original fact"
assert vector.size == 1
@pytest.mark.asyncio
async def test_failed_update_does_not_mutate_a_reference_returning_store(components):
memory, _, _, _, embedder, _ = components
original = Memory(content="Original", user_id="alice", embedding=np.ones(4))
memory._store = AsyncMock()
memory._store.get.return_value = original
embedder.embed.return_value = np.ones(8)
with pytest.raises(ValueError):
await memory.update(original.id, content="Replacement", metadata={"changed": True})
assert original.content == "Original"
assert original.metadata == {}
memory._store.save.assert_not_awaited()
@pytest.mark.asyncio
async def test_matching_embeddings_and_explicit_no_embedding_still_work(components):
memory, store, vector, _, embedder, _ = components
embedder.embed.return_value = np.ones(4)
with_vector = await memory.add("Indexed fact", user_id="alice")
without_vector = await memory.add("Text-only fact", user_id="alice", auto_embed=False)
assert (await store.get(with_vector.id)).embedding.shape == (4,)
assert (await store.get(without_vector.id)).embedding is None
assert vector.size == 1