mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(memory): keep the last entry when a batch re-indexes the same id twice
Review of the first round: index_batch sent every existing id to one add_items call, so a batch that names an id twice could leave the graph on a different vector than the cached embedding and metadata if hnswlib applied the rows out of order. Collapse the updates by id, keeping the last entry, before the call.
This commit is contained in:
@@ -494,7 +494,11 @@ class HNSWVectorIndex:
|
||||
self._metadata[memory.id] = IndexedMemoryMetadata.from_memory(memory)
|
||||
self._embeddings[memory.id] = embedding.copy()
|
||||
|
||||
# Updates: add_items with an existing label replaces its vector.
|
||||
# Updates: add_items with an existing label replaces its vector. A batch
|
||||
# can name the same id more than once; keep only the last entry so the
|
||||
# graph, the cached embedding and the metadata end on the same vector
|
||||
# (hnswlib may apply rows for one label out of order across threads).
|
||||
update_memories = list({m.id: (m, emb) for m, emb in update_memories}.values())
|
||||
if update_memories:
|
||||
self._index.add_items(
|
||||
np.vstack([emb for _, emb in update_memories]).astype(np.float32),
|
||||
|
||||
@@ -88,3 +88,20 @@ async def test_index_batch_on_existing_id_replaces_the_vector() -> None:
|
||||
found, similarity = await _top(index, NEW)
|
||||
assert found == "m1"
|
||||
assert similarity == pytest.approx(1.0, abs=1e-5)
|
||||
|
||||
|
||||
async def test_index_batch_repeated_id_ends_on_the_last_vector() -> None:
|
||||
index = _index()
|
||||
await _seed(index)
|
||||
|
||||
first = Memory(
|
||||
id="m1", content="first", user_id="u", embedding=np.eye(DIM, dtype=np.float32)[2]
|
||||
)
|
||||
last = Memory(id="m1", content="last", user_id="u", embedding=NEW.copy())
|
||||
await index.index_batch([first, last])
|
||||
|
||||
assert index.size == 2
|
||||
found, similarity = await _top(index, NEW)
|
||||
assert found == "m1"
|
||||
assert similarity == pytest.approx(1.0, abs=1e-5)
|
||||
assert np.array_equal(index._embeddings["m1"], NEW)
|
||||
|
||||
Reference in New Issue
Block a user