trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-28 07:03:20 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
ab4e9148b92721137b2ef2ef635cf2f67b7ffcd4
codebase-memory-mcp/tests/test_sqlite_writer.c
T
Andrew Hundt 210d5ffe11 feat(store): add exact-delta metadata schema 
Add additive metadata tables and indexes for future staged exact-delta indexing: generations, file state, node/edge owners, symbol exports, import refs, and derived view freshness.

Keep this slice behavior-neutral. Normal store open migrates the schema, query-only open still does not create missing DBs, and writer-produced DBs remain minimal but migrate through cbm_store_open_path().

Validation: make -f Makefile.cbm build/c/test-runner; CBM_ONLY_SUITE=store_nodes ./build/c/test-runner; CBM_ONLY_SUITE=sqlite_writer ./build/c/test-runner; CBM_ONLY_SUITE=tool_consolidation ./build/c/test-runner; CBM_ONLY_SUITE=store_search ./build/c/test-runner; CBM_ONLY_SUITE=store_bulk ./build/c/test-runner; CBM_ONLY_SUITE=incremental ./build/c/test-runner; bash scripts/check-source-safety.sh; git diff --check.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>
2026-06-30 11:00:27 -04:00

920 lines
31 KiB
C
Raw Blame History

/*
* test_sqlite_writer.c — Tests for direct SQLite page writer.
*
* Ports from internal/cbm/sqlite_writer_test.go:
* TestWriteDB_MinimalData, TestWriteDB_ScaleAndIndexes, TestWriteDB_Empty
*
* The page writer (cbm_write_db) constructs B-tree pages directly,
* bypassing the SQL parser entirely. These tests verify integrity.
*/
#include "../src/foundation/compat.h"
#include "../src/foundation/compat_thread.h"
#include "../src/foundation/constants.h"
#include "test_framework.h"
#include "test_sqlite_helpers.h"
#include <store/store.h>
/* sqlite_writer.h is at internal/cbm/ — Makefile adds -Iinternal/cbm */
#include "sqlite_writer.h" /* CBMDumpNode, CBMDumpEdge, cbm_write_db */
#include "sqlite3.h" /* vendored/sqlite3/ via -Ivendored/sqlite3 */
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* ── Helper: create temp file path ─────────────────────────────── */
static int make_temp_db(char *path, size_t pathsz) {
snprintf(path, pathsz, "/tmp/cbm_sw_test_XXXXXX");
int fd = cbm_mkstemp(path);
if (fd < 0)
return -1;
close(fd);
return 0;
}
static int verify_writer_db(const char *path, const char *project, const char *root_path,
int expected_nodes, int expected_edges) {
sqlite3 *db = NULL;
if (sqlite3_open(path, &db) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW ||
strcmp((const char *)sqlite3_column_text(stmt, 0), "ok") != 0) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT root_path FROM projects WHERE name=?1", -1, &stmt, NULL) !=
SQLITE_OK) {
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, -1, SQLITE_STATIC);
if (sqlite3_step(stmt) != SQLITE_ROW ||
strcmp((const char *)sqlite3_column_text(stmt, 0), root_path) != 0) {
sqlite3_finalize(stmt);
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != expected_nodes) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != expected_edges) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
sqlite3_close(db);
return 0;
}
/* ── Tests ─────────────────────────────────────────────────────── */
TEST(sw_minimal_data) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[2] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.go",
.start_line = 1,
.end_line = 10,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Function",
.name = "hello",
.qualified_name = "test.main.hello",
.file_path = "main.go",
.start_line = 3,
.end_line = 5,
.properties = "{}"},
};
CBMDumpEdge edges[1] = {
{.id = 1,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "DEFINES",
.properties = "{}",
.url_path = ""},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 2, edges, 1,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
/* Verify via SQLite */
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
const char *integrity = (const char *)sqlite3_column_text(stmt, 0);
ASSERT_STR_EQ(integrity, "ok");
sqlite3_finalize(stmt);
/* Node count */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 2);
sqlite3_finalize(stmt);
/* Edge count */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
sqlite3_finalize(stmt);
/* Project row */
sqlite3_prepare_v2(db, "SELECT name, root_path FROM projects", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "test");
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 1), "/tmp/test");
sqlite3_finalize(stmt);
/* Node content: check node 2 */
sqlite3_prepare_v2(db, "SELECT qualified_name, label FROM nodes WHERE id=2", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "test.main.hello");
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 1), "Function");
sqlite3_finalize(stmt);
/* Edge content: check edge 1 */
sqlite3_prepare_v2(db, "SELECT source_id, target_id, type FROM edges WHERE id=1", -1, &stmt,
NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1);
ASSERT_EQ(sqlite3_column_int64(stmt, 1), 2);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 2), "DEFINES");
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
TEST(sw_store_open_migrates_exact_delta_metadata) {
char path[CBM_SZ_256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[1] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.go",
.start_line = 1,
.end_line = 1,
.properties = "{}"},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 1, NULL, 0,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *raw = NULL;
ASSERT_EQ(sqlite3_open(path, &raw), SQLITE_OK);
ASSERT_FALSE(cbm_test_sqlite_object_exists(raw, "table", "file_state"));
sqlite3_close(raw);
cbm_store_t *store = cbm_store_open_path(path);
ASSERT_NOT_NULL(store);
sqlite3 *db = cbm_store_get_db(store);
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "table", "file_state"));
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "table", "node_owners"));
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "index", "idx_node_owners_path"));
cbm_store_close(store);
unlink(path);
PASS();
}
TEST(sw_scale_and_indexes) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* 100 nodes across multiple files/labels */
CBMDumpNode nodes[100];
const char *labels[] = {"Function", "Method", "Class", "Module", "Variable"};
const char *files[] = {"alpha.go", "beta.go", "gamma.py", "delta.ts", "epsilon.rs"};
char names[100][32];
char qns[100][64];
for (int i = 0; i < 100; i++) {
snprintf(names[i], sizeof(names[i]), "sym_%03d", i);
snprintf(qns[i], sizeof(qns[i]), "proj.pkg.sym_%03d", i);
nodes[i] = (CBMDumpNode){
.id = i + 1,
.project = "proj",
.label = labels[i % 5],
.name = names[i],
.qualified_name = qns[i],
.file_path = files[i % 5],
.start_line = i * 10 + 1,
.end_line = i * 10 + 9,
.properties = "{}",
};
}
/* 200 edges with varied types — build unique (source, target, type) combos */
const char *edge_types[] = {"CALLS", "DEFINES", "IMPORTS", "IMPLEMENTS", "USES"};
CBMDumpEdge edges[200];
char eprops[200][80];
int edge_count = 0;
int64_t edge_id = 1;
/* Track seen keys via simple hash — good enough for 200 edges */
typedef struct {
int64_t s, t;
int ty;
} ekey_t;
ekey_t seen[200];
int nseen = 0;
for (int i = 0; edge_count < 200 && i < 10000; i++) {
int64_t src = (i % 100) + 1;
int64_t tgt = ((i * 7 + 3) % 100) + 1;
if (tgt == src)
tgt = (tgt % 100) + 1;
int ty_idx = i % 5;
/* Check duplicate */
int dup = 0;
for (int j = 0; j < nseen; j++) {
if (seen[j].s == src && seen[j].t == tgt && seen[j].ty == ty_idx) {
dup = 1;
break;
}
}
if (dup)
continue;
seen[nseen++] = (ekey_t){src, tgt, ty_idx};
snprintf(eprops[edge_count], sizeof(eprops[edge_count]), "{\"weight\":%d}", i);
edges[edge_count] = (CBMDumpEdge){
.id = edge_id++,
.project = "proj",
.source_id = src,
.target_id = tgt,
.type = edge_types[ty_idx],
.properties = eprops[edge_count],
.url_path = "",
};
edge_count++;
}
int rc = cbm_write_db(path, "proj", "/repo", "2026-03-14T12:00:00Z", nodes, 100, edges,
edge_count, NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* Row counts */
int nc = 0, ec = 0;
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
nc = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(nc, 100);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ec = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(ec, edge_count);
/* Index queries — exercise each index */
int cnt = 0;
/* label index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes WHERE project='proj' AND label='Function'",
-1, &stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 20);
/* name index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes WHERE project='proj' AND name='sym_042'", -1,
&stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 1);
/* file_path index */
sqlite3_prepare_v2(db,
"SELECT COUNT(*) FROM nodes WHERE project='proj' AND file_path='alpha.go'",
-1, &stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 20);
/* edge type index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges WHERE project='proj' AND type='DEFINES'", -1,
&stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_GT(cnt, 0);
/* QN unique lookup */
sqlite3_prepare_v2(
db, "SELECT id FROM nodes WHERE project='proj' AND qualified_name='proj.pkg.sym_050'", -1,
&stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 51);
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
/* Index keys longer than the index-page max-local payload (~16.4 KB at the
* 64 KB page size) MUST spill to overflow pages; writing them fully inline
* makes SQLite read key bytes as an overflow page number — PRAGMA
* integrity_check reports "invalid page number 0x43654C6C ('CeLl')" and name
* lookups silently return nothing (seen on elasticsearch: very long
* Section/heading names). */
TEST(sw_long_index_keys_overflow) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
enum { LONGN = 20000 };
char *longname = malloc(LONGN + 1);
char *longqn = malloc(LONGN + 16);
ASSERT_NOT_NULL(longname);
ASSERT_NOT_NULL(longqn);
memset(longname, 'N', LONGN);
longname[LONGN] = '\0';
snprintf(longqn, LONGN + 16, "test.%s", longname);
CBMDumpNode nodes[3] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.md",
.start_line = 1,
.end_line = 2,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Section",
.name = longname,
.qualified_name = longqn,
.file_path = "main.md",
.start_line = 3,
.end_line = 9,
.properties = "{}"},
{.id = 3,
.project = "test",
.label = "Function",
.name = "after",
.qualified_name = "test.main.after",
.file_path = "main.md",
.start_line = 10,
.end_line = 12,
.properties = "{}"},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 3, NULL, 0,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
const char *integrity = (const char *)sqlite3_column_text(stmt, 0);
ASSERT_STR_EQ(integrity, "ok");
sqlite3_finalize(stmt);
/* The long-named row must be findable via the name index. */
sqlite3_prepare_v2(db, "SELECT id FROM nodes WHERE name = ?1", -1, &stmt, NULL);
sqlite3_bind_text(stmt, 1, longname, -1, SQLITE_STATIC);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 2);
sqlite3_finalize(stmt);
sqlite3_close(db);
free(longname);
free(longqn);
unlink(path);
PASS();
}
TEST(sw_empty) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", NULL, 0, NULL, 0, NULL,
0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
TEST(sw_vectors_and_token_vectors) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[2] = {
{.id = 1,
.project = "test",
.label = "Function",
.name = "source",
.qualified_name = "test.source",
.file_path = "main.py",
.start_line = 1,
.end_line = 3,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Function",
.name = "target",
.qualified_name = "test.target",
.file_path = "main.py",
.start_line = 5,
.end_line = 8,
.properties = "{}"},
};
CBMDumpEdge edges[1] = {
{.id = 1,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "SEMANTICALLY_RELATED",
.properties = "{\"score\":0.75}",
.url_path = ""},
};
static const uint8_t node_vec[] = {1, 2, 3, 4};
static const uint8_t token_vec[] = {5, 6, 7, 8, 9};
CBMDumpVector vectors[1] = {
{.node_id = 1, .project = "test", .vector = node_vec, .vector_len = sizeof(node_vec)},
};
CBMDumpTokenVec token_vecs[1] = {
{.id = 1, .project = "test", .token = "source", .vector = token_vec,
.vector_len = sizeof(token_vec), .idf = 1.25f},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 2, edges, 1,
vectors, 1, token_vecs, 1);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT length(vector) FROM node_vectors WHERE node_id=1", -1, &stmt,
NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int(stmt, 0), (int)sizeof(node_vec));
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT token, length(vector), idf FROM token_vectors WHERE id=1", -1,
&stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "source");
ASSERT_EQ(sqlite3_column_int(stmt, 1), (int)sizeof(token_vec));
enum { TEST_IDF_FIXED_POINT = 1250 };
ASSERT_EQ(sqlite3_column_int(stmt, 2), TEST_IDF_FIXED_POINT);
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
/* --- Ported from scale_debug_test.go: TestWriteDB_MultiPage --- */
TEST(sw_multi_page) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* 192 nodes — enough to trigger multi-page B-tree */
int N = 192;
CBMDumpNode nodes[192];
char node_names[192][16];
char node_qns[192][32];
char node_files[192][32];
for (int i = 0; i < N; i++) {
snprintf(node_names[i], sizeof(node_names[i]), "f%04d", i);
snprintf(node_qns[i], sizeof(node_qns[i]), "p.f%04d", i);
snprintf(node_files[i], sizeof(node_files[i]), "pkg%d/file.go", i % 10);
nodes[i] = (CBMDumpNode){
.id = (int64_t)(i + 1),
.project = "p",
.label = "Function",
.name = node_names[i],
.qualified_name = node_qns[i],
.file_path = node_files[i],
.start_line = i,
.end_line = i + 1,
.properties = "{}",
};
}
int rc =
cbm_write_db(path, "p", "/r", "2026-01-01T00:00:00Z", nodes, N, NULL, 0, NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* COUNT(*) must be exactly N */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), N);
sqlite3_finalize(stmt);
/* Verify no rowid gaps: min=1, max=N, count=N */
sqlite3_prepare_v2(db, "SELECT MIN(rowid), MAX(rowid), COUNT(DISTINCT rowid) FROM nodes", -1,
&stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
ASSERT_EQ(sqlite3_column_int(stmt, 1), N);
ASSERT_EQ(sqlite3_column_int(stmt, 2), N);
sqlite3_finalize(stmt);
/* Check first and last node by rowid */
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE rowid=1", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "f0000");
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE rowid=192", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "f0191");
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
/* ── Oversized node: properties JSON > 65KB triggers overflow pages ─ */
TEST(sw_oversized_node) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* Build a properties JSON string that exceeds max_local (65501 bytes).
* Use 70000 bytes of padding inside the JSON value so the full record,
* which includes other text columns, is well above the threshold. */
int prop_len = 70000;
char *big_props = (char *)malloc(prop_len + 1);
ASSERT_NOT_NULL(big_props);
memset(big_props, 'x', prop_len);
big_props[0] = '"';
big_props[prop_len - 1] = '"';
big_props[prop_len] = '\0';
CBMDumpNode nodes[1] = {{
.id = 1,
.project = "test",
.label = "Function",
.name = "huge_fn",
.qualified_name = "test.huge_fn",
.file_path = "huge.go",
.start_line = 1,
.end_line = 9999,
.properties = big_props,
}};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-28T00:00:00Z", nodes, 1, NULL, 0,
NULL, 0, NULL, 0);
free(big_props);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check — SQLite will validate overflow page chain */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* Verify we can read the node back */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
sqlite3_finalize(stmt);
/* Verify the name round-trips correctly */
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE id=1", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "huge_fn");
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
/* ── Suite ─────────────────────────────────────────────────────── */
/* B1 (#8) repro probe: the full pipeline intermittently stores a NUMERIC
* root_path + wildly varying edge counts (43K vs 275K) on the ~16K-node
* fastapi repo, while sw_minimal_data (tiny) round-trips cleanly. This test
* writes a comparable-scale DB in ISOLATION (no pipeline/parallelism) and
* verifies root_path round-trips exactly + integrity_check stays "ok". If this
* fails, the writer itself corrupts at scale (directly debuggable); if it
* passes, the B1 corruption is pipeline/parallel-side, not the writer. */
TEST(sw_scale_root_path_integrity) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
const int N = 20000;
const int E = 200000;
CBMDumpNode *nodes = (CBMDumpNode *)calloc((size_t)N, sizeof(CBMDumpNode));
CBMDumpEdge *edges = (CBMDumpEdge *)calloc((size_t)E, sizeof(CBMDumpEdge));
char (*namebuf)[32] = malloc((size_t)N * 32);
char (*qnbuf)[64] = malloc((size_t)N * 64);
char (*filebuf)[48] = malloc((size_t)N * 48);
char (*propsbuf)[2048] = malloc((size_t)N * 2048);
ASSERT_NOT_NULL(nodes);
ASSERT_NOT_NULL(edges);
ASSERT_NOT_NULL(namebuf);
ASSERT_NOT_NULL(qnbuf);
ASSERT_NOT_NULL(filebuf);
ASSERT_NOT_NULL(propsbuf);
for (int i = 0; i < N; i++) {
snprintf(namebuf[i], 32, "fn_%d", i);
snprintf(qnbuf[i], 64, "proj.mod.fn_%d", i);
snprintf(filebuf[i], 48, "src/file_%d.py", i % 400);
nodes[i].id = i + 1;
nodes[i].project = "proj";
nodes[i].label = "Function";
nodes[i].name = namebuf[i];
nodes[i].qualified_name = qnbuf[i];
nodes[i].file_path = filebuf[i];
nodes[i].start_line = i + 1;
nodes[i].end_line = i + 2;
/* Variable-length properties (mirrors real data) to stress page
* boundaries in the writer — the B1 trigger hypothesis (uniform
* records never cross boundaries the way real variable records do). */
static const int plens[] = {20, 200, 800, 1500, 50, 400, 1000, 100};
int padlen = plens[i % 8] - 8; /* {"k":""} overhead */
if (padlen < 0) padlen = 0;
if (padlen > 2040) padlen = 2040;
propsbuf[i][0] = '{';
propsbuf[i][1] = '"';
propsbuf[i][2] = 'k';
propsbuf[i][3] = '"';
propsbuf[i][4] = ':';
propsbuf[i][5] = '"';
memset(propsbuf[i] + 6, 'y', (size_t)padlen);
propsbuf[i][6 + padlen] = '"';
propsbuf[i][6 + padlen + 1] = '}';
propsbuf[i][6 + padlen + 2] = '\0';
nodes[i].properties = propsbuf[i];
}
for (int i = 0; i < E; i++) {
edges[i].id = i + 1;
edges[i].project = "proj";
/* edges has UNIQUE(source_id, target_id, type) — generate distinct
* (source,target) pairs so the test exercises the writer, not the
* constraint: source cycles 1..N, target = block (i/N), giving E unique
* pairs for E <= N*N. */
edges[i].source_id = (i % N) + 1;
edges[i].target_id = ((i / N) % N) + 1;
edges[i].type = "CALLS";
edges[i].properties = "{}";
edges[i].url_path = "";
}
const char *ROOT = "/tmp/scale_root_path_test";
int rc = cbm_write_db(path, "proj", ROOT, "2026-06-25T00:00:00Z", nodes, N, edges, E, NULL, 0,
NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* root_path MUST round-trip exactly — B1 reproduces as a numeric value. */
sqlite3_prepare_v2(db, "SELECT root_path FROM projects", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), ROOT);
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), N);
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), E);
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
free(nodes);
free(edges);
free(namebuf);
free(qnbuf);
free(filebuf);
free(propsbuf);
PASS();
}
typedef struct {
char path[CBM_PATH_MAX];
char project[CBM_SZ_32];
char root_path[CBM_PATH_MAX];
atomic_int *start;
int node_count;
int edge_count;
int rc;
} sw_concurrent_job_t;
static void *sw_concurrent_writer_thread(void *arg) {
sw_concurrent_job_t *job = (sw_concurrent_job_t *)arg;
while (atomic_load(job->start) == 0) {
}
CBMDumpNode *nodes = (CBMDumpNode *)calloc((size_t)job->node_count, sizeof(*nodes));
CBMDumpEdge *edges = (CBMDumpEdge *)calloc((size_t)job->edge_count, sizeof(*edges));
char (*names)[CBM_SZ_32] = malloc((size_t)job->node_count * CBM_SZ_32);
char (*qns)[CBM_SZ_64] = malloc((size_t)job->node_count * CBM_SZ_64);
char (*files)[CBM_SZ_32] = malloc((size_t)job->node_count * CBM_SZ_32);
if (!nodes || !edges || !names || !qns || !files) {
free(nodes);
free(edges);
free(names);
free(qns);
free(files);
job->rc = CBM_NOT_FOUND;
return NULL;
}
for (int i = 0; i < job->node_count; i++) {
snprintf(names[i], CBM_SZ_32, "fn_%04d", i);
snprintf(qns[i], CBM_SZ_64, "%s.mod.fn_%04d", job->project, i);
snprintf(files[i], CBM_SZ_32, "src/file_%03d.py", i % CBM_SZ_128);
nodes[i].id = i + 1;
nodes[i].project = job->project;
nodes[i].label = (i % PAIR_LEN) == 0 ? "Function" : "Class";
nodes[i].name = names[i];
nodes[i].qualified_name = qns[i];
nodes[i].file_path = files[i];
nodes[i].start_line = i + SKIP_ONE;
nodes[i].end_line = i + PAIR_LEN;
nodes[i].properties = "{}";
}
for (int i = 0; i < job->edge_count; i++) {
edges[i].id = i + 1;
edges[i].project = job->project;
edges[i].source_id = (i % job->node_count) + 1;
edges[i].target_id = ((i / job->node_count) % job->node_count) + 1;
edges[i].type = "CALLS";
edges[i].properties = "{}";
edges[i].url_path = "";
}
job->rc = cbm_write_db(job->path, job->project, job->root_path, "2026-06-30T00:00:00Z",
nodes, job->node_count, edges, job->edge_count, NULL, 0, NULL, 0);
free(nodes);
free(edges);
free(names);
free(qns);
free(files);
return NULL;
}
TEST(sw_concurrent_writes_are_independent) {
enum {
CONCURRENT_WRITERS = PAIR_LEN,
CONCURRENT_NODES = CBM_SZ_1K,
CONCURRENT_EDGES = CBM_SZ_4K,
};
atomic_int start = 0;
sw_concurrent_job_t jobs[CONCURRENT_WRITERS] = {0};
cbm_thread_t threads[CONCURRENT_WRITERS];
for (int i = 0; i < CONCURRENT_WRITERS; i++) {
ASSERT_EQ(make_temp_db(jobs[i].path, sizeof(jobs[i].path)), 0);
snprintf(jobs[i].project, sizeof(jobs[i].project), "proj%d", i);
snprintf(jobs[i].root_path, sizeof(jobs[i].root_path), "/tmp/sw_concurrent_root_%d", i);
jobs[i].start = &start;
jobs[i].node_count = CONCURRENT_NODES;
jobs[i].edge_count = CONCURRENT_EDGES;
jobs[i].rc = CBM_NOT_FOUND;
ASSERT_EQ(cbm_thread_create(&threads[i], 0, sw_concurrent_writer_thread, &jobs[i]), 0);
}
atomic_store(&start, CBM_INIT_DONE);
for (int i = 0; i < CONCURRENT_WRITERS; i++) {
ASSERT_EQ(cbm_thread_join(&threads[i]), 0);
ASSERT_EQ(jobs[i].rc, 0);
ASSERT_EQ(verify_writer_db(jobs[i].path, jobs[i].project, jobs[i].root_path,
jobs[i].node_count, jobs[i].edge_count),
0);
unlink(jobs[i].path);
}
PASS();
}
SUITE(sqlite_writer) {
RUN_TEST(sw_minimal_data);
RUN_TEST(sw_store_open_migrates_exact_delta_metadata);
RUN_TEST(sw_scale_and_indexes);
RUN_TEST(sw_long_index_keys_overflow);
RUN_TEST(sw_empty);
RUN_TEST(sw_vectors_and_token_vectors);
RUN_TEST(sw_multi_page);
RUN_TEST(sw_oversized_node);
RUN_TEST(sw_scale_root_path_integrity);
RUN_TEST(sw_concurrent_writes_are_independent);
}
Reference in New Issue View Git Blame Copy Permalink