trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-28 07:03:20 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
09c0762938c24d71ac00aeb12de0f3d5c2faa1ec
codebase-memory-mcp/tests/test_sqlite_writer.c
T
Andrew Hundt 59def6aaeb merge(upstream): integrate main through Laravel route fixes 
Merge upstream/main at 2469ecc3 into integrate-upstream-split-search-tools while retaining branch-specific incremental indexing, MCP API consolidation, and graph extraction behavior.

The staged tree adds upstream's Mojo and Perl grammar support, issue reproduction suites, Graph UI changes, Windows smoke coverage, TOON graph output, subprocess supervision, and Laravel facade route handling. Conflict resolutions in src/pipeline/httplink.c and pass_httplinks.c retain dynamic growth and IMPORTS edge identity; scripts/smoke-test.sh and scripts/soak-test.sh retain hermetic cache isolation.

Validation against the merge plus the follow-up working-tree reconciliation: ASan/UBSan MCP suite 216/216 passed. Follow-up semantic and concurrency fixes remain intentionally unstaged for the next focused commit.
2026-07-14 01:47:43 -04:00

1012 lines
34 KiB
C
Raw Blame History

/*
* test_sqlite_writer.c — Tests for direct SQLite page writer.
*
* Ports from internal/cbm/sqlite_writer_test.go:
* TestWriteDB_MinimalData, TestWriteDB_ScaleAndIndexes, TestWriteDB_Empty
*
* The page writer (cbm_write_db) constructs B-tree pages directly,
* bypassing the SQL parser entirely. These tests verify integrity.
*/
#include "../src/foundation/compat.h"
#include "../src/foundation/compat_fs.h"
#include "../src/foundation/compat_thread.h"
#include "../src/foundation/constants.h"
#include "test_framework.h"
#include "test_sqlite_helpers.h"
#include <store/store.h>
/* sqlite_writer.h is at internal/cbm/ — Makefile adds -Iinternal/cbm */
#include "sqlite_writer.h" /* CBMDumpNode, CBMDumpEdge, cbm_write_db */
#include "sqlite3.h" /* vendored/sqlite3/ via -Ivendored/sqlite3 */
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* ── Helper: create temp file path ─────────────────────────────── */
static int make_temp_db(char *path, size_t pathsz) {
snprintf(path, pathsz, "/tmp/cbm_sw_test_XXXXXX");
int fd = cbm_mkstemp(path);
if (fd < 0)
return -1;
close(fd);
return 0;
}
static int verify_writer_db(const char *path, const char *project, const char *root_path,
int expected_nodes, int expected_edges) {
sqlite3 *db = NULL;
if (sqlite3_open(path, &db) != SQLITE_OK) {
return CBM_NOT_FOUND;
}
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW ||
strcmp((const char *)sqlite3_column_text(stmt, 0), "ok") != 0) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT root_path FROM projects WHERE name=?1", -1, &stmt, NULL) !=
SQLITE_OK) {
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_bind_text(stmt, 1, project, -1, SQLITE_STATIC);
if (sqlite3_step(stmt) != SQLITE_ROW ||
strcmp((const char *)sqlite3_column_text(stmt, 0), root_path) != 0) {
sqlite3_finalize(stmt);
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != expected_nodes) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
stmt = NULL;
if (sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL) != SQLITE_OK ||
sqlite3_step(stmt) != SQLITE_ROW || sqlite3_column_int(stmt, 0) != expected_edges) {
if (stmt) {
sqlite3_finalize(stmt);
}
sqlite3_close(db);
return CBM_NOT_FOUND;
}
sqlite3_finalize(stmt);
sqlite3_close(db);
return 0;
}
/* ── Tests ─────────────────────────────────────────────────────── */
TEST(sw_minimal_data) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[2] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.go",
.start_line = 1,
.end_line = 10,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Function",
.name = "hello",
.qualified_name = "test.main.hello",
.file_path = "main.go",
.start_line = 3,
.end_line = 5,
.properties = "{}"},
};
CBMDumpEdge edges[1] = {
{.id = 1,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "DEFINES",
.properties = "{}",
.url_path = ""},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 2, edges, 1,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
/* Verify via SQLite */
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
const char *integrity = (const char *)sqlite3_column_text(stmt, 0);
ASSERT_STR_EQ(integrity, "ok");
sqlite3_finalize(stmt);
/* Node count */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 2);
sqlite3_finalize(stmt);
/* Edge count */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
sqlite3_finalize(stmt);
/* Project row */
sqlite3_prepare_v2(db, "SELECT name, root_path FROM projects", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "test");
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 1), "/tmp/test");
sqlite3_finalize(stmt);
/* Node content: check node 2 */
sqlite3_prepare_v2(db, "SELECT qualified_name, label FROM nodes WHERE id=2", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "test.main.hello");
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 1), "Function");
sqlite3_finalize(stmt);
/* Edge content: check edge 1 */
sqlite3_prepare_v2(db, "SELECT source_id, target_id, type FROM edges WHERE id=1", -1, &stmt,
NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1);
ASSERT_EQ(sqlite3_column_int64(stmt, 1), 2);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 2), "DEFINES");
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
TEST(sw_store_open_migrates_exact_delta_metadata) {
char path[CBM_SZ_256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[1] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.go",
.start_line = 1,
.end_line = 1,
.properties = "{}"},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 1, NULL, 0,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *raw = NULL;
ASSERT_EQ(sqlite3_open(path, &raw), SQLITE_OK);
ASSERT_FALSE(cbm_test_sqlite_object_exists(raw, "table", "file_state"));
sqlite3_close(raw);
cbm_store_t *store = cbm_store_open_path(path);
ASSERT_NOT_NULL(store);
sqlite3 *db = cbm_store_get_db(store);
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "table", "file_state"));
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "table", "node_owners"));
ASSERT_TRUE(cbm_test_sqlite_object_exists(db, "index", "idx_node_owners_path"));
cbm_store_close(store);
cbm_unlink(path);
PASS();
}
/* #768 schema half: two IMPORTS edges between the same two files, keyed apart
* by local_name, must be representable in the dumped DB. The dump's edges DDL
* and the hand-built sqlite_autoindex_edges_1 must carry the local_name
* discriminator — with the old UNIQUE(source_id,target_id,type) the two rows
* violate the dump's own constraint and PRAGMA integrity_check flags a
* non-unique autoindex entry. */
TEST(sw_imports_local_name_unique) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[2] = {
{.id = 1,
.project = "test",
.label = "File",
.name = "consumer.ts",
.qualified_name = "test.consumer.ts.__file__",
.file_path = "consumer.ts",
.start_line = 1,
.end_line = 3,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "File",
.name = "lib.ts",
.qualified_name = "test.lib.ts.__file__",
.file_path = "lib.ts",
.start_line = 1,
.end_line = 2,
.properties = "{}"},
};
CBMDumpEdge edges[2] = {
{.id = 1,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "IMPORTS",
.properties = "{\"local_name\":\"A\"}",
.url_path = "",
.local_name = "A"},
{.id = 2,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "IMPORTS",
.properties = "{\"local_name\":\"B\"}",
.url_path = "",
.local_name = "B"},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 2, edges, 2,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* The dumped DB must satisfy its own UNIQUE constraint. */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
const char *integrity = (const char *)sqlite3_column_text(stmt, 0);
ASSERT_STR_EQ(integrity, "ok");
sqlite3_finalize(stmt);
/* Both sibling imports are queryable. */
sqlite3_prepare_v2(db,
"SELECT COUNT(*) FROM edges "
"WHERE source_id=1 AND target_id=2 AND type='IMPORTS'",
-1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 2);
sqlite3_finalize(stmt);
/* The generated discriminator matches what the writer hand-indexed. */
sqlite3_prepare_v2(db, "SELECT local_name_gen FROM edges ORDER BY id", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "A");
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "B");
sqlite3_finalize(stmt);
sqlite3_close(db);
unlink(path);
PASS();
}
TEST(sw_scale_and_indexes) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* 100 nodes across multiple files/labels */
CBMDumpNode nodes[100];
const char *labels[] = {"Function", "Method", "Class", "Module", "Variable"};
const char *files[] = {"alpha.go", "beta.go", "gamma.py", "delta.ts", "epsilon.rs"};
char names[100][32];
char qns[100][64];
for (int i = 0; i < 100; i++) {
snprintf(names[i], sizeof(names[i]), "sym_%03d", i);
snprintf(qns[i], sizeof(qns[i]), "proj.pkg.sym_%03d", i);
nodes[i] = (CBMDumpNode){
.id = i + 1,
.project = "proj",
.label = labels[i % 5],
.name = names[i],
.qualified_name = qns[i],
.file_path = files[i % 5],
.start_line = i * 10 + 1,
.end_line = i * 10 + 9,
.properties = "{}",
};
}
/* 200 edges with varied types — build unique (source, target, type) combos */
const char *edge_types[] = {"CALLS", "DEFINES", "IMPORTS", "IMPLEMENTS", "USES"};
CBMDumpEdge edges[200];
char eprops[200][80];
int edge_count = 0;
int64_t edge_id = 1;
/* Track seen keys via simple hash — good enough for 200 edges */
typedef struct {
int64_t s, t;
int ty;
} ekey_t;
ekey_t seen[200];
int nseen = 0;
for (int i = 0; edge_count < 200 && i < 10000; i++) {
int64_t src = (i % 100) + 1;
int64_t tgt = ((i * 7 + 3) % 100) + 1;
if (tgt == src)
tgt = (tgt % 100) + 1;
int ty_idx = i % 5;
/* Check duplicate */
int dup = 0;
for (int j = 0; j < nseen; j++) {
if (seen[j].s == src && seen[j].t == tgt && seen[j].ty == ty_idx) {
dup = 1;
break;
}
}
if (dup)
continue;
seen[nseen++] = (ekey_t){src, tgt, ty_idx};
snprintf(eprops[edge_count], sizeof(eprops[edge_count]), "{\"weight\":%d}", i);
edges[edge_count] = (CBMDumpEdge){
.id = edge_id++,
.project = "proj",
.source_id = src,
.target_id = tgt,
.type = edge_types[ty_idx],
.properties = eprops[edge_count],
.url_path = "",
};
edge_count++;
}
int rc = cbm_write_db(path, "proj", "/repo", "2026-03-14T12:00:00Z", nodes, 100, edges,
edge_count, NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* Row counts */
int nc = 0, ec = 0;
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
nc = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(nc, 100);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ec = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(ec, edge_count);
/* Index queries — exercise each index */
int cnt = 0;
/* label index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes WHERE project='proj' AND label='Function'",
-1, &stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 20);
/* name index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes WHERE project='proj' AND name='sym_042'", -1,
&stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 1);
/* file_path index */
sqlite3_prepare_v2(db,
"SELECT COUNT(*) FROM nodes WHERE project='proj' AND file_path='alpha.go'",
-1, &stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_EQ(cnt, 20);
/* edge type index */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges WHERE project='proj' AND type='DEFINES'", -1,
&stmt, NULL);
sqlite3_step(stmt);
cnt = sqlite3_column_int(stmt, 0);
sqlite3_finalize(stmt);
ASSERT_GT(cnt, 0);
/* QN unique lookup */
sqlite3_prepare_v2(
db, "SELECT id FROM nodes WHERE project='proj' AND qualified_name='proj.pkg.sym_050'", -1,
&stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 51);
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
/* Index keys longer than the index-page max-local payload (~16.4 KB at the
* 64 KB page size) MUST spill to overflow pages; writing them fully inline
* makes SQLite read key bytes as an overflow page number — PRAGMA
* integrity_check reports "invalid page number 0x43654C6C ('CeLl')" and name
* lookups silently return nothing (seen on elasticsearch: very long
* Section/heading names). */
TEST(sw_long_index_keys_overflow) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
enum { LONGN = 20000 };
char *longname = malloc(LONGN + 1);
char *longqn = malloc(LONGN + 16);
ASSERT_NOT_NULL(longname);
ASSERT_NOT_NULL(longqn);
memset(longname, 'N', LONGN);
longname[LONGN] = '\0';
snprintf(longqn, LONGN + 16, "test.%s", longname);
CBMDumpNode nodes[3] = {
{.id = 1,
.project = "test",
.label = "Module",
.name = "main",
.qualified_name = "test.main",
.file_path = "main.md",
.start_line = 1,
.end_line = 2,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Section",
.name = longname,
.qualified_name = longqn,
.file_path = "main.md",
.start_line = 3,
.end_line = 9,
.properties = "{}"},
{.id = 3,
.project = "test",
.label = "Function",
.name = "after",
.qualified_name = "test.main.after",
.file_path = "main.md",
.start_line = 10,
.end_line = 12,
.properties = "{}"},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 3, NULL, 0,
NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
const char *integrity = (const char *)sqlite3_column_text(stmt, 0);
ASSERT_STR_EQ(integrity, "ok");
sqlite3_finalize(stmt);
/* The long-named row must be findable via the name index. */
sqlite3_prepare_v2(db, "SELECT id FROM nodes WHERE name = ?1", -1, &stmt, NULL);
sqlite3_bind_text(stmt, 1, longname, -1, SQLITE_STATIC);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int64(stmt, 0), 2);
sqlite3_finalize(stmt);
sqlite3_close(db);
free(longname);
free(longqn);
cbm_unlink(path);
PASS();
}
TEST(sw_empty) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", NULL, 0, NULL, 0, NULL,
0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
TEST(sw_vectors_and_token_vectors) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
CBMDumpNode nodes[2] = {
{.id = 1,
.project = "test",
.label = "Function",
.name = "source",
.qualified_name = "test.source",
.file_path = "main.py",
.start_line = 1,
.end_line = 3,
.properties = "{}"},
{.id = 2,
.project = "test",
.label = "Function",
.name = "target",
.qualified_name = "test.target",
.file_path = "main.py",
.start_line = 5,
.end_line = 8,
.properties = "{}"},
};
CBMDumpEdge edges[1] = {
{.id = 1,
.project = "test",
.source_id = 1,
.target_id = 2,
.type = "SEMANTICALLY_RELATED",
.properties = "{\"score\":0.75}",
.url_path = ""},
};
static const uint8_t node_vec[] = {1, 2, 3, 4};
static const uint8_t token_vec[] = {5, 6, 7, 8, 9};
CBMDumpVector vectors[1] = {
{.node_id = 1, .project = "test", .vector = node_vec, .vector_len = sizeof(node_vec)},
};
CBMDumpTokenVec token_vecs[1] = {
{.id = 1, .project = "test", .token = "source", .vector = token_vec,
.vector_len = sizeof(token_vec), .idf = 1.25f},
};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-14T00:00:00Z", nodes, 2, edges, 1,
vectors, 1, token_vecs, 1);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT length(vector) FROM node_vectors WHERE node_id=1", -1, &stmt,
NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_EQ(sqlite3_column_int(stmt, 0), (int)sizeof(node_vec));
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT token, length(vector), idf FROM token_vectors WHERE id=1", -1,
&stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "source");
ASSERT_EQ(sqlite3_column_int(stmt, 1), (int)sizeof(token_vec));
enum { TEST_IDF_FIXED_POINT = 1250 };
ASSERT_EQ(sqlite3_column_int(stmt, 2), TEST_IDF_FIXED_POINT);
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
/* --- Ported from scale_debug_test.go: TestWriteDB_MultiPage --- */
TEST(sw_multi_page) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* 192 nodes — enough to trigger multi-page B-tree */
int N = 192;
CBMDumpNode nodes[192];
char node_names[192][16];
char node_qns[192][32];
char node_files[192][32];
for (int i = 0; i < N; i++) {
snprintf(node_names[i], sizeof(node_names[i]), "f%04d", i);
snprintf(node_qns[i], sizeof(node_qns[i]), "p.f%04d", i);
snprintf(node_files[i], sizeof(node_files[i]), "pkg%d/file.go", i % 10);
nodes[i] = (CBMDumpNode){
.id = (int64_t)(i + 1),
.project = "p",
.label = "Function",
.name = node_names[i],
.qualified_name = node_qns[i],
.file_path = node_files[i],
.start_line = i,
.end_line = i + 1,
.properties = "{}",
};
}
int rc =
cbm_write_db(path, "p", "/r", "2026-01-01T00:00:00Z", nodes, N, NULL, 0, NULL, 0, NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* COUNT(*) must be exactly N */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), N);
sqlite3_finalize(stmt);
/* Verify no rowid gaps: min=1, max=N, count=N */
sqlite3_prepare_v2(db, "SELECT MIN(rowid), MAX(rowid), COUNT(DISTINCT rowid) FROM nodes", -1,
&stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
ASSERT_EQ(sqlite3_column_int(stmt, 1), N);
ASSERT_EQ(sqlite3_column_int(stmt, 2), N);
sqlite3_finalize(stmt);
/* Check first and last node by rowid */
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE rowid=1", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "f0000");
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE rowid=192", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "f0191");
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
/* ── Oversized node: properties JSON > 65KB triggers overflow pages ─ */
TEST(sw_oversized_node) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
/* Build a properties JSON string that exceeds max_local (65501 bytes).
* Use 70000 bytes of padding inside the JSON value so the full record,
* which includes other text columns, is well above the threshold. */
int prop_len = 70000;
char *big_props = (char *)malloc(prop_len + 1);
ASSERT_NOT_NULL(big_props);
memset(big_props, 'x', prop_len);
big_props[0] = '"';
big_props[prop_len - 1] = '"';
big_props[prop_len] = '\0';
CBMDumpNode nodes[1] = {{
.id = 1,
.project = "test",
.label = "Function",
.name = "huge_fn",
.qualified_name = "test.huge_fn",
.file_path = "huge.go",
.start_line = 1,
.end_line = 9999,
.properties = big_props,
}};
int rc = cbm_write_db(path, "test", "/tmp/test", "2026-03-28T00:00:00Z", nodes, 1, NULL, 0,
NULL, 0, NULL, 0);
free(big_props);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
rc = sqlite3_open(path, &db);
ASSERT_EQ(rc, SQLITE_OK);
/* Integrity check — SQLite will validate overflow page chain */
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* Verify we can read the node back */
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), 1);
sqlite3_finalize(stmt);
/* Verify the name round-trips correctly */
sqlite3_prepare_v2(db, "SELECT name FROM nodes WHERE id=1", -1, &stmt, NULL);
rc = sqlite3_step(stmt);
ASSERT_EQ(rc, SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "huge_fn");
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
PASS();
}
/* ── Suite ─────────────────────────────────────────────────────── */
/* B1 (#8) repro probe: the full pipeline intermittently stores a NUMERIC
* root_path + wildly varying edge counts (43K vs 275K) on the ~16K-node
* fastapi repo, while sw_minimal_data (tiny) round-trips cleanly. This test
* writes a comparable-scale DB in ISOLATION (no pipeline/parallelism) and
* verifies root_path round-trips exactly + integrity_check stays "ok". If this
* fails, the writer itself corrupts at scale (directly debuggable); if it
* passes, the B1 corruption is pipeline/parallel-side, not the writer. */
TEST(sw_scale_root_path_integrity) {
char path[256];
ASSERT_EQ(make_temp_db(path, sizeof(path)), 0);
const int N = 20000;
const int E = 200000;
CBMDumpNode *nodes = (CBMDumpNode *)calloc((size_t)N, sizeof(CBMDumpNode));
CBMDumpEdge *edges = (CBMDumpEdge *)calloc((size_t)E, sizeof(CBMDumpEdge));
char (*namebuf)[32] = malloc((size_t)N * 32);
char (*qnbuf)[64] = malloc((size_t)N * 64);
char (*filebuf)[48] = malloc((size_t)N * 48);
char (*propsbuf)[2048] = malloc((size_t)N * 2048);
ASSERT_NOT_NULL(nodes);
ASSERT_NOT_NULL(edges);
ASSERT_NOT_NULL(namebuf);
ASSERT_NOT_NULL(qnbuf);
ASSERT_NOT_NULL(filebuf);
ASSERT_NOT_NULL(propsbuf);
for (int i = 0; i < N; i++) {
snprintf(namebuf[i], 32, "fn_%d", i);
snprintf(qnbuf[i], 64, "proj.mod.fn_%d", i);
snprintf(filebuf[i], 48, "src/file_%d.py", i % 400);
nodes[i].id = i + 1;
nodes[i].project = "proj";
nodes[i].label = "Function";
nodes[i].name = namebuf[i];
nodes[i].qualified_name = qnbuf[i];
nodes[i].file_path = filebuf[i];
nodes[i].start_line = i + 1;
nodes[i].end_line = i + 2;
/* Variable-length properties (mirrors real data) to stress page
* boundaries in the writer — the B1 trigger hypothesis (uniform
* records never cross boundaries the way real variable records do). */
static const int plens[] = {20, 200, 800, 1500, 50, 400, 1000, 100};
int padlen = plens[i % 8] - 8; /* {"k":""} overhead */
if (padlen < 0) padlen = 0;
if (padlen > 2040) padlen = 2040;
propsbuf[i][0] = '{';
propsbuf[i][1] = '"';
propsbuf[i][2] = 'k';
propsbuf[i][3] = '"';
propsbuf[i][4] = ':';
propsbuf[i][5] = '"';
memset(propsbuf[i] + 6, 'y', (size_t)padlen);
propsbuf[i][6 + padlen] = '"';
propsbuf[i][6 + padlen + 1] = '}';
propsbuf[i][6 + padlen + 2] = '\0';
nodes[i].properties = propsbuf[i];
}
for (int i = 0; i < E; i++) {
edges[i].id = i + 1;
edges[i].project = "proj";
/* edges has UNIQUE(source_id, target_id, type) — generate distinct
* (source,target) pairs so the test exercises the writer, not the
* constraint: source cycles 1..N, target = block (i/N), giving E unique
* pairs for E <= N*N. */
edges[i].source_id = (i % N) + 1;
edges[i].target_id = ((i / N) % N) + 1;
edges[i].type = "CALLS";
edges[i].properties = "{}";
edges[i].url_path = "";
}
const char *ROOT = "/tmp/scale_root_path_test";
int rc = cbm_write_db(path, "proj", ROOT, "2026-06-25T00:00:00Z", nodes, N, edges, E, NULL, 0,
NULL, 0);
ASSERT_EQ(rc, 0);
sqlite3 *db = NULL;
ASSERT_EQ(sqlite3_open(path, &db), SQLITE_OK);
sqlite3_stmt *stmt = NULL;
sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), "ok");
sqlite3_finalize(stmt);
/* root_path MUST round-trip exactly — B1 reproduces as a numeric value. */
sqlite3_prepare_v2(db, "SELECT root_path FROM projects", -1, &stmt, NULL);
ASSERT_EQ(sqlite3_step(stmt), SQLITE_ROW);
ASSERT_STR_EQ((const char *)sqlite3_column_text(stmt, 0), ROOT);
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM nodes", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), N);
sqlite3_finalize(stmt);
sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM edges", -1, &stmt, NULL);
sqlite3_step(stmt);
ASSERT_EQ(sqlite3_column_int(stmt, 0), E);
sqlite3_finalize(stmt);
sqlite3_close(db);
cbm_unlink(path);
free(nodes);
free(edges);
free(namebuf);
free(qnbuf);
free(filebuf);
free(propsbuf);
PASS();
}
typedef struct {
char path[CBM_PATH_MAX];
char project[CBM_SZ_32];
char root_path[CBM_PATH_MAX];
atomic_int *start;
int node_count;
int edge_count;
int rc;
} sw_concurrent_job_t;
static void *sw_concurrent_writer_thread(void *arg) {
sw_concurrent_job_t *job = (sw_concurrent_job_t *)arg;
while (atomic_load(job->start) == 0) {
}
CBMDumpNode *nodes = (CBMDumpNode *)calloc((size_t)job->node_count, sizeof(*nodes));
CBMDumpEdge *edges = (CBMDumpEdge *)calloc((size_t)job->edge_count, sizeof(*edges));
char (*names)[CBM_SZ_32] = malloc((size_t)job->node_count * CBM_SZ_32);
char (*qns)[CBM_SZ_64] = malloc((size_t)job->node_count * CBM_SZ_64);
char (*files)[CBM_SZ_32] = malloc((size_t)job->node_count * CBM_SZ_32);
if (!nodes || !edges || !names || !qns || !files) {
free(nodes);
free(edges);
free(names);
free(qns);
free(files);
job->rc = CBM_NOT_FOUND;
return NULL;
}
for (int i = 0; i < job->node_count; i++) {
snprintf(names[i], CBM_SZ_32, "fn_%04d", i);
snprintf(qns[i], CBM_SZ_64, "%s.mod.fn_%04d", job->project, i);
snprintf(files[i], CBM_SZ_32, "src/file_%03d.py", i % CBM_SZ_128);
nodes[i].id = i + 1;
nodes[i].project = job->project;
nodes[i].label = (i % PAIR_LEN) == 0 ? "Function" : "Class";
nodes[i].name = names[i];
nodes[i].qualified_name = qns[i];
nodes[i].file_path = files[i];
nodes[i].start_line = i + SKIP_ONE;
nodes[i].end_line = i + PAIR_LEN;
nodes[i].properties = "{}";
}
for (int i = 0; i < job->edge_count; i++) {
edges[i].id = i + 1;
edges[i].project = job->project;
edges[i].source_id = (i % job->node_count) + 1;
edges[i].target_id = ((i / job->node_count) % job->node_count) + 1;
edges[i].type = "CALLS";
edges[i].properties = "{}";
edges[i].url_path = "";
}
job->rc = cbm_write_db(job->path, job->project, job->root_path, "2026-06-30T00:00:00Z",
nodes, job->node_count, edges, job->edge_count, NULL, 0, NULL, 0);
free(nodes);
free(edges);
free(names);
free(qns);
free(files);
return NULL;
}
TEST(sw_concurrent_writes_are_independent) {
enum {
CONCURRENT_WRITERS = PAIR_LEN,
CONCURRENT_NODES = CBM_SZ_1K,
CONCURRENT_EDGES = CBM_SZ_4K,
};
atomic_int start = 0;
sw_concurrent_job_t jobs[CONCURRENT_WRITERS] = {0};
cbm_thread_t threads[CONCURRENT_WRITERS];
for (int i = 0; i < CONCURRENT_WRITERS; i++) {
ASSERT_EQ(make_temp_db(jobs[i].path, sizeof(jobs[i].path)), 0);
snprintf(jobs[i].project, sizeof(jobs[i].project), "proj%d", i);
snprintf(jobs[i].root_path, sizeof(jobs[i].root_path), "/tmp/sw_concurrent_root_%d", i);
jobs[i].start = &start;
jobs[i].node_count = CONCURRENT_NODES;
jobs[i].edge_count = CONCURRENT_EDGES;
jobs[i].rc = CBM_NOT_FOUND;
ASSERT_EQ(cbm_thread_create(&threads[i], 0, sw_concurrent_writer_thread, &jobs[i]), 0);
}
atomic_store(&start, CBM_INIT_DONE);
for (int i = 0; i < CONCURRENT_WRITERS; i++) {
ASSERT_EQ(cbm_thread_join(&threads[i]), 0);
ASSERT_EQ(jobs[i].rc, 0);
ASSERT_EQ(verify_writer_db(jobs[i].path, jobs[i].project, jobs[i].root_path,
jobs[i].node_count, jobs[i].edge_count),
0);
cbm_unlink(jobs[i].path);
}
PASS();
}
SUITE(sqlite_writer) {
RUN_TEST(sw_minimal_data);
RUN_TEST(sw_store_open_migrates_exact_delta_metadata);
RUN_TEST(sw_imports_local_name_unique);
RUN_TEST(sw_scale_and_indexes);
RUN_TEST(sw_long_index_keys_overflow);
RUN_TEST(sw_empty);
RUN_TEST(sw_vectors_and_token_vectors);
RUN_TEST(sw_multi_page);
RUN_TEST(sw_oversized_node);
RUN_TEST(sw_scale_root_path_integrity);
RUN_TEST(sw_concurrent_writes_are_independent);
}
Reference in New Issue View Git Blame Copy Permalink