From 8adef05c5dd6bb1992b2eafabd0622da9f97548d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 3 Jul 2026 19:04:01 -0400 Subject: [PATCH] feat(store): add overlay-aware search primitive Add cbm_store_search_overlay_view() as an opt-in active node read view over canonical nodes minus ready overlay file tombstones plus owned nodes from the latest ready overlay per changed file. Refactor canonical search to share the SQL execution, allocation, scan, cleanup, and error path with the new overlay search primitive. Default search_graph/search_code/query behavior remains unchanged. Tests cover canonical fallback with no ready overlay and a full-rebuild oracle comparison where old rows are hidden, stable rows remain visible, latest overlay rows win, non-canonical overlay ids stay zero, and the degree-filter SQL wrapper is exercised. Validation: CBM_ONLY_SUITE=store_nodes make -f Makefile.cbm -j8 test -> /private/tmp/cbm-pan83-overlay-search-store-suite-20260703T1910.log (92 passed); CBM_ONLY_SUITE=mcp make -f Makefile.cbm -j8 test -> /private/tmp/cbm-pan83-overlay-search-mcp-suite-20260703T1914.log (144 passed); make -f Makefile.cbm -j8 cbm -> /private/tmp/cbm-pan83-overlay-search-product-build-20260703T1915.log; bash scripts/check-source-safety.sh -> /private/tmp/cbm-pan83-overlay-search-source-safety-20260703T1915.log; git diff --check clean; isolated dogfood index/search artifacts under /private/tmp/cbm-pan83-overlay-search-dogfood-*. Signed-off-by: Andrew Hundt --- src/store/store.c | 295 +++++++++++++++++++++++++++++---------- src/store/store.h | 5 + tests/test_store_nodes.c | 119 ++++++++++++++++ 3 files changed, 349 insertions(+), 70 deletions(-) diff --git a/src/store/store.c b/src/store/store.c index 591db245d..5773f3e28 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -6282,6 +6282,97 @@ static int search_build_where(const cbm_search_params_t *params, char *where, in return nparams; } +static int search_execute_sql(cbm_store_t *s, const char *sql, const char *count_sql, + search_bind_t *binds, int bind_idx, + search_like_pool_t *like_pool, bool use_pagerank, + const char *op, cbm_search_output_t *out) { + enum { + SEARCH_COL_IN_DEG = ST_COL_9, + SEARCH_COL_OUT_DEG = CBM_DECIMAL_BASE, + SEARCH_COL_PAGERANK = 11 + }; + const char *op_base = op ? op : "search"; + char prepare_op[CBM_SZ_128]; + char step_op[CBM_SZ_128]; + snprintf(prepare_op, sizeof(prepare_op), "%s prepare", op_base); + snprintf(step_op, sizeof(step_op), "%s step", op_base); + + sqlite3_stmt *cnt_stmt = NULL; + int rc = sqlite3_prepare_v2(s->db, count_sql, CBM_NOT_FOUND, &cnt_stmt, NULL); + if (rc == SQLITE_OK) { + for (int i = 0; i < bind_idx; i++) { + bind_text(cnt_stmt, i + SKIP_ONE, binds[i].text); + } + if (sqlite3_step(cnt_stmt) == SQLITE_ROW) { + out->total = sqlite3_column_int(cnt_stmt, 0); + } + sqlite3_finalize(cnt_stmt); + } + + sqlite3_stmt *main_stmt = NULL; + rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &main_stmt, NULL); + if (rc != SQLITE_OK) { + store_set_error_sqlite(s, prepare_op); + like_pool_free(like_pool); + return CBM_STORE_ERR; + } + + for (int i = 0; i < bind_idx; i++) { + bind_text(main_stmt, i + SKIP_ONE, binds[i].text); + } + + int cap = ST_INIT_CAP_16; + int n = 0; + cbm_search_result_t *results = calloc((size_t)cap, sizeof(cbm_search_result_t)); + if (!results) { + sqlite3_finalize(main_stmt); + like_pool_free(like_pool); + store_set_error(s, "search out of memory"); + return CBM_STORE_ERR; + } + + int step_rc = SQLITE_OK; + while ((step_rc = sqlite3_step(main_stmt)) == SQLITE_ROW) { + if (n >= cap) { + if (store_grow_array(s, (void **)&results, &cap, sizeof(*results), + "search out of memory", true) != CBM_STORE_OK) { + cbm_search_output_t partial = {.results = results, .count = n}; + cbm_store_search_free(&partial); + sqlite3_finalize(main_stmt); + like_pool_free(like_pool); + return CBM_STORE_ERR; + } + } + if (scan_node(s, main_stmt, &results[n].node) != CBM_STORE_OK) { + cbm_search_output_t partial = {.results = results, .count = n + 1}; + cbm_store_search_free(&partial); + sqlite3_finalize(main_stmt); + like_pool_free(like_pool); + return CBM_STORE_ERR; + } + results[n].in_degree = sqlite3_column_int(main_stmt, SEARCH_COL_IN_DEG); + results[n].out_degree = sqlite3_column_int(main_stmt, SEARCH_COL_OUT_DEG); + results[n].pagerank_score = + use_pagerank ? sqlite3_column_double(main_stmt, SEARCH_COL_PAGERANK) : 0.0; + n++; + } + if (step_rc != SQLITE_DONE) { + cbm_search_output_t partial = {.results = results, .count = n}; + cbm_store_search_free(&partial); + sqlite3_finalize(main_stmt); + like_pool_free(like_pool); + store_set_error_sqlite(s, step_op); + return CBM_STORE_ERR; + } + + sqlite3_finalize(main_stmt); + like_pool_free(like_pool); + + out->results = results; + out->count = n; + return CBM_STORE_OK; +} + int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out) { memset(out, 0, sizeof(*out)); if (!s || !s->db) { @@ -6479,84 +6570,148 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear } strncat(sql, order_limit, sizeof(sql) - strlen(sql) - 1); - /* Execute count query */ - sqlite3_stmt *cnt_stmt = NULL; - int rc = sqlite3_prepare_v2(s->db, count_sql, CBM_NOT_FOUND, &cnt_stmt, NULL); - if (rc == SQLITE_OK) { - for (int i = 0; i < bind_idx; i++) { - bind_text(cnt_stmt, i + SKIP_ONE, binds[i].text); - } - if (sqlite3_step(cnt_stmt) == SQLITE_ROW) { - out->total = sqlite3_column_int(cnt_stmt, 0); - } - sqlite3_finalize(cnt_stmt); - } + return search_execute_sql(s, sql, count_sql, binds, bind_idx, &like_pool, use_pagerank, + "search", out); +} - /* Execute main query */ - sqlite3_stmt *main_stmt = NULL; - rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &main_stmt, NULL); - if (rc != SQLITE_OK) { - store_set_error_sqlite(s, "search prepare"); - like_pool_free(&like_pool); +int cbm_store_search_overlay_view(cbm_store_t *s, const cbm_search_params_t *params, + cbm_search_output_t *out) { + memset(out, 0, sizeof(*out)); + if (!s || !s->db || !params) { return CBM_STORE_ERR; } - for (int i = 0; i < bind_idx; i++) { - bind_text(main_stmt, i + SKIP_ONE, binds[i].text); - } - - int cap = ST_INIT_CAP_16; - int n = 0; - cbm_search_result_t *results = calloc((size_t)cap, sizeof(cbm_search_result_t)); - if (!results) { - sqlite3_finalize(main_stmt); - like_pool_free(&like_pool); - store_set_error(s, "search out of memory"); - return CBM_STORE_ERR; - } - - int step_rc = SQLITE_OK; - while ((step_rc = sqlite3_step(main_stmt)) == SQLITE_ROW) { - if (n >= cap) { - if (store_grow_array(s, (void **)&results, &cap, sizeof(*results), - "search out of memory", true) != CBM_STORE_OK) { - cbm_search_output_t partial = {.results = results, .count = n}; - cbm_store_search_free(&partial); - sqlite3_finalize(main_stmt); - like_pool_free(&like_pool); - return CBM_STORE_ERR; - } + const char *freshness_project = + (params->project && params->project[0] && !params->project_pattern) ? params->project : NULL; + if (freshness_project) { + cbm_store_overlay_node_view_summary_t summary = {0}; + if (cbm_store_get_overlay_node_view_summary(s, freshness_project, &summary) != + CBM_STORE_OK || + summary.active_file_tombstones <= 0) { + return cbm_store_search(s, params, out); } - if (scan_node(s, main_stmt, &results[n].node) != CBM_STORE_OK) { - cbm_search_output_t partial = {.results = results, .count = n + 1}; - cbm_store_search_free(&partial); - sqlite3_finalize(main_stmt); - like_pool_free(&like_pool); - return CBM_STORE_ERR; - } - results[n].in_degree = sqlite3_column_int(main_stmt, ST_COL_9); - results[n].out_degree = sqlite3_column_int(main_stmt, CBM_DECIMAL_BASE); - /* MERGE: fork delta — pagerank_score at column 11 (only when use_pagerank - * selected the pr_rank column; otherwise no such column exists). */ - results[n].pagerank_score = - use_pagerank ? sqlite3_column_double(main_stmt, 11) : 0.0; - n++; - } - if (step_rc != SQLITE_DONE) { - cbm_search_output_t partial = {.results = results, .count = n}; - cbm_store_search_free(&partial); - sqlite3_finalize(main_stmt); - like_pool_free(&like_pool); - store_set_error_sqlite(s, "search step"); - return CBM_STORE_ERR; } - sqlite3_finalize(main_stmt); - like_pool_free(&like_pool); + out->pagerank_stale = + freshness_project && + cbm_store_derived_view_is_stale(s, freshness_project, CBM_STORE_DERIVED_VIEW_PAGERANK); + out->linkrank_stale = + freshness_project && + cbm_store_derived_view_is_stale(s, freshness_project, CBM_STORE_DERIVED_VIEW_LINKRANK); + out->node_degree_stale = + freshness_project && + cbm_store_derived_view_is_stale(s, freshness_project, CBM_STORE_DERIVED_VIEW_NODE_DEGREE); - out->results = results; - out->count = n; - return CBM_STORE_OK; + bool use_pagerank = + (!params->sort_by || strcmp(params->sort_by, "relevance") == 0) && !out->pagerank_stale; + const char *select_cols = + use_pagerank + ? "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties, " + "(SELECT COUNT(*) FROM edges e WHERE e.target_id = n.id) AS in_deg, " + "(SELECT COUNT(*) FROM edges e WHERE e.source_id = n.id) AS out_deg, " + "COALESCE(pr.rank, 0.0) AS pr_rank " + : "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties, " + "(SELECT COUNT(*) FROM edges e WHERE e.target_id = n.id) AS in_deg, " + "(SELECT COUNT(*) FROM edges e WHERE e.source_id = n.id) AS out_deg "; + const char *from_join = + use_pagerank ? "FROM active_nodes n LEFT JOIN pagerank pr ON pr.node_id = n.id" + : "FROM active_nodes n"; + + char active_cte[CBM_SZ_4K]; + snprintf(active_cte, sizeof(active_cte), + "WITH active_files AS (" + " SELECT t.project, t.rel_path, MAX(t.overlay_generation) AS overlay_generation" + " FROM overlay_tombstones t" + " JOIN overlay_generations g" + " ON g.project = t.project AND g.overlay_generation = t.overlay_generation" + " WHERE g.status = ?1 AND t.entity_kind = ?2 AND t.active = %d" + " GROUP BY t.project, t.rel_path" + "), active_nodes AS (" + " SELECT n.id, n.project, n.label, n.name, n.qualified_name, n.file_path," + " n.start_line, n.end_line, n.properties" + " FROM nodes n" + " WHERE NOT EXISTS (SELECT 1 FROM active_files af" + " WHERE af.project = n.project AND af.rel_path = n.file_path)" + " UNION ALL" + " SELECT %d AS id, n.project, n.label, n.name, n.qualified_name, n.file_path," + " n.start_line, n.end_line, n.properties" + " FROM overlay_nodes n" + " JOIN active_files af" + " ON af.project = n.project AND af.rel_path = n.rel_path" + " AND af.overlay_generation = n.overlay_generation" + " WHERE n.owned = %d" + ") ", + STORE_OVERLAY_TOMBSTONE_ACTIVE, CBM_STORE_NO_NODE_ID, STORE_OVERLAY_ROW_OWNED); + + char where[CBM_SZ_2K] = ""; + search_bind_t binds[ST_SEARCH_MAX_BINDS]; + search_like_pool_t like_pool = {0}; + int bind_idx = 0; + where_bind_text(binds, &bind_idx, CBM_STORE_OVERLAY_STATUS_READY); + where_bind_text(binds, &bind_idx, CBM_STORE_OVERLAY_TOMBSTONE_FILE); + int nparams = + search_build_where(params, where, (int)sizeof(where), binds, &bind_idx, &like_pool); + + char sql[CBM_SZ_4K]; + char count_sql[CBM_SZ_4K]; + if (nparams > 0) { + snprintf(sql, sizeof(sql), "%s%s %s WHERE %s", active_cte, select_cols, from_join, where); + snprintf(count_sql, sizeof(count_sql), "%sSELECT COUNT(*) FROM active_nodes n WHERE %s", + active_cte, where); + } else { + snprintf(sql, sizeof(sql), "%s%s %s", active_cte, select_cols, from_join); + snprintf(count_sql, sizeof(count_sql), "%sSELECT COUNT(*) FROM active_nodes n", + active_cte); + } + + bool has_degree_filter = (params->min_degree >= 0 || params->max_degree >= 0); + search_apply_degree_filter(sql, sizeof(sql), params); + if (has_degree_filter) { + snprintf(count_sql, sizeof(count_sql), "SELECT COUNT(*) FROM (%s)", sql); + } + + int limit = params->limit > 0 ? params->limit : CBM_DEFAULT_SEARCH_LIMIT; + int offset = params->offset; + const char *name_col = has_degree_filter ? "name" : "n.name"; + const char *id_col = has_degree_filter ? "id" : "n.id"; + const char *proj_col = has_degree_filter ? "project" : "n.project"; + bool scope_has_project = (params->project != NULL || params->project_pattern != NULL); + char dep_last[CBM_SZ_128]; + if (scope_has_project && !params->disable_dep_ranking) { + snprintf(dep_last, sizeof(dep_last), + "CASE WHEN %s LIKE '%%.dep.%%' THEN 1 ELSE 0 END, ", proj_col); + } else { + dep_last[0] = '\0'; + } + + char order_limit[CBM_SZ_256]; + if (use_pagerank) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY pr_rank DESC, %s%s, %s LIMIT %d OFFSET %d", + dep_last, name_col, id_col, limit, offset); + } else if (params->sort_by && strcmp(params->sort_by, "degree") == 0) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY (in_deg + out_deg) DESC, %s%s, %s LIMIT %d OFFSET %d", + dep_last, name_col, id_col, limit, offset); + } else if (params->sort_by && strcmp(params->sort_by, "calls") == 0) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY (in_deg + out_deg) DESC, %s%s, %s LIMIT %d OFFSET %d", + dep_last, name_col, id_col, limit, offset); + } else if (params->sort_by && strcmp(params->sort_by, "linkrank") == 0) { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY (in_deg + out_deg) DESC, %s%s, %s LIMIT %d OFFSET %d", + dep_last, name_col, id_col, limit, offset); + } else { + snprintf(order_limit, sizeof(order_limit), + " ORDER BY %s%s, %s LIMIT %d OFFSET %d", + dep_last, name_col, id_col, limit, offset); + } + strncat(sql, order_limit, sizeof(sql) - strlen(sql) - 1); + + return search_execute_sql(s, sql, count_sql, binds, bind_idx, &like_pool, use_pagerank, + "search_overlay_view", out); } void cbm_store_search_free(cbm_search_output_t *out) { diff --git a/src/store/store.h b/src/store/store.h index 65a37f22e..6076bf367 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -746,6 +746,11 @@ int cbm_store_delete_file_delta_complete(cbm_store_t *s, const char *project, /* ── Search ─────────────────────────────────────────────────────── */ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out); +/* Opt-in active node read view: canonical nodes whose files are not hidden by + * active ready overlay file tombstones, plus owned nodes from the latest ready + * overlay for each changed file. Overlay rows use id=CBM_STORE_NO_NODE_ID. */ +int cbm_store_search_overlay_view(cbm_store_t *s, const cbm_search_params_t *params, + cbm_search_output_t *out); /* Free a search output's allocated memory. */ void cbm_store_search_free(cbm_search_output_t *out); diff --git a/tests/test_store_nodes.c b/tests/test_store_nodes.c index caf5f3973..adc4852f6 100644 --- a/tests/test_store_nodes.c +++ b/tests/test_store_nodes.c @@ -1584,6 +1584,123 @@ TEST(store_find_nodes_by_file_overlay_view_returns_latest_ready_rows) { PASS(); } +TEST(store_search_overlay_view_without_ready_overlay_matches_canonical_search) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + ASSERT_EQ(cbm_store_upsert_project(s, "test", "/tmp/test"), CBM_STORE_OK); + + cbm_node_t old_main = {.project = "test", + .label = "Function", + .name = "old_main", + .qualified_name = "test.old_main", + .file_path = "main.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(s, &old_main), 0); + + cbm_search_params_t params = {.project = "test", + .pattern = "main", + .sort_by = "name", + .limit = 10}; + cbm_search_output_t active = {0}; + ASSERT_EQ(cbm_store_search_overlay_view(s, ¶ms, &active), CBM_STORE_OK); + ASSERT_EQ(active.total, 1); + ASSERT_EQ(active.count, 1); + ASSERT_GT(active.results[0].node.id, CBM_STORE_NO_NODE_ID); + ASSERT_STR_EQ(active.results[0].node.name, "old_main"); + cbm_store_search_free(&active); + + cbm_store_close(s); + PASS(); +} + +TEST(store_search_overlay_view_matches_full_rebuild_oracle) { + enum { BASE_GENERATION = 1 }; + cbm_store_t *live = cbm_store_open_memory(); + cbm_store_t *oracle = cbm_store_open_memory(); + ASSERT_NOT_NULL(live); + ASSERT_NOT_NULL(oracle); + ASSERT_EQ(cbm_store_upsert_project(live, "test", "/tmp/test"), CBM_STORE_OK); + ASSERT_EQ(cbm_store_upsert_project(oracle, "test", "/tmp/test"), CBM_STORE_OK); + + cbm_node_t old_main = {.project = "test", + .label = "Function", + .name = "old_main", + .qualified_name = "test.old_main", + .file_path = "main.go", + .properties_json = "{}"}; + cbm_node_t stable = {.project = "test", + .label = "Function", + .name = "stable", + .qualified_name = "test.stable", + .file_path = "stable.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(live, &old_main), 0); + ASSERT_GT(cbm_store_upsert_node(live, &stable), 0); + ASSERT_GT(cbm_store_upsert_node(oracle, &stable), 0); + + int64_t first_overlay = 0; + ASSERT_EQ(cbm_store_reserve_overlay_generation(live, "test", BASE_GENERATION, + &first_overlay), + CBM_STORE_OK); + cbm_node_t first_main = {.project = "test", + .label = "Function", + .name = "new_main", + .qualified_name = "test.new_main", + .file_path = "main.go", + .properties_json = "{}"}; + cbm_store_file_delta_t first_delta = {.project = "test", + .rel_path = "main.go", + .generation = BASE_GENERATION, + .nodes = &first_main, + .node_count = 1}; + ASSERT_EQ(cbm_store_publish_overlay_file_delta(live, &first_delta, first_overlay), + CBM_STORE_OK); + + int64_t second_overlay = 0; + ASSERT_EQ(cbm_store_reserve_overlay_generation(live, "test", BASE_GENERATION, + &second_overlay), + CBM_STORE_OK); + cbm_node_t newer_main = {.project = "test", + .label = "Function", + .name = "newer_main", + .qualified_name = "test.newer_main", + .file_path = "main.go", + .properties_json = "{}"}; + cbm_store_file_delta_t second_delta = {.project = "test", + .rel_path = "main.go", + .generation = BASE_GENERATION, + .nodes = &newer_main, + .node_count = 1}; + ASSERT_EQ(cbm_store_publish_overlay_file_delta(live, &second_delta, second_overlay), + CBM_STORE_OK); + ASSERT_GT(cbm_store_upsert_node(oracle, &newer_main), 0); + + cbm_search_params_t params = {.project = "test", + .pattern = "main|stable", + .sort_by = "name", + .limit = 10, + .min_degree = 0}; + cbm_search_output_t active = {0}; + cbm_search_output_t expected = {0}; + ASSERT_EQ(cbm_store_search_overlay_view(live, ¶ms, &active), CBM_STORE_OK); + ASSERT_EQ(cbm_store_search(oracle, ¶ms, &expected), CBM_STORE_OK); + ASSERT_EQ(active.total, expected.total); + ASSERT_EQ(active.count, expected.count); + ASSERT_EQ(active.count, 2); + ASSERT_STR_EQ(active.results[0].node.name, expected.results[0].node.name); + ASSERT_STR_EQ(active.results[1].node.name, expected.results[1].node.name); + ASSERT_STR_EQ(active.results[0].node.name, "newer_main"); + ASSERT_STR_EQ(active.results[1].node.name, "stable"); + ASSERT_EQ(active.results[0].node.id, CBM_STORE_NO_NODE_ID); + ASSERT_GT(active.results[1].node.id, CBM_STORE_NO_NODE_ID); + + cbm_store_search_free(&active); + cbm_store_search_free(&expected); + cbm_store_close(live); + cbm_store_close(oracle); + PASS(); +} + TEST(store_owner_metadata_crud) { cbm_store_t *s = cbm_store_open_memory(); ASSERT_NOT_NULL(s); @@ -4321,6 +4438,8 @@ SUITE(store_nodes) { RUN_TEST(store_overlay_file_delta_publish_rejects_failed_generation); RUN_TEST(store_overlay_node_view_summary_counts_latest_ready_overlay); RUN_TEST(store_find_nodes_by_file_overlay_view_returns_latest_ready_rows); + RUN_TEST(store_search_overlay_view_without_ready_overlay_matches_canonical_search); + RUN_TEST(store_search_overlay_view_matches_full_rebuild_oracle); RUN_TEST(store_owner_metadata_crud); RUN_TEST(store_rebuild_file_delta_owners_derives_from_graph); RUN_TEST(store_import_export_metadata_crud);