mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-20 05:49:37 +08:00
Break-glass merge PR #2642
This commit is contained in:
+2
-1
@@ -449,7 +449,7 @@ PREPROCESSOR_SRC = $(CBM_DIR)/preprocessor.cpp
|
||||
SQLITE_WRITER_SRC = $(CBM_DIR)/sqlite_writer.c
|
||||
|
||||
# Store module (new)
|
||||
STORE_SRCS = src/store/store.c src/store/architecture_projection.c src/store/store_impact.c src/store/store_graph_digest.c src/store/store_content_digest.c
|
||||
STORE_SRCS = src/store/store.c src/store/graph_mem.c src/store/architecture_projection.c src/store/store_impact.c src/store/store_graph_digest.c src/store/store_content_digest.c
|
||||
|
||||
# Cypher module (new)
|
||||
CYPHER_SRCS = src/cypher/cypher.c
|
||||
@@ -746,6 +746,7 @@ TEST_STORE_SRCS = \
|
||||
tests/test_store_impact.c \
|
||||
tests/test_store_scope.c \
|
||||
tests/test_store_graph_digest.c \
|
||||
tests/test_graph_mem.c \
|
||||
tests/test_store_search.c \
|
||||
tests/test_store_arch.c \
|
||||
tests/test_architecture_projection.c \
|
||||
|
||||
@@ -24,7 +24,7 @@ src/cli/config_text_edit.c 72
|
||||
src/cli/config_toml_edit.c 73
|
||||
src/cli/config_yaml_edit.c 117
|
||||
src/cli/hook_augment.c 31
|
||||
src/cypher/cypher.c 132
|
||||
src/cypher/cypher.c 125
|
||||
src/daemon/application.c 107
|
||||
src/daemon/bootstrap.c 13
|
||||
src/daemon/daemon.c 31
|
||||
|
||||
+1054
-315
File diff suppressed because it is too large
Load Diff
@@ -359,6 +359,9 @@ void cbm_cypher_test_set_deadline_ms(int64_t budget_ms);
|
||||
/* Override only the deadline clock on this thread; NULL restores real time. */
|
||||
typedef uint64_t (*cbm_cypher_test_clock_fn)(void);
|
||||
void cbm_cypher_test_set_deadline_clock(cbm_cypher_test_clock_fn clock_fn);
|
||||
/* Start nodes per streamed execution chunk on this thread; 0 restores the
|
||||
* default. INT_MAX runs the whole start set as one chunk (a single pass). */
|
||||
void cbm_cypher_test_set_stream_chunk(int start_nodes);
|
||||
#endif
|
||||
|
||||
/* Worst-case binding slot count for a node cross-join. Computes the count in
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
#include "mcp/mcp.h"
|
||||
#include "mcp/mcp_internal.h"
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "store/graph_mem.h"
|
||||
#include "ui/config.h"
|
||||
#include "watcher/watcher.h"
|
||||
|
||||
@@ -1893,7 +1894,13 @@ static size_t application_active_job_count_locked(cbm_daemon_application_t *appl
|
||||
/* Decision 2 (#1997 #832): a worker's memory slice is the aggregate budget
|
||||
* divided by the jobs active at spawn time. The job being spawned is already
|
||||
* in the table, so the divisor never drops below one; a zero aggregate means
|
||||
* "no cap" and the worker falls back to its own RAM-fraction budget. */
|
||||
* "no cap" and the worker falls back to its own RAM-fraction budget.
|
||||
*
|
||||
* The daemon's resident adjacency layers (store/graph_mem.h) are part of the
|
||||
* same aggregate, so workers split what is left after them. They are capped at
|
||||
* a quarter of the daemon budget, which the aggregate defaults to; only a
|
||||
* smaller configured aggregate can be exceeded, and then every job still gets
|
||||
* at least one byte, as above — a zero slice would read as "no cap". */
|
||||
static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application,
|
||||
size_t *active_jobs_out) {
|
||||
size_t active = application_active_job_count_locked(application);
|
||||
@@ -1903,7 +1910,16 @@ static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *a
|
||||
if (active_jobs_out) {
|
||||
*active_jobs_out = active;
|
||||
}
|
||||
return application->aggregate_memory_budget_bytes / active;
|
||||
size_t aggregate = application->aggregate_memory_budget_bytes;
|
||||
if (aggregate == 0) {
|
||||
return 0;
|
||||
}
|
||||
size_t held = cbm_graph_mem_cached_bytes();
|
||||
size_t available = held < aggregate ? aggregate - held : 0;
|
||||
if (available < active) {
|
||||
available = active;
|
||||
}
|
||||
return available / active;
|
||||
}
|
||||
|
||||
/* Compare the effective index request, not its JSON spelling. yyjson's deep
|
||||
|
||||
@@ -67,7 +67,7 @@ static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT];
|
||||
|
||||
static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = {
|
||||
"other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", "extract", "arena",
|
||||
"ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array",
|
||||
"ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array", "graph_cache",
|
||||
};
|
||||
|
||||
const char *cbm_mem_class_name(cbm_mem_class_t cls) {
|
||||
|
||||
@@ -66,6 +66,10 @@ typedef enum {
|
||||
CBM_MEM_CLASS_STORE, /* SQLite (bound mem methods) + store batches and row buffers */
|
||||
CBM_MEM_CLASS_HASH_TABLE, /* CBMHashTable buckets/entries not claimed by an owner class */
|
||||
CBM_MEM_CLASS_DYN_ARRAY, /* CBM_DYN_ARRAY item storage (every cbm_da_* user) */
|
||||
/* The daemon's in-memory adjacency layers (store/graph_mem.h): resident
|
||||
* across requests, so they are subtracted from index-worker budgets and
|
||||
* must read apart from SQLite's request-scoped memory (STORE). */
|
||||
CBM_MEM_CLASS_GRAPH_CACHE,
|
||||
CBM_MEM_CLASS_COUNT
|
||||
} cbm_mem_class_t;
|
||||
|
||||
|
||||
+44
-5
@@ -44,11 +44,14 @@ static bool git_validate_repo_path(const char *repo_path) {
|
||||
return cbm_validate_shell_path_arg(repo_path);
|
||||
}
|
||||
|
||||
static int git_capture(const char *repo_path, const char *git_args, char **out) {
|
||||
if (!out) {
|
||||
/* Run `git -C repo_path git_args` and copy the first output line (newlines
|
||||
* trimmed) into buf. Returns 0 only when git exited 0 with a non-empty line. */
|
||||
static int git_capture_into(const char *repo_path, const char *git_args, char *buf,
|
||||
size_t buf_size) {
|
||||
if (!buf || buf_size == 0) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*out = NULL;
|
||||
buf[0] = '\0';
|
||||
if (!repo_path || !git_args || !git_validate_repo_path(repo_path)) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
@@ -76,8 +79,8 @@ static int git_capture(const char *repo_path, const char *git_args, char **out)
|
||||
}
|
||||
|
||||
cbm_log_debug("git.capture", "stage", "read", "args", git_args);
|
||||
char buf[GIT_OUTPUT_MAX];
|
||||
if (!fgets(buf, sizeof(buf), fp)) {
|
||||
if (!fgets(buf, (int)buf_size, fp)) {
|
||||
buf[0] = '\0';
|
||||
cbm_log_debug("git.capture", "stage", "close", "args", git_args, "output", "none");
|
||||
cbm_pclose(fp);
|
||||
cbm_log_debug("git.capture", "stage", "done", "args", git_args);
|
||||
@@ -89,9 +92,21 @@ static int git_capture(const char *repo_path, const char *git_args, char **out)
|
||||
int rc = cbm_pclose(fp);
|
||||
cbm_log_debug("git.capture", "stage", "done", "args", git_args);
|
||||
if (rc != 0 || buf[0] == '\0') {
|
||||
buf[0] = '\0';
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int git_capture(const char *repo_path, const char *git_args, char **out) {
|
||||
if (!out) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*out = NULL;
|
||||
char buf[GIT_OUTPUT_MAX];
|
||||
if (git_capture_into(repo_path, git_args, buf, sizeof(buf)) != 0) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*out = git_strdup(buf);
|
||||
return *out ? 0 : CBM_NOT_FOUND;
|
||||
}
|
||||
@@ -313,6 +328,30 @@ int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cbm_git_branch_resolve(const char *path, bool *is_git, char *branch, size_t branch_size) {
|
||||
if (!is_git || !branch || branch_size == 0) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*is_git = false;
|
||||
branch[0] = '\0';
|
||||
if (!path || !path[0]) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
if (!cbm_is_dir(path)) {
|
||||
return 0;
|
||||
}
|
||||
char worktree_root[GIT_OUTPUT_MAX];
|
||||
if (git_capture_into(path, "rev-parse --show-toplevel", worktree_root, sizeof(worktree_root)) !=
|
||||
0) {
|
||||
return 0;
|
||||
}
|
||||
*is_git = true;
|
||||
if (git_capture_into(path, "symbolic-ref --quiet --short HEAD", branch, branch_size) != 0) {
|
||||
snprintf(branch, branch_size, "%s", "DETACHED");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx) {
|
||||
const char *project = project_name && project_name[0] ? project_name : "project";
|
||||
const char *slug = "working-tree";
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#define CBM_GIT_CONTEXT_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
typedef struct {
|
||||
bool is_git;
|
||||
@@ -20,6 +21,12 @@ typedef struct {
|
||||
} cbm_git_context_t;
|
||||
|
||||
int cbm_git_context_resolve(const char *path, cbm_git_context_t *out);
|
||||
/* Only the is_git and branch fields of cbm_git_context_resolve, from the same
|
||||
* two git commands it uses for them (2 git processes instead of 7). branch
|
||||
* gets "" when path is not in a git work tree; pass CBM_GIT_BRANCH_MAX bytes
|
||||
* to read exactly what cbm_git_context_resolve reads. */
|
||||
#define CBM_GIT_BRANCH_MAX 4096
|
||||
int cbm_git_branch_resolve(const char *path, bool *is_git, char *branch, size_t branch_size);
|
||||
void cbm_git_context_free(cbm_git_context_t *ctx);
|
||||
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx);
|
||||
int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size);
|
||||
|
||||
+676
-59
@@ -59,6 +59,7 @@ enum {
|
||||
#include "mcp/architecture_jobs.h"
|
||||
#include "mcp/test_impact.h"
|
||||
#include "mcp/test_impact_engine.h"
|
||||
#include "store/graph_mem.h"
|
||||
#include "store/store.h"
|
||||
#include "store/store_impact.h"
|
||||
#include <sqlite3.h>
|
||||
@@ -82,6 +83,7 @@ enum {
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/hash_table.h"
|
||||
#include "foundation/subprocess.h"
|
||||
#include "foundation/sha256.h"
|
||||
#include "mcp/index_supervisor.h"
|
||||
@@ -1863,6 +1865,7 @@ struct cbm_mcp_server {
|
||||
cbm_mcp_snapshot_read_test_hook_fn snapshot_read_test_hook;
|
||||
void *snapshot_read_test_context;
|
||||
#endif
|
||||
bool read_batch_tool; /* this request's store reads run in a read batch */
|
||||
size_t search_output_limit_override;
|
||||
const char *search_scan_command_override;
|
||||
const char *search_scratch_dir_override; /* NULL = cbm_tmpdir() */
|
||||
@@ -2648,7 +2651,11 @@ static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *pr
|
||||
|
||||
static cbm_store_t *resolve_store(cbm_mcp_server_t *srv, const char *project) {
|
||||
/* Query-only callers (every read tool): strictly non-mutating resolve. */
|
||||
return resolve_store_internal(srv, project, false, false, NULL, false);
|
||||
cbm_store_t *store = resolve_store_internal(srv, project, false, false, NULL, false);
|
||||
if (store && srv->read_batch_tool) {
|
||||
(void)cbm_store_read_batch_begin(store);
|
||||
}
|
||||
return store;
|
||||
}
|
||||
|
||||
/* Forward decl — definition lives below in handle_trace_call_path's helpers. */
|
||||
@@ -3018,16 +3025,15 @@ static bool populate_project_record(const char *dir_path, bool include_stats,
|
||||
* null for non-git roots — cost ~10KB across a full cache and is one
|
||||
* index_status call away for the project you actually care about. */
|
||||
if (record->root_path[0]) {
|
||||
cbm_git_context_t gctx = {0};
|
||||
(void)cbm_git_context_resolve(record->root_path, &gctx);
|
||||
if (gctx.is_git && gctx.branch) {
|
||||
record->branch = heap_strdup(gctx.branch);
|
||||
bool is_git = false;
|
||||
char branch[CBM_GIT_BRANCH_MAX];
|
||||
(void)cbm_git_branch_resolve(record->root_path, &is_git, branch, sizeof(branch));
|
||||
if (is_git && branch[0]) {
|
||||
record->branch = heap_strdup(branch);
|
||||
if (!record->branch) {
|
||||
cbm_git_context_free(&gctx);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
cbm_git_context_free(&gctx);
|
||||
}
|
||||
record->populated = true;
|
||||
return true;
|
||||
@@ -10432,20 +10438,33 @@ static char *read_file_lines(const char *path, int start, int end) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Whole line spans per memchr instead of one locked fgetc and one append
|
||||
* per byte: search_code's full mode reads the source of every result, and
|
||||
* the byte loop was half of a 500-result page. Same text-mode stream, so
|
||||
* Windows CRLF translation is unchanged. */
|
||||
enum { READ_FILE_LINES_CHUNK = 16384 };
|
||||
cbm_sb_t selected;
|
||||
cbm_sb_init(&selected);
|
||||
int lineno = 1;
|
||||
int byte;
|
||||
while ((byte = fgetc(fp)) != EOF) {
|
||||
if (lineno >= start && lineno <= end) {
|
||||
char ch = (char)byte;
|
||||
cbm_sb_append_n(&selected, &ch, 1U);
|
||||
}
|
||||
if (byte == '\n') {
|
||||
if (lineno >= end) {
|
||||
break;
|
||||
char chunk[READ_FILE_LINES_CHUNK];
|
||||
bool done = false;
|
||||
size_t got;
|
||||
while (!done && (got = fread(chunk, 1, sizeof(chunk), fp)) > 0) {
|
||||
size_t pos = 0;
|
||||
while (pos < got) {
|
||||
const char *newline = memchr(chunk + pos, '\n', got - pos);
|
||||
size_t span_end = newline ? (size_t)(newline - chunk) + SKIP_ONE : got;
|
||||
if (lineno >= start && lineno <= end) {
|
||||
cbm_sb_append_n(&selected, chunk + pos, span_end - pos);
|
||||
}
|
||||
pos = span_end;
|
||||
if (newline) {
|
||||
if (lineno >= end) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
lineno++;
|
||||
}
|
||||
lineno++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12739,6 +12758,64 @@ bool cbm_path_within_root(const char *root_path, const char *abs_path) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* cbm_path_within_root with the root resolved once per request: search_code
|
||||
* checks every hit file and every rendered result against the same root, and
|
||||
* resolving the root again for each check was a third of a 500-result page.
|
||||
* An unresolvable root fails every check, exactly as cbm_path_within_root. */
|
||||
typedef struct {
|
||||
char canonical[CBM_SZ_4K];
|
||||
bool resolved;
|
||||
/* Verdict per absolute path, for the life of one request: the reply is
|
||||
* rendered up to dozens of times while it is fitted to its byte budget,
|
||||
* and each render checks every result's file again. Keys are owned. */
|
||||
CBMHashTable *verdicts;
|
||||
} root_containment_t;
|
||||
|
||||
enum { ROOT_CONTAINMENT_INSIDE = 1, ROOT_CONTAINMENT_OUTSIDE = 2 };
|
||||
|
||||
static void root_containment_init(root_containment_t *containment, const char *root_path) {
|
||||
containment->verdicts = NULL;
|
||||
containment->resolved = root_path && resolve_canonical_path(root_path, containment->canonical,
|
||||
sizeof(containment->canonical));
|
||||
}
|
||||
|
||||
static void root_containment_free_key(const char *key, void *value, void *userdata) {
|
||||
(void)value;
|
||||
(void)userdata;
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, (void *)key);
|
||||
}
|
||||
|
||||
static void root_containment_free(root_containment_t *containment) {
|
||||
if (containment->verdicts) {
|
||||
cbm_ht_foreach(containment->verdicts, root_containment_free_key, NULL);
|
||||
cbm_ht_free(containment->verdicts);
|
||||
containment->verdicts = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool root_containment_check(root_containment_t *containment, const char *abs_path) {
|
||||
if (!containment->resolved || !abs_path) {
|
||||
return false;
|
||||
}
|
||||
void *cached = containment->verdicts ? cbm_ht_get(containment->verdicts, abs_path) : NULL;
|
||||
if (cached) {
|
||||
return (uintptr_t)cached == ROOT_CONTAINMENT_INSIDE;
|
||||
}
|
||||
char real_file[CBM_SZ_4K];
|
||||
bool inside = resolve_canonical_path(abs_path, real_file, sizeof(real_file)) &&
|
||||
canonical_path_has_root(containment->canonical, real_file);
|
||||
if (!containment->verdicts) {
|
||||
containment->verdicts = cbm_ht_create(0);
|
||||
}
|
||||
char *key = containment->verdicts ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, abs_path) : NULL;
|
||||
if (key) {
|
||||
(void)cbm_ht_set(
|
||||
containment->verdicts, key,
|
||||
(void *)(uintptr_t)(inside ? ROOT_CONTAINMENT_INSIDE : ROOT_CONTAINMENT_OUTSIDE));
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
/* #1690: a path-shaped `project` names a directory; the installed guidance tells
|
||||
* agents to pass their working directory. Turning it into the path-DERIVED name
|
||||
* alone missed a checkout indexed under an explicit name and every directory
|
||||
@@ -14286,7 +14363,7 @@ static yyjson_mut_val *build_dedup_files_array(yyjson_mut_doc *doc, search_resul
|
||||
/* Attach source or context lines to a search result JSON item. */
|
||||
static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, search_result_t *r,
|
||||
int mode, int context_lines, int source_max_lines,
|
||||
const char *root_path) {
|
||||
const char *root_path, root_containment_t *containment) {
|
||||
enum { MODE_FULL = 1 };
|
||||
if (r->start_line <= 0 || r->end_line <= 0) {
|
||||
return;
|
||||
@@ -14308,7 +14385,7 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
|
||||
* project root (a `..` segment, or a symlink/junction that discovery
|
||||
* followed) must not be read back into the response. Same guard the
|
||||
* snippet path already uses. */
|
||||
if (!cbm_path_within_root(root_path, abs_path)) {
|
||||
if (!root_containment_check(containment, abs_path)) {
|
||||
free(abs_path);
|
||||
return;
|
||||
}
|
||||
@@ -14713,7 +14790,8 @@ static char *assemble_search_output(
|
||||
int gm_count, int result_start, int result_limit, int output_count, int raw_start,
|
||||
int raw_output, int directory_start, int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines, int source_max_lines, const char *root_path,
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit) {
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit,
|
||||
root_containment_t *containment) {
|
||||
enum { MODE_COMPACT = 0, MODE_FULL = 1, MODE_FILES = 2, SEARCH_SLOW_MS = 5000 };
|
||||
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
@@ -14770,7 +14848,8 @@ static char *assemble_search_output(
|
||||
yyjson_mut_arr_add_int(doc, row, r->out_degree);
|
||||
if (mode == MODE_FULL || attach_context) {
|
||||
yyjson_mut_val *src = yyjson_mut_obj(doc);
|
||||
attach_result_source(doc, src, r, mode, context_lines, source_max_lines, root_path);
|
||||
attach_result_source(doc, src, r, mode, context_lines, source_max_lines, root_path,
|
||||
containment);
|
||||
yyjson_mut_arr_add_val(row, src);
|
||||
}
|
||||
yyjson_mut_arr_add_val(results_arr, row);
|
||||
@@ -14954,15 +15033,13 @@ static char *assemble_search_output(
|
||||
* form. Compact search has a purpose-built table renderer so repeated paths
|
||||
* and qualified-name prefixes can use the response-local directory. Full and
|
||||
* files mode first build the canonical JSON model, then project it to tree. */
|
||||
static char *render_search_payload(search_result_t *sr, int sr_count, grep_match_t *raw,
|
||||
int raw_count, int raw_content_truncated, int gm_count,
|
||||
int result_start, int result_limit, int output_count,
|
||||
int raw_start, int raw_output, int directory_start,
|
||||
int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines,
|
||||
int source_max_lines, const char *root_path, bool scan_saturated,
|
||||
bool warn_literal_pipe, const search_metrics_t *metrics,
|
||||
bool budget_hit, bool json_format) {
|
||||
static char *render_search_payload(
|
||||
search_result_t *sr, int sr_count, grep_match_t *raw, int raw_count, int raw_content_truncated,
|
||||
int gm_count, int result_start, int result_limit, int output_count, int raw_start,
|
||||
int raw_output, int directory_start, int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines, int source_max_lines, const char *root_path,
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit,
|
||||
bool json_format, root_containment_t *request_containment) {
|
||||
if (mode == 0 && !json_format) {
|
||||
return assemble_search_output_toon(sr, sr_count, raw, raw_count, raw_content_truncated,
|
||||
gm_count, result_start, result_limit, output_count,
|
||||
@@ -14971,11 +15048,22 @@ static char *render_search_payload(search_result_t *sr, int sr_count, grep_match
|
||||
warn_literal_pipe, metrics, budget_hit, -1);
|
||||
}
|
||||
|
||||
/* A caller that renders once passes NULL; search_code passes the
|
||||
* containment it keeps for the whole request (see root_containment_t). */
|
||||
root_containment_t local_containment;
|
||||
root_containment_t *containment = request_containment;
|
||||
if (!containment) {
|
||||
root_containment_init(&local_containment, root_path);
|
||||
containment = &local_containment;
|
||||
}
|
||||
char *json = assemble_search_output(
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start, result_limit,
|
||||
output_count, raw_start, raw_output, directory_start, directory_output, raw_limit,
|
||||
directory_limit, match_limit, mode, context_lines, source_max_lines, root_path,
|
||||
scan_saturated, warn_literal_pipe, metrics, budget_hit);
|
||||
scan_saturated, warn_literal_pipe, metrics, budget_hit, containment);
|
||||
if (!request_containment) {
|
||||
root_containment_free(&local_containment);
|
||||
}
|
||||
if (!json || json_format) {
|
||||
return json;
|
||||
}
|
||||
@@ -15017,7 +15105,7 @@ char *cbm_mcp_render_search_rows_for_testing(const char *const *qualified_names,
|
||||
char *payload =
|
||||
render_search_payload(results, row_count, NULL, 0, 0, row_count, 0, row_count, row_count, 0,
|
||||
0, 0, row_count, 0, row_count > 0 ? 1 : 0, row_count > 0 ? 1 : 0, 0,
|
||||
0, 0, "", false, false, &zero_metrics, false, json_format);
|
||||
0, 0, "", false, false, &zero_metrics, false, json_format, NULL);
|
||||
free_search_results(results, initialized);
|
||||
return payload;
|
||||
}
|
||||
@@ -15251,7 +15339,7 @@ char *cbm_mcp_render_raw_preview_for_testing(const char *content, bool content_o
|
||||
search_raw_preview(&raw, content, content_offset_set, content_offset, false, 0, 0);
|
||||
return render_search_payload(NULL, 0, &raw, 1, raw.content_truncated ? 1 : 0, 1, 0, 0, 0, 0, 1,
|
||||
0, 0, 1, 0, 0, 0, 0, 0, "", false, false, &zero_metrics, false,
|
||||
json_format);
|
||||
json_format, NULL);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -15325,13 +15413,87 @@ static bool append_search_match_line(search_result_t *result, int line) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a grep hit to the search result set (merge into existing or create new). */
|
||||
static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap, cbm_node_t *n,
|
||||
int line) {
|
||||
for (int j = 0; j < *sr_count; j++) {
|
||||
if ((*sr)[j].node_id == n->id) {
|
||||
return append_search_match_line(&(*sr)[j], line);
|
||||
/* Node id -> index into the search result array, so merging a hit into its
|
||||
* node's result is O(1). Finding it by scanning every result so far made a
|
||||
* common pattern quadratic: `def ` on django is 33,672 hits against a result
|
||||
* list that grows to tens of thousands. Open addressing over a power-of-two
|
||||
* table; a slot stores index + 1, so 0 is free and no node id is reserved. */
|
||||
typedef struct {
|
||||
int64_t *ids;
|
||||
int *slots;
|
||||
int capacity;
|
||||
int count;
|
||||
} search_result_index_t;
|
||||
|
||||
enum { SEARCH_RESULT_INDEX_MIN_CAPACITY = 64 };
|
||||
|
||||
static uint32_t search_result_index_hash(int64_t id, int capacity) {
|
||||
uint64_t mixed = (uint64_t)id * UINT64_C(0x9E3779B97F4A7C15);
|
||||
return (uint32_t)(mixed >> 32) & (uint32_t)(capacity - 1);
|
||||
}
|
||||
|
||||
static void search_result_index_free(search_result_index_t *index) {
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, index->ids);
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, index->slots);
|
||||
memset(index, 0, sizeof(*index));
|
||||
}
|
||||
|
||||
static int search_result_index_find(const search_result_index_t *index, int64_t id) {
|
||||
if (index->capacity == 0) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
uint32_t at = search_result_index_hash(id, index->capacity);
|
||||
while (index->slots[at] != 0) {
|
||||
if (index->ids[at] == id) {
|
||||
return index->slots[at] - SKIP_ONE;
|
||||
}
|
||||
at = (at + SKIP_ONE) & (uint32_t)(index->capacity - 1);
|
||||
}
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
/* Insert into a table with room (search_result_index_reserve succeeded). */
|
||||
static void search_result_index_insert(search_result_index_t *index, int64_t id, int value) {
|
||||
uint32_t at = search_result_index_hash(id, index->capacity);
|
||||
while (index->slots[at] != 0) {
|
||||
at = (at + SKIP_ONE) & (uint32_t)(index->capacity - 1);
|
||||
}
|
||||
index->ids[at] = id;
|
||||
index->slots[at] = value + SKIP_ONE;
|
||||
index->count++;
|
||||
}
|
||||
|
||||
/* Make room for one more entry (at most half full), growing the table. */
|
||||
static bool search_result_index_reserve(search_result_index_t *index) {
|
||||
if ((index->count + SKIP_ONE) * PAIR_LEN <= index->capacity) {
|
||||
return true;
|
||||
}
|
||||
int capacity = index->capacity ? index->capacity * PAIR_LEN : SEARCH_RESULT_INDEX_MIN_CAPACITY;
|
||||
search_result_index_t grown = {
|
||||
.ids = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)capacity * sizeof(int64_t)),
|
||||
.slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)capacity * sizeof(int)),
|
||||
.capacity = capacity,
|
||||
};
|
||||
if (!grown.ids || !grown.slots) {
|
||||
search_result_index_free(&grown);
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < index->capacity; i++) {
|
||||
if (index->slots[i] != 0) {
|
||||
search_result_index_insert(&grown, index->ids[i], index->slots[i] - SKIP_ONE);
|
||||
}
|
||||
}
|
||||
search_result_index_free(index);
|
||||
*index = grown;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a grep hit to the search result set (merge into existing or create new). */
|
||||
static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap,
|
||||
search_result_index_t *index, cbm_node_t *n, int line) {
|
||||
int existing = search_result_index_find(index, n->id);
|
||||
if (existing >= 0) {
|
||||
return append_search_match_line(&(*sr)[existing], line);
|
||||
}
|
||||
if (*sr_count >= *sr_cap) {
|
||||
int next_capacity = *sr_cap * PAIR_LEN;
|
||||
@@ -15343,6 +15505,11 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c
|
||||
*sr = grown;
|
||||
*sr_cap = next_capacity;
|
||||
}
|
||||
/* Room in the index first, so nothing below can fail after the result's
|
||||
* strings exist. */
|
||||
if (!search_result_index_reserve(index)) {
|
||||
return false;
|
||||
}
|
||||
search_result_t *r = &(*sr)[*sr_count];
|
||||
if (!set_search_result_identity(r, n)) {
|
||||
return false;
|
||||
@@ -15354,18 +15521,20 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c
|
||||
r->file = NULL;
|
||||
return false;
|
||||
}
|
||||
search_result_index_insert(index, n->id, *sr_count);
|
||||
(*sr_count)++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Match a single grep hit to the tightest containing node, then add to sr or raw. */
|
||||
static bool classify_grep_hit(grep_match_t *hit, cbm_node_t *file_nodes, int file_node_count,
|
||||
search_result_t **sr, int *sr_count, int *sr_cap, grep_match_t **raw,
|
||||
int raw_offset, int raw_limit, int *raw_count, int *raw_stored_count,
|
||||
int *raw_cap, int *raw_content_truncated) {
|
||||
search_result_t **sr, int *sr_count, int *sr_cap,
|
||||
search_result_index_t *sr_index, grep_match_t **raw, int raw_offset,
|
||||
int raw_limit, int *raw_count, int *raw_stored_count, int *raw_cap,
|
||||
int *raw_content_truncated) {
|
||||
int best = find_tightest_node(file_nodes, file_node_count, hit->line);
|
||||
if (best >= 0) {
|
||||
return add_to_search_results(sr, sr_count, sr_cap, &file_nodes[best], hit->line);
|
||||
return add_to_search_results(sr, sr_count, sr_cap, sr_index, &file_nodes[best], hit->line);
|
||||
} else {
|
||||
if (*raw_count == INT_MAX) {
|
||||
return false;
|
||||
@@ -15422,13 +15591,14 @@ static void free_file_nodes(cbm_node_t *nodes, int count) {
|
||||
* applies before it reads a result's source, applied to the match itself,
|
||||
* once per distinct file (grep groups a file's hits). A path too long for
|
||||
* the canonical-path buffers is one that check cannot resolve either. */
|
||||
static bool search_hit_file_within_root(const char *root_path, const char *file) {
|
||||
static bool search_hit_file_within_root(root_containment_t *containment, const char *root_path,
|
||||
const char *file) {
|
||||
char abs_path[CBM_SZ_4K];
|
||||
int length = snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, file);
|
||||
if (length < 0 || (size_t)length >= sizeof(abs_path)) {
|
||||
return false;
|
||||
}
|
||||
return cbm_path_within_root(root_path, abs_path);
|
||||
return root_containment_check(containment, abs_path);
|
||||
}
|
||||
|
||||
/* Parse and classify the complete grep stream without retaining one object per
|
||||
@@ -15447,6 +15617,17 @@ static bool scan_and_classify_grep_matches(
|
||||
bool current_file_inside = false;
|
||||
cbm_node_t *file_nodes = NULL;
|
||||
int file_node_count = 0;
|
||||
root_containment_t containment;
|
||||
root_containment_init(&containment, root_path);
|
||||
search_result_index_t sr_index = {0};
|
||||
for (int i = 0; i < *sr_count; i++) {
|
||||
if (!search_result_index_reserve(&sr_index)) {
|
||||
search_result_index_free(&sr_index);
|
||||
root_containment_free(&containment);
|
||||
return false;
|
||||
}
|
||||
search_result_index_insert(&sr_index, (*sr)[i].node_id, i);
|
||||
}
|
||||
cbm_regex_t content_regex;
|
||||
bool content_regex_ready = use_regex && pattern &&
|
||||
cbm_regcomp(&content_regex, pattern, CBM_REG_EXTENDED) == CBM_REG_OK;
|
||||
@@ -15500,7 +15681,7 @@ static bool scan_and_classify_grep_matches(
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
current_file_inside = search_hit_file_within_root(root_path, file);
|
||||
current_file_inside = search_hit_file_within_root(&containment, root_path, file);
|
||||
if (store && current_file_inside) {
|
||||
(void)cbm_store_find_nodes_by_file(store, project, file, &file_nodes,
|
||||
&file_node_count);
|
||||
@@ -15532,8 +15713,8 @@ static bool scan_and_classify_grep_matches(
|
||||
&match_start, &match_end);
|
||||
search_raw_preview(&hit, content, raw_content_offset_set, raw_content_offset, match_known,
|
||||
match_start, match_end);
|
||||
if (!classify_grep_hit(&hit, file_nodes, file_node_count, sr, sr_count, sr_cap, raw,
|
||||
raw_offset, raw_limit, raw_count, raw_stored_count, raw_cap,
|
||||
if (!classify_grep_hit(&hit, file_nodes, file_node_count, sr, sr_count, sr_cap, &sr_index,
|
||||
raw, raw_offset, raw_limit, raw_count, raw_stored_count, raw_cap,
|
||||
raw_content_truncated)) {
|
||||
free(hit.file);
|
||||
ok = false;
|
||||
@@ -15542,6 +15723,8 @@ static bool scan_and_classify_grep_matches(
|
||||
free(hit.file);
|
||||
}
|
||||
|
||||
search_result_index_free(&sr_index);
|
||||
root_containment_free(&containment);
|
||||
if (content_regex_ready) {
|
||||
cbm_regfree(&content_regex);
|
||||
}
|
||||
@@ -15833,6 +16016,374 @@ static void search_scratch_close(search_scratch_t *scratch) {
|
||||
}
|
||||
}
|
||||
|
||||
/* ── In-process content scan ────────────────────────────────────────
|
||||
* search_code used to hand the scoped file list to `xargs sh grep` on POSIX and
|
||||
* to a PowerShell Select-String pipeline on Windows: a process spawn per call,
|
||||
* BSD grep's ~130 MB/s on macOS, and one cmdlet invocation per file on Windows
|
||||
* (23-31 s per call on GitHub's windows-11-arm image, against the 30 s scan
|
||||
* deadline). This scanner reads the same list and writes the records
|
||||
* `grep -Hn` printed, so the classifier and everything after it are unchanged:
|
||||
* - patterns: the pattern split on '\n', as `grep -f` reads its file; an
|
||||
* empty pattern matches every line (grep special-cases it in -E mode too);
|
||||
* - literal mode is a substring match (grep -F); regex mode runs each line
|
||||
* through cbm_regex, the platform's POSIX ERE engine (grep -E);
|
||||
* - a NUL byte in the first 32 KiB marks the file binary and yields no
|
||||
* record, as grep prints no lines for it; after a later NUL nothing more
|
||||
* is reported, as GNU grep stops line output there;
|
||||
* - a record is `<listed path><sep><line number><sep><line bytes>\n` with
|
||||
* ':' (POSIX) or TAB (Windows), the separators the classifier reads; the
|
||||
* line keeps a CR, which the classifier strips as before.
|
||||
* Deadline, cancellation and the output limit end the scan with the causes the
|
||||
* subprocess runner reported; an unreadable listed file fails the scan closed,
|
||||
* as grep's error exit did. */
|
||||
enum {
|
||||
SEARCH_SCAN_BINARY_PROBE = 32768,
|
||||
SEARCH_SCAN_CHECK_EVERY = 4096, /* lines between deadline/cancel checks */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
char **items;
|
||||
size_t *lengths;
|
||||
int count;
|
||||
cbm_regex_t *regexes; /* regex mode: one per non-empty pattern */
|
||||
int compiled;
|
||||
bool any_empty; /* an empty pattern: every line matches */
|
||||
} search_scan_patterns_t;
|
||||
|
||||
static void search_scan_patterns_free(search_scan_patterns_t *patterns) {
|
||||
for (int i = 0; i < patterns->count; i++) {
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, patterns->items[i]);
|
||||
}
|
||||
for (int i = 0; patterns->regexes && i < patterns->count; i++) {
|
||||
if (i < patterns->compiled && patterns->lengths[i] > 0) {
|
||||
cbm_regfree(&patterns->regexes[i]);
|
||||
}
|
||||
}
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, patterns->items);
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, patterns->lengths);
|
||||
cbm_free(CBM_MEM_CLASS_OTHER, patterns->regexes);
|
||||
memset(patterns, 0, sizeof(*patterns));
|
||||
}
|
||||
|
||||
static bool search_scan_patterns_parse(search_scan_patterns_t *patterns, const char *pattern,
|
||||
bool use_regex) {
|
||||
memset(patterns, 0, sizeof(*patterns));
|
||||
int count = 1;
|
||||
for (const char *p = pattern; *p; p++) {
|
||||
count += *p == '\n' ? SKIP_ONE : 0;
|
||||
}
|
||||
patterns->items = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(char *));
|
||||
patterns->lengths = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(size_t));
|
||||
patterns->regexes =
|
||||
use_regex ? cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(cbm_regex_t)) : NULL;
|
||||
if (!patterns->items || !patterns->lengths || (use_regex && !patterns->regexes)) {
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
const char *start = pattern;
|
||||
for (int i = 0; i < count; i++) {
|
||||
const char *end = strchr(start, '\n');
|
||||
size_t length = end ? (size_t)(end - start) : strlen(start);
|
||||
patterns->items[i] = cbm_alloc(CBM_MEM_CLASS_OTHER, length + SKIP_ONE);
|
||||
if (!patterns->items[i]) {
|
||||
patterns->count = i;
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
memcpy(patterns->items[i], start, length);
|
||||
patterns->items[i][length] = '\0';
|
||||
patterns->lengths[i] = length;
|
||||
patterns->any_empty = patterns->any_empty || length == 0;
|
||||
start = end ? end + SKIP_ONE : start + length;
|
||||
}
|
||||
patterns->count = count;
|
||||
for (int i = 0; use_regex && i < count; i++) {
|
||||
if (patterns->lengths[i] > 0 &&
|
||||
cbm_regcomp(&patterns->regexes[i], patterns->items[i],
|
||||
CBM_REG_EXTENDED | CBM_REG_NOSUB) != CBM_REG_OK) {
|
||||
patterns->compiled = i;
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
patterns->compiled = i + SKIP_ONE;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* First occurrence of needle in hay; memmem where the C library has it. */
|
||||
static const char *search_scan_find(const char *hay, size_t hay_length, const char *needle,
|
||||
size_t needle_length) {
|
||||
if (needle_length == 0) {
|
||||
return hay;
|
||||
}
|
||||
if (needle_length > hay_length) {
|
||||
return NULL;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
const char *last = hay + (hay_length - needle_length);
|
||||
for (const char *p = hay; p <= last;) {
|
||||
p = memchr(p, (unsigned char)needle[0], (size_t)(last - p) + SKIP_ONE);
|
||||
if (!p) {
|
||||
return NULL;
|
||||
}
|
||||
if (memcmp(p, needle, needle_length) == 0) {
|
||||
return p;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return NULL;
|
||||
#else
|
||||
return memmem(hay, hay_length, needle, needle_length);
|
||||
#endif
|
||||
}
|
||||
|
||||
static size_t search_scan_count_newlines(const char *from, const char *to) {
|
||||
size_t count = 0;
|
||||
while (from < to) {
|
||||
const char *newline = memchr(from, '\n', (size_t)(to - from));
|
||||
if (!newline) {
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
from = newline + SKIP_ONE;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
FILE *out;
|
||||
size_t written;
|
||||
size_t output_limit;
|
||||
char separator;
|
||||
} search_scan_sink_t;
|
||||
|
||||
/* One `grep -Hn` record. False on a write error or once past the output limit. */
|
||||
static bool search_scan_emit(search_scan_sink_t *sink, const char *path, size_t path_length,
|
||||
size_t line_number, const char *line, size_t line_length,
|
||||
bool *over_limit) {
|
||||
char number[CBM_SZ_32];
|
||||
int number_length =
|
||||
snprintf(number, sizeof(number), "%c%zu%c", sink->separator, line_number, sink->separator);
|
||||
if (number_length <= 0 || (size_t)number_length >= sizeof(number) ||
|
||||
fwrite(path, 1, path_length, sink->out) != path_length ||
|
||||
fwrite(number, 1, (size_t)number_length, sink->out) != (size_t)number_length ||
|
||||
fwrite(line, 1, line_length, sink->out) != line_length || fputc('\n', sink->out) == EOF) {
|
||||
return false;
|
||||
}
|
||||
sink->written += path_length + (size_t)number_length + line_length + SKIP_ONE;
|
||||
if (sink->output_limit > 0 && sink->written > sink->output_limit) {
|
||||
*over_limit = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool search_scan_line_matches(const search_scan_patterns_t *patterns, bool use_regex,
|
||||
const char *line) {
|
||||
if (patterns->any_empty) {
|
||||
return true;
|
||||
}
|
||||
for (int i = 0; i < patterns->count; i++) {
|
||||
if (use_regex ? cbm_regexec(&patterns->regexes[i], line, 0, NULL, 0) == CBM_REG_OK
|
||||
: strstr(line, patterns->items[i]) != NULL) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Scan one file. `buffer` holds its bytes plus one spare byte. */
|
||||
static mcp_scan_cause_t search_scan_buffer(cbm_mcp_server_t *srv, search_scan_sink_t *sink,
|
||||
const search_scan_patterns_t *patterns, bool use_regex,
|
||||
const char *path, size_t path_length, char *buffer,
|
||||
size_t length, uint64_t deadline_ms) {
|
||||
size_t probe = length < SEARCH_SCAN_BINARY_PROBE ? length : SEARCH_SCAN_BINARY_PROBE;
|
||||
if (memchr(buffer, '\0', probe)) {
|
||||
return MCP_SCAN_SUCCESS;
|
||||
}
|
||||
const char *nul = memchr(buffer, '\0', length);
|
||||
if (nul) {
|
||||
/* Lines up to the last complete line before the NUL. */
|
||||
size_t keep = 0;
|
||||
for (size_t i = (size_t)(nul - buffer); i > 0; i--) {
|
||||
if (buffer[i - SKIP_ONE] == '\n') {
|
||||
keep = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
length = keep;
|
||||
}
|
||||
buffer[length] = '\0';
|
||||
bool over_limit = false;
|
||||
size_t line_number = 1;
|
||||
size_t since_check = 0;
|
||||
bool single_literal = !use_regex && !patterns->any_empty && patterns->count == 1;
|
||||
size_t pos = 0;
|
||||
while (pos < length) {
|
||||
if (++since_check >= SEARCH_SCAN_CHECK_EVERY) {
|
||||
since_check = 0;
|
||||
if (mcp_request_cancelled(srv)) {
|
||||
return MCP_SCAN_CANCELLED;
|
||||
}
|
||||
if (cbm_now_ms() >= deadline_ms) {
|
||||
return MCP_SCAN_DEADLINE;
|
||||
}
|
||||
}
|
||||
size_t line_start = pos;
|
||||
if (single_literal) {
|
||||
/* Jump to the next occurrence instead of visiting every line. */
|
||||
const char *hit = search_scan_find(buffer + pos, length - pos, patterns->items[0],
|
||||
patterns->lengths[0]);
|
||||
if (!hit) {
|
||||
break;
|
||||
}
|
||||
line_start = (size_t)(hit - buffer);
|
||||
while (line_start > pos && buffer[line_start - SKIP_ONE] != '\n') {
|
||||
line_start--;
|
||||
}
|
||||
line_number += search_scan_count_newlines(buffer + pos, buffer + line_start);
|
||||
}
|
||||
char *newline = memchr(buffer + line_start, '\n', length - line_start);
|
||||
size_t line_end = newline ? (size_t)(newline - buffer) : length;
|
||||
/* Windows matched Select-String's Line, which has no trailing CR, so
|
||||
* `foo$` matched a CRLF line there; POSIX grep matches with the CR. */
|
||||
size_t match_end = line_end;
|
||||
#ifdef _WIN32
|
||||
if (match_end > line_start && buffer[match_end - SKIP_ONE] == '\r') {
|
||||
match_end--;
|
||||
}
|
||||
#endif
|
||||
char saved = buffer[match_end];
|
||||
buffer[match_end] = '\0';
|
||||
bool matched =
|
||||
single_literal || search_scan_line_matches(patterns, use_regex, buffer + line_start);
|
||||
buffer[match_end] = saved;
|
||||
if (newline) {
|
||||
*newline = '\n';
|
||||
}
|
||||
if (matched && !search_scan_emit(sink, path, path_length, line_number, buffer + line_start,
|
||||
line_end - line_start, &over_limit)) {
|
||||
return over_limit ? MCP_SCAN_OUTPUT_LIMIT : MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
pos = line_end + SKIP_ONE;
|
||||
line_number++;
|
||||
}
|
||||
return MCP_SCAN_SUCCESS;
|
||||
}
|
||||
|
||||
/* Read a whole file into *buffer (grown as needed, CBM_MEM_CLASS_EXTRACT),
|
||||
* leaving one spare byte. */
|
||||
static bool search_scan_read_file(const char *path, char **buffer, size_t *capacity,
|
||||
size_t *length) {
|
||||
FILE *fp = cbm_fopen(path, "rb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
size_t used = 0;
|
||||
bool ok = true;
|
||||
for (;;) {
|
||||
if (*capacity - used < CBM_SZ_64K + SKIP_ONE) {
|
||||
size_t grown = *capacity ? *capacity * PAIR_LEN : (size_t)CBM_SZ_64K * PAIR_LEN;
|
||||
if (grown <= *capacity || grown > (size_t)INT_MAX) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
char *next = cbm_realloc(CBM_MEM_CLASS_EXTRACT, *buffer, grown);
|
||||
if (!next) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
*buffer = next;
|
||||
*capacity = grown;
|
||||
}
|
||||
size_t want = *capacity - used - SKIP_ONE;
|
||||
size_t got = fread(*buffer + used, 1, want, fp);
|
||||
used += got;
|
||||
if (got < want) {
|
||||
ok = !ferror(fp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
(void)fclose(fp);
|
||||
*length = used;
|
||||
return ok;
|
||||
}
|
||||
|
||||
static mcp_scan_cause_t search_scan_in_process(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, FILE *out,
|
||||
size_t output_limit, uint64_t deadline_ms) {
|
||||
#ifdef _WIN32
|
||||
const char record_separator = '\n';
|
||||
const char field_separator = '\t';
|
||||
#else
|
||||
const char record_separator = '\0';
|
||||
const char field_separator = ':';
|
||||
#endif
|
||||
search_scan_patterns_t patterns;
|
||||
if (!search_scan_patterns_parse(&patterns, pattern, use_regex)) {
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
char *list = NULL;
|
||||
size_t list_capacity = 0;
|
||||
size_t list_length = 0;
|
||||
if (!search_scan_read_file(filelist_path, &list, &list_capacity, &list_length)) {
|
||||
cbm_free(CBM_MEM_CLASS_EXTRACT, list);
|
||||
search_scan_patterns_free(&patterns);
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
search_scan_sink_t sink = {
|
||||
.out = out, .output_limit = output_limit, .separator = field_separator};
|
||||
char *buffer = NULL;
|
||||
size_t capacity = 0;
|
||||
mcp_scan_cause_t cause = MCP_SCAN_SUCCESS;
|
||||
size_t pos = 0;
|
||||
while (cause == MCP_SCAN_SUCCESS && pos < list_length) {
|
||||
const char *end = memchr(list + pos, record_separator, list_length - pos);
|
||||
size_t record_end = end ? (size_t)(end - list) : list_length;
|
||||
list[record_end] = '\0';
|
||||
const char *path = list + pos;
|
||||
size_t path_length = record_end - pos;
|
||||
pos = record_end + SKIP_ONE;
|
||||
if (path_length == 0) {
|
||||
continue;
|
||||
}
|
||||
if (mcp_request_cancelled(srv)) {
|
||||
cause = MCP_SCAN_CANCELLED;
|
||||
break;
|
||||
}
|
||||
if (cbm_now_ms() >= deadline_ms) {
|
||||
cause = MCP_SCAN_DEADLINE;
|
||||
break;
|
||||
}
|
||||
size_t length = 0;
|
||||
if (!search_scan_read_file(path, &buffer, &capacity, &length)) {
|
||||
cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
break;
|
||||
}
|
||||
cause = search_scan_buffer(srv, &sink, &patterns, use_regex, path, path_length, buffer,
|
||||
length, deadline_ms);
|
||||
}
|
||||
cbm_free(CBM_MEM_CLASS_EXTRACT, buffer);
|
||||
cbm_free(CBM_MEM_CLASS_EXTRACT, list);
|
||||
search_scan_patterns_free(&patterns);
|
||||
return cause;
|
||||
}
|
||||
|
||||
int cbm_mcp_search_scan_for_test(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, const char *out_path,
|
||||
size_t output_limit, uint64_t deadline_ms) {
|
||||
FILE *out = cbm_fopen(out_path, "wb");
|
||||
if (!out) {
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
mcp_scan_cause_t cause = search_scan_in_process(srv, filelist_path, pattern, use_regex, out,
|
||||
output_limit, deadline_ms);
|
||||
if (fclose(out) != 0 && cause == MCP_SCAN_SUCCESS) {
|
||||
cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
return (int)cause;
|
||||
}
|
||||
|
||||
/* Open the scratch directory, write `pattern` to the grep -f file, and leave the
|
||||
* file list open for write_scoped_filelist. Returns true on success; on failure
|
||||
* everything already created is removed before returning. */
|
||||
@@ -16259,10 +16810,39 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
: MCP_SEARCH_OUTPUT_MAX;
|
||||
const char *scan_command =
|
||||
srv->search_scan_command_override ? srv->search_scan_command_override : cmd;
|
||||
mcp_scan_cause_t scan_cause = mcp_run_shell_command_cancellable_bounded(
|
||||
srv, scan_command, output_path, scan_output_limit, scan_deadline_ms, true,
|
||||
scan_deadline_latched, /*exit_one_is_no_match=*/false, /*git_child=*/false,
|
||||
&scan_result);
|
||||
/* An indexed project is scanned in process (see search_scan_in_process).
|
||||
* The recursive fallback for an unindexed root, a Windows file_pattern
|
||||
* whose -like post-filter cannot be applied ahead of the scan, and the
|
||||
* test command override keep the subprocess runner. */
|
||||
bool scan_in_process = scoped && !srv->search_scan_command_override;
|
||||
#ifdef _WIN32
|
||||
scan_in_process =
|
||||
scan_in_process &&
|
||||
(!file_pattern || cbm_search_code_file_pattern_can_prefilter(file_pattern));
|
||||
#endif
|
||||
mcp_scan_cause_t scan_cause;
|
||||
if (scan_in_process) {
|
||||
FILE *scan_out = NULL;
|
||||
if (scan_cancellation_latched || mcp_request_cancelled(srv)) {
|
||||
scan_cause = MCP_SCAN_CANCELLED;
|
||||
} else if (scan_deadline_latched || cbm_now_ms() >= scan_deadline_ms) {
|
||||
scan_cause = MCP_SCAN_DEADLINE;
|
||||
} else if (!(scan_out = search_scratch_file(scratch.dir, "scan", output_path,
|
||||
sizeof(output_path)))) {
|
||||
scan_cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
} else {
|
||||
scan_cause = search_scan_in_process(srv, filelist, pattern, use_regex, scan_out,
|
||||
scan_output_limit, scan_deadline_ms);
|
||||
}
|
||||
if (scan_out && fclose(scan_out) != 0 && scan_cause == MCP_SCAN_SUCCESS) {
|
||||
scan_cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
} else {
|
||||
scan_cause = mcp_run_shell_command_cancellable_bounded(
|
||||
srv, scan_command, output_path, scan_output_limit, scan_deadline_ms, true,
|
||||
scan_deadline_latched, /*exit_one_is_no_match=*/false, /*git_child=*/false,
|
||||
&scan_result);
|
||||
}
|
||||
/* Both POSIX commands wrap grep and map its no-match status to 0, so any
|
||||
* non-zero exit (a failed find/sort, an unreadable operand, a broken
|
||||
* grep) is an incomplete scan and fails closed. */
|
||||
@@ -16410,11 +16990,15 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
directory_output = directory_limit;
|
||||
}
|
||||
bool warn_literal_pipe = pat_has_pipe && !use_regex;
|
||||
/* Every render below re-checks each result's file against the root; one
|
||||
* containment for the request keeps that to one resolution per file. */
|
||||
root_containment_t request_containment;
|
||||
root_containment_init(&request_containment, root_path);
|
||||
char *payload = render_search_payload(
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start, result_limit,
|
||||
output_count, raw_start, raw_output, directory_start, directory_output, raw_limit,
|
||||
directory_limit, match_limit, mode, context_lines, source_max_lines, root_path,
|
||||
scan_saturated, warn_literal_pipe, &metrics, false, sc_legacy_json);
|
||||
scan_saturated, warn_literal_pipe, &metrics, false, sc_legacy_json, &request_containment);
|
||||
|
||||
if (payload && strlen(payload) > byte_budget) {
|
||||
free(payload);
|
||||
@@ -16422,7 +17006,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
|
||||
/* Preserve ranked graph answers. Unclassified raw grep rows and directory
|
||||
@@ -16434,7 +17019,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
while (payload && strlen(payload) > byte_budget && directory_output > 0) {
|
||||
directory_output--;
|
||||
@@ -16443,7 +17029,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
|
||||
/* Full source is the next detail tier. Find the largest whole-line window
|
||||
@@ -16460,7 +17047,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, output_count, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
middle, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
if (candidate && strlen(candidate) <= byte_budget) {
|
||||
free(best_payload);
|
||||
best_payload = candidate;
|
||||
@@ -16482,7 +17069,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, output_count, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
source_max_lines, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16498,7 +17085,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, candidate_rows, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
source_max_lines, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
if (candidate && strlen(candidate) <= byte_budget) {
|
||||
best_payload = candidate;
|
||||
break;
|
||||
@@ -16515,6 +17102,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
raw_start, raw_limit, directory_start, directory_limit,
|
||||
dir_total, scan_saturated, sc_legacy_json, max_output_tokens);
|
||||
}
|
||||
root_containment_free(&request_containment);
|
||||
char *result = cbm_mcp_text_result(payload ? payload : "out of memory", payload == NULL);
|
||||
free(payload);
|
||||
free_search_results(sr, sr_count);
|
||||
@@ -19964,6 +20552,24 @@ static char *dispatch_tool(cbm_mcp_server_t *srv, const char *tool_name, const c
|
||||
return cbm_mcp_text_result(msg, true);
|
||||
}
|
||||
|
||||
/* Read tools whose work is store reads from start to end run them in one store
|
||||
* read batch (cbm_store_read_batch_begin). Left out: tools that write, open
|
||||
* their own transaction or read scope (detect_changes), or hold the store
|
||||
* across a long scan or a git subprocess (search_code, index_status). */
|
||||
static bool mcp_tool_reads_in_batch(const char *tool_name) {
|
||||
static const char *const batched[] = {
|
||||
"query_graph", "trace_path", "trace_call_path", "search_graph",
|
||||
"get_architecture", "get_graph_schema", "get_code_snippet", "get_file_outline",
|
||||
"compare_graphs", "check_index_coverage",
|
||||
};
|
||||
for (size_t i = 0; tool_name && i < sizeof(batched) / sizeof(batched[0]); i++) {
|
||||
if (strcmp(tool_name, batched[i]) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* File-backed query stores are request-scoped. Keeping one open between MCP
|
||||
* calls pins an old database generation after another process atomically
|
||||
* replaces the project DB. On Windows it can also prevent that replacement
|
||||
@@ -19999,13 +20605,24 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
|
||||
return cbm_mcp_text_result("request cancellation scope unavailable", true);
|
||||
}
|
||||
cbm_mem_phase_mark("request.dispatch_tool");
|
||||
if (srv) {
|
||||
srv->read_batch_tool = mcp_tool_reads_in_batch(tool_name);
|
||||
}
|
||||
char *result = dispatch_tool(srv, tool_name, args_json);
|
||||
if (srv) {
|
||||
/* An embedded store outlives the request: end its batch here. */
|
||||
srv->read_batch_tool = false;
|
||||
cbm_store_read_batch_end(srv->store);
|
||||
}
|
||||
cbm_mem_phase_mark("request.scope_end");
|
||||
if (srv) {
|
||||
cbm_mcp_server_request_scope_end(srv);
|
||||
}
|
||||
cbm_mem_phase_mark("request.release_store");
|
||||
release_request_store(srv);
|
||||
/* Adjacency layers nobody used for a while go now: the next request is
|
||||
* the daemon's clock, so no timer thread is needed (store/graph_mem.h). */
|
||||
cbm_graph_mem_evict_expired();
|
||||
cbm_mem_phase_mark("idle");
|
||||
/* One census per completed request, so growth can be attributed to a POOL
|
||||
* rather than inferred from a process total (#581). Emitted after the
|
||||
|
||||
@@ -39,6 +39,13 @@ void cbm_mcp_server_set_search_scan_timeout_for_test(cbm_mcp_server_t *srv, uint
|
||||
* scratch count cannot see other processes' entries in the shared temp dir.
|
||||
* The server borrows dir until it is reset or the server is freed. */
|
||||
void cbm_mcp_server_set_search_scratch_dir_for_test(cbm_mcp_server_t *srv, const char *dir);
|
||||
/* search_code's in-process scan of `filelist_path` (records separated as the
|
||||
* scoped list is written: NUL on POSIX, newline on Windows) into `out_path`.
|
||||
* Returns the scan cause in mcp_scan_cause_t order: 0 success, 1 command
|
||||
* failure, 3 output limit, 4 deadline, 5 cancelled. */
|
||||
int cbm_mcp_search_scan_for_test(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, const char *out_path,
|
||||
size_t output_limit, uint64_t deadline_ms);
|
||||
|
||||
/* Release only the constructor-created pristine in-memory store. Public
|
||||
* cbm_mcp_server_new(NULL) semantics remain unchanged; daemon sessions use
|
||||
|
||||
+119
-19
@@ -260,6 +260,15 @@ struct cbm_pipeline {
|
||||
* "invalid_existing_db" (#1864). */
|
||||
bool final_existed;
|
||||
bool existing_generation;
|
||||
/* The copy into the stage is deferred until routing needs it: an
|
||||
* unchanged repository is proven current on the live generation
|
||||
* (live_db_path, read-only) and never copies, seals or renames a
|
||||
* database. backup_succeeded records the copy once it ran; live_noop
|
||||
* marks a run that ended on the live proof with nothing staged. */
|
||||
bool stage_backup_pending;
|
||||
bool backup_succeeded;
|
||||
bool live_noop;
|
||||
const char *live_db_path; /* borrowed from cbm_pipeline_run */
|
||||
|
||||
/* ADR (project_summaries) captured before a full-reindex DB delete, so it
|
||||
* can be restored after the rebuild. NULL when no ADR existed. Issue #516. */
|
||||
@@ -848,7 +857,10 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path,
|
||||
p->repo_path = strdup(repo_path);
|
||||
p->db_path = db_path ? strdup(db_path) : NULL;
|
||||
p->project_name = cbm_project_name_from_path(repo_path);
|
||||
(void)cbm_git_context_resolve(repo_path, &p->git_ctx);
|
||||
/* The Git context is resolved once, when the run starts
|
||||
* (pipeline_refresh_git_context): a context resolved here was always
|
||||
* replaced before any use, at the cost of seven git subprocesses per
|
||||
* index call. Until then the branch QN names the working tree. */
|
||||
p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx);
|
||||
p->requested_mode = mode;
|
||||
p->mode = mode;
|
||||
@@ -2738,12 +2750,69 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int remove_stage_files(const char *stage_path);
|
||||
|
||||
/* Copy the live generation into this run's stage: every route except the
|
||||
* live no-op works on that copy. A failed copy behaves as the eager copy it
|
||||
* replaces did: the stage files go (the name and its lock stay this run's,
|
||||
* the rebuilt generation is renamed over it later) and the run has no
|
||||
* previous generation, so it rebuilds in full. */
|
||||
static void materialize_stage_backup(cbm_pipeline_t *p) {
|
||||
p->stage_backup_pending = false;
|
||||
p->backup_succeeded = cbm_store_backup_path(p->live_db_path, p->db_path) == CBM_STORE_OK;
|
||||
if (!p->backup_succeeded) {
|
||||
cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path",
|
||||
p->live_db_path);
|
||||
(void)remove_stage_files(p->db_path);
|
||||
}
|
||||
p->existing_generation = p->backup_succeeded;
|
||||
}
|
||||
|
||||
/* The live generation is provably current: read-only, it passes the shallow
|
||||
* integrity and format checks the staged route applies to its copy, and the
|
||||
* exact semantic-manifest test the staged route would answer no-op with. A
|
||||
* copy of it would route to that no-op and publish the same database again,
|
||||
* so the run ends here with nothing copied, sealed or renamed. Any doubt
|
||||
* answers false and the run copies and routes exactly as before. */
|
||||
static bool live_generation_current(cbm_pipeline_t *p, const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count) {
|
||||
struct stat db_st;
|
||||
if (!p->live_db_path || stat(p->live_db_path, &db_st) != 0) {
|
||||
return false;
|
||||
}
|
||||
cbm_store_t *store = cbm_store_open_path_query(p->live_db_path);
|
||||
if (!store) {
|
||||
return false;
|
||||
}
|
||||
int fmt = 0;
|
||||
bool current = cbm_store_check_integrity(store);
|
||||
if (current) {
|
||||
cbm_store_get_format_version(store, &fmt);
|
||||
current =
|
||||
fmt == CBM_INDEX_FORMAT_VERSION &&
|
||||
cbm_pipeline_incremental_generation_exact(p, store, baseline_manifest, baseline_count);
|
||||
}
|
||||
cbm_store_close(store);
|
||||
return current;
|
||||
}
|
||||
|
||||
/* Route an existing generation. Full rebuilds never delete the live DB here:
|
||||
* publication owns the eventual atomic replacement after every pass and
|
||||
* metadata write has succeeded. */
|
||||
static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count,
|
||||
const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count, bool force_full_on_mismatch) {
|
||||
if (p->stage_backup_pending) {
|
||||
if (live_generation_current(p, baseline_manifest, baseline_count)) {
|
||||
p->stage_backup_pending = false;
|
||||
p->live_noop = true;
|
||||
cbm_log_info("pipeline.route", "path", "incremental_manifest", "generation", "live");
|
||||
cbm_log_info("incremental.noop", "reason", "semantic_manifest_equal", "generation",
|
||||
"live");
|
||||
return 0;
|
||||
}
|
||||
materialize_stage_backup(p);
|
||||
}
|
||||
if (!p->existing_generation) {
|
||||
/* Nothing to be incremental against: a first index, or a
|
||||
* destination that could not be copied (already reported as
|
||||
@@ -2846,12 +2915,15 @@ static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) {
|
||||
if (!p || !p->project_name) {
|
||||
return false;
|
||||
}
|
||||
char *db_path = resolve_db_path(p);
|
||||
/* Until routing copies it, the stage is an empty placeholder: the
|
||||
* coverage the copy would carry is read from the live generation. */
|
||||
char *resolved = p->stage_backup_pending ? NULL : resolve_db_path(p);
|
||||
const char *db_path = p->stage_backup_pending ? p->live_db_path : resolved;
|
||||
if (!db_path) {
|
||||
return false;
|
||||
}
|
||||
cbm_store_t *store = cbm_store_open_path_query(db_path);
|
||||
free(db_path);
|
||||
free(resolved);
|
||||
if (!store) {
|
||||
return false;
|
||||
}
|
||||
@@ -4542,6 +4614,29 @@ static void sweep_orphan_stages(const char *final_path) {
|
||||
free(dir_path);
|
||||
}
|
||||
|
||||
/* End a run whose exact no-op was proven on the live generation. Nothing was
|
||||
* staged, so nothing is sealed or renamed and the live database stays the
|
||||
* current generation. The checks publication makes on the run's sources and
|
||||
* destination still decide success, and the artifact is refreshed from the
|
||||
* live database exactly as it would be from a republished one. */
|
||||
static int finish_live_noop(cbm_pipeline_t *p, const char *final_path) {
|
||||
cbm_userconfig_t *fresh_config = cbm_userconfig_load_with_source(p->repo_path);
|
||||
bool source_matches = cbm_pipeline_project_source_matches(p, fresh_config);
|
||||
cbm_userconfig_free(fresh_config);
|
||||
struct stat final_st;
|
||||
int rc = 0;
|
||||
if (!source_matches) {
|
||||
cbm_log_error("pipeline.err", "phase", "test_declarations_source_changed");
|
||||
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
} else if (stat(final_path, &final_st) != 0) {
|
||||
/* The destination vanished during the run: someone else owns it. */
|
||||
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
} else {
|
||||
rc = export_after_publish(p, final_path);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
/* Per-index attribution: peaks and phase totals are about THIS index, not
|
||||
* the process history, so they start clean here. The first mark opens
|
||||
@@ -4571,21 +4666,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
bool backup_succeeded = false;
|
||||
if (final_existed) {
|
||||
backup_succeeded = cbm_store_backup_path(final_path, staging_path) == CBM_STORE_OK;
|
||||
if (!backup_succeeded) {
|
||||
cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path",
|
||||
final_path);
|
||||
/* The copy is gone but the NAME stays ours: the rebuilt
|
||||
* generation is renamed over it by the inner finalize and then
|
||||
* published from it below, so its lock is held to the end. */
|
||||
remove_stage_files(staging_path);
|
||||
}
|
||||
}
|
||||
|
||||
/* The existing generation is copied into the stage only when routing
|
||||
* needs a copy (materialize_stage_backup): an exact no-op is proven on
|
||||
* the live generation and copies nothing -- on a kernel-scale index the
|
||||
* copy alone was a 19 GB write per unchanged re-index. */
|
||||
p->final_existed = final_existed;
|
||||
p->existing_generation = final_existed && backup_succeeded;
|
||||
p->existing_generation = final_existed;
|
||||
char *configured_db_path = p->db_path;
|
||||
p->db_path = strdup(staging_path);
|
||||
if (!p->db_path) {
|
||||
@@ -4595,10 +4681,19 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
free(final_path);
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
p->stage_backup_pending = final_existed;
|
||||
p->backup_succeeded = false;
|
||||
p->live_noop = false;
|
||||
p->live_db_path = final_path;
|
||||
cbm_log_debug("pipeline.setup", "step", "staged");
|
||||
int rc = cbm_pipeline_run_staged(p);
|
||||
free(p->db_path);
|
||||
p->db_path = configured_db_path;
|
||||
bool backup_succeeded = p->backup_succeeded;
|
||||
bool live_noop = p->live_noop;
|
||||
p->stage_backup_pending = false;
|
||||
p->live_noop = false;
|
||||
p->live_db_path = NULL;
|
||||
|
||||
/* Report WHY the run stopped. Everything below happens before the final
|
||||
* rename, so the live database is still the previous generation and an
|
||||
@@ -4612,12 +4707,17 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
|
||||
pipeline_release_test_config(p);
|
||||
return rc;
|
||||
}
|
||||
if (check_cancel(p)) {
|
||||
/* A cancelled run and a live no-op both end here without publishing:
|
||||
* the stage placeholder is discarded, and only the no-op goes on to
|
||||
* finish on the live generation. */
|
||||
bool cancelled = check_cancel(p);
|
||||
if (cancelled || live_noop) {
|
||||
cleanup_staging_db(staging_path);
|
||||
free(staging_path);
|
||||
rc = cancelled ? CBM_PIPELINE_ABORT_PRESERVE_DB : finish_live_noop(p, final_path);
|
||||
free(final_path);
|
||||
pipeline_release_test_config(p);
|
||||
return CBM_PIPELINE_ABORT_PRESERVE_DB;
|
||||
return rc;
|
||||
}
|
||||
if (seal_staging_db(staging_path) != 0) {
|
||||
cleanup_staging_db(staging_path);
|
||||
|
||||
@@ -3743,6 +3743,62 @@ static bool incr_doc_links_current(cbm_store_t *store, const char *project) {
|
||||
return current;
|
||||
}
|
||||
|
||||
/* The exact no-op test: the stored generation's metadata is current for this
|
||||
* mode, its doc-link layer is usable, and its recorded semantic manifest
|
||||
* equals the baseline byte for byte. Read-only. The two out flags report the
|
||||
* halves the staged route also needs for its own routing. */
|
||||
static bool incr_generation_exact(cbm_pipeline_t *p, cbm_store_t *store, const char *project,
|
||||
const cbm_file_hash_t *stored, int stored_count,
|
||||
const cbm_file_hash_t *baseline_manifest, int baseline_count,
|
||||
bool *metadata_current_out, bool *doc_links_current_out) {
|
||||
cbm_coverage_meta_t meta = {0};
|
||||
int meta_rc = cbm_store_coverage_meta_get(store, project, &meta);
|
||||
const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p));
|
||||
bool metadata_current =
|
||||
meta_rc == CBM_STORE_OK && meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION &&
|
||||
meta.hash_records_complete && meta.index_mode && strcmp(meta.index_mode, mode_name) == 0;
|
||||
/* A generation whose doc-link layer is missing or failed is not
|
||||
* current even with identical inputs: index_status tells the user to
|
||||
* re-run, and the re-run has to rebuild. */
|
||||
bool doc_links_current = metadata_current && incr_doc_links_current(store, project);
|
||||
bool exact = doc_links_current && cbm_pipeline_semantic_manifests_equal(
|
||||
stored, stored_count, baseline_manifest, baseline_count);
|
||||
cbm_store_coverage_meta_clear(&meta);
|
||||
*metadata_current_out = metadata_current;
|
||||
*doc_links_current_out = doc_links_current;
|
||||
return exact;
|
||||
}
|
||||
|
||||
bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store,
|
||||
const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count) {
|
||||
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
|
||||
/* A forced legacy-partial run must reach the staged route that takes
|
||||
* the one-shot flag; answering here would consume the run without it. */
|
||||
if (atomic_load(&g_incr_test_force_legacy_partial)) {
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
const char *project = cbm_pipeline_project_name(p);
|
||||
cbm_file_hash_t *stored = NULL;
|
||||
int stored_count = 0;
|
||||
if (!store ||
|
||||
cbm_store_get_file_hashes(store, project, &stored, &stored_count) != CBM_STORE_OK) {
|
||||
return false;
|
||||
}
|
||||
bool metadata_current = false;
|
||||
bool doc_links_current = false;
|
||||
bool exact = incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest,
|
||||
baseline_count, &metadata_current, &doc_links_current);
|
||||
cbm_store_free_file_hashes(stored, stored_count);
|
||||
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
|
||||
if (exact) {
|
||||
incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
}
|
||||
#endif
|
||||
return exact;
|
||||
}
|
||||
|
||||
int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
|
||||
int file_count, const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count, bool force_full_on_mismatch) {
|
||||
@@ -3788,21 +3844,11 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
* rebuilds the complete graph. This avoids partial LSP/package/config
|
||||
* convergence gaps while retaining the common unchanged-repo fast path. */
|
||||
if (!force_legacy_partial) {
|
||||
cbm_coverage_meta_t meta = {0};
|
||||
int meta_rc = cbm_store_coverage_meta_get(store, project, &meta);
|
||||
const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p));
|
||||
bool metadata_current = meta_rc == CBM_STORE_OK &&
|
||||
meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION &&
|
||||
meta.hash_records_complete && meta.index_mode &&
|
||||
strcmp(meta.index_mode, mode_name) == 0;
|
||||
/* A generation whose doc-link layer is missing or failed is not
|
||||
* current even with identical inputs: index_status tells the user to
|
||||
* re-run, and the re-run has to rebuild. */
|
||||
bool doc_links_current = metadata_current && incr_doc_links_current(store, project);
|
||||
bool exact = doc_links_current &&
|
||||
cbm_pipeline_semantic_manifests_equal(stored, stored_count, baseline_manifest,
|
||||
baseline_count);
|
||||
cbm_store_coverage_meta_clear(&meta);
|
||||
bool metadata_current = false;
|
||||
bool doc_links_current = false;
|
||||
bool exact =
|
||||
incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest,
|
||||
baseline_count, &metadata_current, &doc_links_current);
|
||||
if (exact) {
|
||||
cbm_store_free_file_hashes(stored, stored_count);
|
||||
cbm_store_close(store);
|
||||
|
||||
@@ -1482,6 +1482,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
int file_count, const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count, bool force_full_on_mismatch);
|
||||
|
||||
/* The exact no-op test cbm_pipeline_run_incremental routes by, read-only on
|
||||
* an already opened generation: true when that generation is current for the
|
||||
* pipeline's mode and records exactly the baseline manifest. */
|
||||
bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store,
|
||||
const cbm_file_hash_t *baseline_manifest,
|
||||
int baseline_count);
|
||||
|
||||
/* Exact semantic inputs for no-op/forced-full routing. The manifest contains
|
||||
* every discovered source plus repository controls actually consumed by
|
||||
* discovery, package mapping, path aliases, and extension overrides. */
|
||||
|
||||
@@ -0,0 +1,837 @@
|
||||
/*
|
||||
* graph_mem.c — read-only in-memory adjacency over a published store's edges.
|
||||
* See graph_mem.h for the identity, lifetime and budget contract.
|
||||
*/
|
||||
#include "store/graph_mem.h"
|
||||
#include "foundation/constants.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/mem.h"
|
||||
#include "foundation/mem_core.h"
|
||||
#include "foundation/platform.h"
|
||||
|
||||
#include <sqlite3.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
enum {
|
||||
GM_MAX_TYPES = 64, /* one bit each in cbm_graph_mem_types_t */
|
||||
GM_VERSION_MAX = 96, /* generation + file state, see store_graph_mem */
|
||||
GM_CAP_DIVISOR = 4, /* all layers together: at most budget / 4 */
|
||||
GM_SET_MIN_CAP = 64, /* BFS visited set: initial slots (power of two) */
|
||||
GM_SET_LOAD_NUM = 7, /* grow the visited set past 7/10 load */
|
||||
GM_SET_LOAD_DEN = 10,
|
||||
GM_GROWTH = 2,
|
||||
GM_COL_TARGET = 1,
|
||||
GM_COL_TYPE = 2,
|
||||
};
|
||||
|
||||
/* Ids and edge counts must fit the uint32 CSR with room for "+1" offsets. */
|
||||
#define GM_ID_LIMIT ((int64_t)INT32_MAX - 1)
|
||||
#define GM_HASH_MUL 2654435761U
|
||||
|
||||
struct cbm_graph_mem {
|
||||
/* Registry state, guarded by the registry mutex. */
|
||||
struct cbm_graph_mem *next;
|
||||
char *path;
|
||||
char version[GM_VERSION_MAX];
|
||||
int refs;
|
||||
bool ready; /* built; false while its builder runs */
|
||||
bool detached; /* unlinked from the registry; freed at the last release */
|
||||
uint64_t last_use;
|
||||
uint64_t idle_since_ms; /* when the last reference went away */
|
||||
size_t bytes; /* reserved estimate while building, actual once ready */
|
||||
|
||||
/* The graph, immutable once ready. Offsets cover node ids 0..n_ids-1:
|
||||
* the out-edges of id u are out_nbr/out_type[out_off[u] .. out_off[u+1]). */
|
||||
uint32_t n_ids;
|
||||
uint32_t n_edges;
|
||||
uint32_t *out_off;
|
||||
uint32_t *out_nbr; /* target of each out-edge */
|
||||
uint8_t *out_type;
|
||||
uint32_t *in_off;
|
||||
uint32_t *in_nbr; /* source of each in-edge */
|
||||
uint8_t *in_type;
|
||||
int type_count;
|
||||
char *types[GM_MAX_TYPES];
|
||||
size_t type_len[GM_MAX_TYPES];
|
||||
};
|
||||
|
||||
static cbm_graph_mem_t *g_layers; /* linked (not detached) layers */
|
||||
static size_t g_total_bytes; /* every live layer, linked or detached */
|
||||
static uint64_t g_use_clock; /* logical LRU clock */
|
||||
static uint64_t g_builds;
|
||||
|
||||
/* A static SQLite mutex needs no init and works on every platform SQLite
|
||||
* builds on (SQLITE_THREADSAFE=1). store.c owns APP1. */
|
||||
static sqlite3_mutex *gm_registry_mutex(void) {
|
||||
return sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2);
|
||||
}
|
||||
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
static uint64_t (*g_clock_seam)(void);
|
||||
static size_t g_cached_bytes_seam;
|
||||
void cbm_graph_mem_set_clock_for_testing(uint64_t (*now_ms)(void)) {
|
||||
g_clock_seam = now_ms;
|
||||
}
|
||||
void cbm_graph_mem_set_cached_bytes_for_testing(size_t bytes) {
|
||||
g_cached_bytes_seam = bytes;
|
||||
}
|
||||
static uint64_t gm_now_ms(void) {
|
||||
return g_clock_seam ? g_clock_seam() : cbm_now_ms();
|
||||
}
|
||||
#else
|
||||
static uint64_t gm_now_ms(void) {
|
||||
return cbm_now_ms();
|
||||
}
|
||||
#endif
|
||||
|
||||
static void *gm_alloc(size_t bytes) {
|
||||
return cbm_alloc(CBM_MEM_CLASS_GRAPH_CACHE, bytes ? bytes : 1);
|
||||
}
|
||||
|
||||
static void gm_free_ptr(void *p) {
|
||||
cbm_free(CBM_MEM_CLASS_GRAPH_CACHE, p);
|
||||
}
|
||||
|
||||
static void gm_free_graph(cbm_graph_mem_t *g) {
|
||||
gm_free_ptr(g->out_off);
|
||||
gm_free_ptr(g->out_nbr);
|
||||
gm_free_ptr(g->out_type);
|
||||
gm_free_ptr(g->in_off);
|
||||
gm_free_ptr(g->in_nbr);
|
||||
gm_free_ptr(g->in_type);
|
||||
g->out_off = g->out_nbr = g->in_off = g->in_nbr = NULL;
|
||||
g->out_type = g->in_type = NULL;
|
||||
for (int i = 0; i < g->type_count; i++) {
|
||||
gm_free_ptr(g->types[i]);
|
||||
g->types[i] = NULL;
|
||||
}
|
||||
g->type_count = 0;
|
||||
}
|
||||
|
||||
static void gm_destroy(cbm_graph_mem_t *g) {
|
||||
gm_free_graph(g);
|
||||
gm_free_ptr(g->path);
|
||||
gm_free_ptr(g);
|
||||
}
|
||||
|
||||
/* ── Registry (caller holds the mutex) ───────────────────────────── */
|
||||
|
||||
static cbm_graph_mem_t *gm_find(const char *path, const char *version) {
|
||||
for (cbm_graph_mem_t *g = g_layers; g; g = g->next) {
|
||||
if (strcmp(g->path, path) == 0 && strcmp(g->version, version) == 0) {
|
||||
return g;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void gm_unlink(cbm_graph_mem_t *g) {
|
||||
for (cbm_graph_mem_t **p = &g_layers; *p; p = &(*p)->next) {
|
||||
if (*p == g) {
|
||||
*p = g->next;
|
||||
g->next = NULL;
|
||||
g->detached = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Unlink a layer; free it now when nobody holds it, else at its last release. */
|
||||
static void gm_retire(cbm_graph_mem_t *g) {
|
||||
gm_unlink(g);
|
||||
if (g->refs == 0) {
|
||||
g_total_bytes -= g->bytes;
|
||||
gm_destroy(g);
|
||||
}
|
||||
}
|
||||
|
||||
/* Older versions of a path are never asked for again by a new store (one
|
||||
* opened later reads the current version), so they go now. */
|
||||
static void gm_retire_other_versions(const char *path, const char *version) {
|
||||
cbm_graph_mem_t *g = g_layers;
|
||||
while (g) {
|
||||
cbm_graph_mem_t *next = g->next;
|
||||
if (strcmp(g->path, path) == 0 && strcmp(g->version, version) != 0) {
|
||||
gm_retire(g);
|
||||
}
|
||||
g = next;
|
||||
}
|
||||
}
|
||||
|
||||
/* Make room for `bytes` under the cap by evicting idle layers, oldest first.
|
||||
* False when it cannot fit: the cap is unset (no budget) or every remaining
|
||||
* byte is held by a request. */
|
||||
static bool gm_make_room(size_t bytes) {
|
||||
size_t budget = cbm_mem_budget();
|
||||
size_t cap = budget / GM_CAP_DIVISOR;
|
||||
if (cap == 0 || bytes > cap) {
|
||||
return false;
|
||||
}
|
||||
while (g_total_bytes + bytes > cap) {
|
||||
cbm_graph_mem_t *victim = NULL;
|
||||
for (cbm_graph_mem_t *g = g_layers; g; g = g->next) {
|
||||
if (g->ready && g->refs == 0 && (!victim || g->last_use < victim->last_use)) {
|
||||
victim = g;
|
||||
}
|
||||
}
|
||||
if (!victim) {
|
||||
return false;
|
||||
}
|
||||
gm_retire(victim);
|
||||
}
|
||||
/* The cap bounds the layers; the process as a whole must still fit. */
|
||||
return cbm_mem_charged() + bytes <= budget;
|
||||
}
|
||||
|
||||
/* Retire every layer no request has held for CBM_GRAPH_MEM_IDLE_EVICT_MS.
|
||||
* Returns the bytes given back. */
|
||||
static size_t gm_evict_expired_locked(uint64_t now) {
|
||||
size_t freed = 0;
|
||||
cbm_graph_mem_t *g = g_layers;
|
||||
while (g) {
|
||||
cbm_graph_mem_t *next = g->next;
|
||||
if (g->ready && g->refs == 0 && now >= g->idle_since_ms &&
|
||||
now - g->idle_since_ms >= CBM_GRAPH_MEM_IDLE_EVICT_MS) {
|
||||
freed += g->bytes;
|
||||
gm_retire(g);
|
||||
}
|
||||
g = next;
|
||||
}
|
||||
return freed;
|
||||
}
|
||||
|
||||
static void gm_log_evicted(size_t freed) {
|
||||
if (freed > 0) {
|
||||
char bytes[CBM_SZ_32];
|
||||
(void)snprintf(bytes, sizeof(bytes), "%zu", freed);
|
||||
cbm_log_info("graph_mem.evict", "reason", "idle", "bytes", bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Build ──────────────────────────────────────────────────────── */
|
||||
|
||||
static bool gm_bounds(sqlite3 *db, int64_t *max_node, int64_t *max_edge) {
|
||||
sqlite3_stmt *st = NULL;
|
||||
if (sqlite3_prepare_v2(db, "SELECT (SELECT max(id) FROM nodes), (SELECT max(id) FROM edges);",
|
||||
-1, &st, NULL) != SQLITE_OK) {
|
||||
sqlite3_finalize(st);
|
||||
return false;
|
||||
}
|
||||
bool ok = sqlite3_step(st) == SQLITE_ROW;
|
||||
if (ok) {
|
||||
*max_node = sqlite3_column_int64(st, 0);
|
||||
*max_edge = sqlite3_column_int64(st, 1);
|
||||
}
|
||||
sqlite3_finalize(st);
|
||||
return ok && *max_node >= 0 && *max_edge >= 0 && *max_node < GM_ID_LIMIT &&
|
||||
*max_edge < GM_ID_LIMIT;
|
||||
}
|
||||
|
||||
/* Upper bound of a layer's bytes: edge ids are unique positive rowids, so the
|
||||
* edge count never exceeds max(edges.id). */
|
||||
static size_t gm_estimate(int64_t max_node, int64_t max_edge) {
|
||||
size_t offsets = ((size_t)max_node + 2) * sizeof(uint32_t);
|
||||
size_t per_edge = sizeof(uint32_t) + sizeof(uint8_t);
|
||||
return GM_GROWTH * offsets + GM_GROWTH * (size_t)max_edge * per_edge;
|
||||
}
|
||||
|
||||
static int gm_intern(cbm_graph_mem_t *g, const unsigned char *text, size_t len, int *last) {
|
||||
if (*last >= 0 && g->type_len[*last] == len && memcmp(g->types[*last], text, len) == 0) {
|
||||
return *last;
|
||||
}
|
||||
for (int i = 0; i < g->type_count; i++) {
|
||||
if (g->type_len[i] == len && memcmp(g->types[i], text, len) == 0) {
|
||||
*last = i;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
if (g->type_count >= GM_MAX_TYPES) {
|
||||
return -1;
|
||||
}
|
||||
char *copy = gm_alloc(len + 1);
|
||||
if (!copy) {
|
||||
return -1;
|
||||
}
|
||||
memcpy(copy, text, len);
|
||||
copy[len] = '\0';
|
||||
g->types[g->type_count] = copy;
|
||||
g->type_len[g->type_count] = len;
|
||||
*last = g->type_count;
|
||||
return g->type_count++;
|
||||
}
|
||||
|
||||
static bool gm_grow_u32(uint32_t **arr, size_t *cap, size_t need) {
|
||||
if (need <= *cap) {
|
||||
return true;
|
||||
}
|
||||
size_t next = *cap ? *cap : 1;
|
||||
while (next < need) {
|
||||
next *= GM_GROWTH;
|
||||
}
|
||||
uint32_t *grown = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, *arr, next * sizeof(uint32_t));
|
||||
if (!grown) {
|
||||
return false;
|
||||
}
|
||||
*arr = grown;
|
||||
*cap = next;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The out-CSR, filled straight from the scan: rows arrive ordered by
|
||||
* source_id, so each node's list is contiguous and its start offset is the
|
||||
* edge count when its first row (or a later id's) shows up. */
|
||||
typedef struct {
|
||||
size_t off_cap;
|
||||
size_t edge_cap;
|
||||
uint32_t next_start; /* smallest id whose start offset is not set yet */
|
||||
uint32_t max_id; /* largest endpoint seen */
|
||||
int64_t prev_src;
|
||||
int last_type;
|
||||
} gm_scan_t;
|
||||
|
||||
static bool gm_scan_row(cbm_graph_mem_t *g, gm_scan_t *sc, sqlite3_stmt *st) {
|
||||
int64_t src = sqlite3_column_int64(st, 0);
|
||||
int64_t dst = sqlite3_column_int64(st, GM_COL_TARGET);
|
||||
const unsigned char *type = sqlite3_column_text(st, GM_COL_TYPE);
|
||||
if (src < 0 || dst < 0 || src >= GM_ID_LIMIT || dst >= GM_ID_LIMIT || src < sc->prev_src ||
|
||||
!type || g->n_edges >= (uint32_t)GM_ID_LIMIT) {
|
||||
return false;
|
||||
}
|
||||
int type_id = gm_intern(g, type, (size_t)sqlite3_column_bytes(st, GM_COL_TYPE), &sc->last_type);
|
||||
if (type_id < 0 || !gm_grow_u32(&g->out_off, &sc->off_cap, (size_t)src + GM_GROWTH)) {
|
||||
return false;
|
||||
}
|
||||
while (sc->next_start <= (uint32_t)src) {
|
||||
g->out_off[sc->next_start++] = g->n_edges;
|
||||
}
|
||||
if (g->n_edges >= sc->edge_cap) {
|
||||
size_t next = sc->edge_cap ? sc->edge_cap * GM_GROWTH : 1;
|
||||
uint32_t *nbr = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_nbr, next * sizeof(uint32_t));
|
||||
if (!nbr) {
|
||||
return false;
|
||||
}
|
||||
g->out_nbr = nbr;
|
||||
uint8_t *types = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_type, next);
|
||||
if (!types) {
|
||||
return false;
|
||||
}
|
||||
g->out_type = types;
|
||||
sc->edge_cap = next;
|
||||
}
|
||||
g->out_nbr[g->n_edges] = (uint32_t)dst;
|
||||
g->out_type[g->n_edges] = (uint8_t)type_id;
|
||||
g->n_edges++;
|
||||
sc->prev_src = src;
|
||||
if ((uint32_t)src > sc->max_id) {
|
||||
sc->max_id = (uint32_t)src;
|
||||
}
|
||||
if ((uint32_t)dst > sc->max_id) {
|
||||
sc->max_id = (uint32_t)dst;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The in-CSR from the out-CSR: count, prefix-sum, then place each edge using
|
||||
* in_off[v] as v's cursor, which leaves in_off shifted by one id — shift it
|
||||
* back. Iterating sources in id order keeps every in-list sorted by source. */
|
||||
static bool gm_build_inbound(cbm_graph_mem_t *g) {
|
||||
size_t n_ids = g->n_ids;
|
||||
g->in_off = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, (n_ids + 1) * sizeof(uint32_t));
|
||||
g->in_nbr = gm_alloc((size_t)g->n_edges * sizeof(uint32_t));
|
||||
g->in_type = gm_alloc(g->n_edges);
|
||||
if (!g->in_off || !g->in_nbr || !g->in_type) {
|
||||
return false;
|
||||
}
|
||||
for (uint32_t e = 0; e < g->n_edges; e++) {
|
||||
g->in_off[g->out_nbr[e] + 1]++;
|
||||
}
|
||||
for (size_t v = 0; v < n_ids; v++) {
|
||||
g->in_off[v + 1] += g->in_off[v];
|
||||
}
|
||||
for (uint32_t u = 0; u < g->n_ids; u++) {
|
||||
for (uint32_t e = g->out_off[u]; e < g->out_off[u + 1]; e++) {
|
||||
uint32_t pos = g->in_off[g->out_nbr[e]]++;
|
||||
g->in_nbr[pos] = u;
|
||||
g->in_type[pos] = g->out_type[e];
|
||||
}
|
||||
}
|
||||
for (size_t v = n_ids; v > 0; v--) {
|
||||
g->in_off[v] = g->in_off[v - 1];
|
||||
}
|
||||
g->in_off[0] = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool gm_build(cbm_graph_mem_t *g, sqlite3 *db, int64_t max_node, int64_t max_edge) {
|
||||
gm_scan_t sc = {.prev_src = 0, .last_type = -1, .max_id = (uint32_t)max_node};
|
||||
sc.edge_cap = (size_t)max_edge;
|
||||
g->out_nbr = gm_alloc(sc.edge_cap * sizeof(uint32_t));
|
||||
g->out_type = gm_alloc(sc.edge_cap);
|
||||
if (!g->out_nbr || !g->out_type ||
|
||||
!gm_grow_u32(&g->out_off, &sc.off_cap, (size_t)max_node + GM_GROWTH)) {
|
||||
return false;
|
||||
}
|
||||
/* ORDER BY the covering UNIQUE index's leading columns: a sequential walk
|
||||
* of sqlite_autoindex_edges_1, no sort and no table access. */
|
||||
sqlite3_stmt *st = NULL;
|
||||
if (sqlite3_prepare_v2(db,
|
||||
"SELECT source_id, target_id, type FROM edges "
|
||||
"ORDER BY source_id, target_id, type;",
|
||||
-1, &st, NULL) != SQLITE_OK) {
|
||||
sqlite3_finalize(st);
|
||||
return false;
|
||||
}
|
||||
int rc;
|
||||
bool ok = true;
|
||||
while ((rc = sqlite3_step(st)) == SQLITE_ROW) {
|
||||
if (!gm_scan_row(g, &sc, st)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(st);
|
||||
if (!ok || rc != SQLITE_DONE) {
|
||||
return false;
|
||||
}
|
||||
g->n_ids = sc.max_id + 1;
|
||||
if (!gm_grow_u32(&g->out_off, &sc.off_cap, (size_t)g->n_ids + 1)) {
|
||||
return false;
|
||||
}
|
||||
while (sc.next_start <= g->n_ids) {
|
||||
g->out_off[sc.next_start++] = g->n_edges;
|
||||
}
|
||||
/* max(edges.id) over-counts after deletions; give the slack back. */
|
||||
if (g->n_edges < sc.edge_cap) {
|
||||
size_t keep = g->n_edges ? g->n_edges : 1;
|
||||
uint32_t *nbr = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_nbr, keep * sizeof(uint32_t));
|
||||
uint8_t *types = nbr ? cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_type, keep) : NULL;
|
||||
if (nbr) {
|
||||
g->out_nbr = nbr;
|
||||
}
|
||||
if (types) {
|
||||
g->out_type = types;
|
||||
}
|
||||
}
|
||||
return gm_build_inbound(g);
|
||||
}
|
||||
|
||||
static size_t gm_actual_bytes(const cbm_graph_mem_t *g) {
|
||||
size_t offsets = ((size_t)g->n_ids + 1) * sizeof(uint32_t);
|
||||
size_t per_edge = sizeof(uint32_t) + sizeof(uint8_t);
|
||||
size_t bytes = GM_GROWTH * offsets + GM_GROWTH * (size_t)g->n_edges * per_edge;
|
||||
for (int i = 0; i < g->type_count; i++) {
|
||||
bytes += g->type_len[i] + 1;
|
||||
}
|
||||
return bytes + sizeof(*g) + strlen(g->path) + 1;
|
||||
}
|
||||
|
||||
/* ── Acquire / release ──────────────────────────────────────────── */
|
||||
|
||||
/* A ready hit takes a reference; a hit still being built answers NULL (the
|
||||
* caller uses SQL rather than waiting on another thread's build). */
|
||||
static cbm_graph_mem_t *gm_take(cbm_graph_mem_t *hit) {
|
||||
if (!hit->ready) {
|
||||
return NULL;
|
||||
}
|
||||
hit->refs++;
|
||||
hit->last_use = ++g_use_clock;
|
||||
return hit;
|
||||
}
|
||||
|
||||
static cbm_graph_mem_t *gm_new_entry(const char *db_path, const char *version, size_t est) {
|
||||
cbm_graph_mem_t *g = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, sizeof(*g));
|
||||
if (!g) {
|
||||
return NULL;
|
||||
}
|
||||
g->path = cbm_mem_strdup(CBM_MEM_CLASS_GRAPH_CACHE, db_path);
|
||||
if (!g->path) {
|
||||
gm_free_ptr(g);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(g->version, version, strlen(version) + 1);
|
||||
g->refs = 1;
|
||||
g->bytes = est;
|
||||
return g;
|
||||
}
|
||||
|
||||
cbm_graph_mem_t *cbm_graph_mem_acquire(sqlite3 *db, const char *db_path, const char *version) {
|
||||
if (!db || !db_path || !version || !version[0] || strlen(version) >= GM_VERSION_MAX) {
|
||||
return NULL;
|
||||
}
|
||||
uint64_t now = gm_now_ms();
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
size_t freed = gm_evict_expired_locked(now);
|
||||
cbm_graph_mem_t *hit = gm_find(db_path, version);
|
||||
cbm_graph_mem_t *taken = hit ? gm_take(hit) : NULL;
|
||||
sqlite3_mutex_leave(mu);
|
||||
gm_log_evicted(freed);
|
||||
if (hit) {
|
||||
return taken;
|
||||
}
|
||||
|
||||
int64_t max_node = 0;
|
||||
int64_t max_edge = 0;
|
||||
if (!gm_bounds(db, &max_node, &max_edge)) {
|
||||
return NULL;
|
||||
}
|
||||
size_t est = gm_estimate(max_node, max_edge);
|
||||
|
||||
sqlite3_mutex_enter(mu);
|
||||
hit = gm_find(db_path, version);
|
||||
if (hit) {
|
||||
taken = gm_take(hit);
|
||||
sqlite3_mutex_leave(mu);
|
||||
return taken;
|
||||
}
|
||||
gm_retire_other_versions(db_path, version);
|
||||
bool fits = gm_make_room(est);
|
||||
cbm_graph_mem_t *g = fits ? gm_new_entry(db_path, version, est) : NULL;
|
||||
if (g) {
|
||||
g->next = g_layers;
|
||||
g_layers = g;
|
||||
g_total_bytes += est;
|
||||
}
|
||||
sqlite3_mutex_leave(mu);
|
||||
if (!g) {
|
||||
if (!fits) {
|
||||
char need[CBM_SZ_32];
|
||||
(void)snprintf(need, sizeof(need), "%zu", est);
|
||||
cbm_log_info("graph_mem.skip", "reason", "budget", "bytes", need);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* The build runs outside the mutex: other keys stay servable, and a
|
||||
* request for this key meanwhile takes the SQL path (gm_take). */
|
||||
uint64_t started_ms = cbm_now_ms();
|
||||
bool ok = gm_build(g, db, max_node, max_edge);
|
||||
if (!ok) {
|
||||
gm_free_graph(g); /* all-or-nothing: no partial layer survives */
|
||||
}
|
||||
size_t actual = ok ? gm_actual_bytes(g) : 0;
|
||||
if (ok) {
|
||||
char edges[CBM_SZ_32];
|
||||
char bytes[CBM_SZ_32];
|
||||
char elapsed[CBM_SZ_32];
|
||||
(void)snprintf(edges, sizeof(edges), "%u", g->n_edges);
|
||||
(void)snprintf(bytes, sizeof(bytes), "%zu", actual);
|
||||
(void)snprintf(elapsed, sizeof(elapsed), "%llu",
|
||||
(unsigned long long)(cbm_now_ms() - started_ms));
|
||||
cbm_log_info("graph_mem.build", "edges", edges, "bytes", bytes, "elapsed_ms", elapsed);
|
||||
}
|
||||
|
||||
sqlite3_mutex_enter(mu);
|
||||
g_total_bytes = g_total_bytes - g->bytes + actual;
|
||||
g->bytes = actual;
|
||||
if (ok) {
|
||||
g->ready = true;
|
||||
g->last_use = ++g_use_clock;
|
||||
g_builds++;
|
||||
} else {
|
||||
g->refs = 0;
|
||||
gm_retire(g);
|
||||
g = NULL;
|
||||
}
|
||||
sqlite3_mutex_leave(mu);
|
||||
return g;
|
||||
}
|
||||
|
||||
void cbm_graph_mem_release(cbm_graph_mem_t *g) {
|
||||
if (!g) {
|
||||
return;
|
||||
}
|
||||
uint64_t now = gm_now_ms();
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
g->refs--;
|
||||
if (g->refs == 0) {
|
||||
g->idle_since_ms = now;
|
||||
}
|
||||
if (g->refs == 0 && g->detached) {
|
||||
g_total_bytes -= g->bytes;
|
||||
gm_destroy(g);
|
||||
}
|
||||
sqlite3_mutex_leave(mu);
|
||||
}
|
||||
|
||||
void cbm_graph_mem_evict_expired(void) {
|
||||
uint64_t now = gm_now_ms();
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
size_t freed = gm_evict_expired_locked(now);
|
||||
sqlite3_mutex_leave(mu);
|
||||
gm_log_evicted(freed);
|
||||
}
|
||||
|
||||
/* ── Queries ────────────────────────────────────────────────────── */
|
||||
|
||||
cbm_graph_mem_types_t cbm_graph_mem_type_mask(const cbm_graph_mem_t *g, const char *const *types,
|
||||
int count) {
|
||||
cbm_graph_mem_types_t mask = 0;
|
||||
for (int i = 0; g && types && i < count; i++) {
|
||||
if (!types[i]) {
|
||||
continue;
|
||||
}
|
||||
size_t len = strlen(types[i]);
|
||||
for (int t = 0; t < g->type_count; t++) {
|
||||
if (g->type_len[t] == len && memcmp(g->types[t], types[i], len) == 0) {
|
||||
mask |= (cbm_graph_mem_types_t)1 << t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
int cbm_graph_mem_degree(const cbm_graph_mem_t *g, int64_t id, bool inbound,
|
||||
cbm_graph_mem_types_t mask) {
|
||||
if (!g || id < 0 || id >= (int64_t)g->n_ids) {
|
||||
return 0;
|
||||
}
|
||||
const uint32_t *off = inbound ? g->in_off : g->out_off;
|
||||
const uint8_t *type = inbound ? g->in_type : g->out_type;
|
||||
int n = 0;
|
||||
for (uint32_t e = off[id]; e < off[id + 1]; e++) {
|
||||
n += (int)((mask >> type[e]) & 1U);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
size_t cbm_graph_mem_bytes(const cbm_graph_mem_t *g) {
|
||||
return g ? g->bytes : 0;
|
||||
}
|
||||
|
||||
/* Visited set for one BFS: open addressing over id + 1 (0 marks a free slot). */
|
||||
typedef struct {
|
||||
uint32_t *slots;
|
||||
size_t cap;
|
||||
size_t count;
|
||||
} gm_set_t;
|
||||
|
||||
static size_t gm_set_home(uint32_t id, size_t mask) {
|
||||
return (size_t)(id * GM_HASH_MUL) & mask;
|
||||
}
|
||||
|
||||
static bool gm_set_grow(gm_set_t *s) {
|
||||
size_t cap = s->cap ? s->cap * GM_GROWTH : GM_SET_MIN_CAP;
|
||||
uint32_t *slots = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, cap * sizeof(uint32_t));
|
||||
if (!slots) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < s->cap; i++) {
|
||||
uint32_t key = s->slots[i];
|
||||
if (key) {
|
||||
size_t j = gm_set_home(key - 1, cap - 1);
|
||||
while (slots[j]) {
|
||||
j = (j + 1) & (cap - 1);
|
||||
}
|
||||
slots[j] = key;
|
||||
}
|
||||
}
|
||||
gm_free_ptr(s->slots);
|
||||
s->slots = slots;
|
||||
s->cap = cap;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* 1 = added, 0 = already present, -1 = out of memory. */
|
||||
static int gm_set_add(gm_set_t *s, uint32_t id) {
|
||||
if ((s->count + 1) * GM_SET_LOAD_DEN > s->cap * GM_SET_LOAD_NUM && !gm_set_grow(s)) {
|
||||
return -1;
|
||||
}
|
||||
size_t mask = s->cap - 1;
|
||||
for (size_t i = gm_set_home(id, mask);; i = (i + 1) & mask) {
|
||||
if (s->slots[i] == 0) {
|
||||
s->slots[i] = id + 1;
|
||||
s->count++;
|
||||
return 1;
|
||||
}
|
||||
if (s->slots[i] == id + 1) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
uint32_t *ids;
|
||||
size_t count;
|
||||
size_t cap;
|
||||
} gm_list_t;
|
||||
|
||||
static bool gm_list_push(gm_list_t *l, uint32_t id) {
|
||||
if (!gm_grow_u32(&l->ids, &l->cap, l->count + 1)) {
|
||||
return false;
|
||||
}
|
||||
l->ids[l->count++] = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
cbm_graph_mem_hop_t *rows;
|
||||
size_t count;
|
||||
size_t cap;
|
||||
} gm_hops_t;
|
||||
|
||||
static bool gm_hops_push(gm_hops_t *h, uint32_t id, int hop) {
|
||||
if (h->count == h->cap) {
|
||||
size_t next = h->cap ? h->cap * GM_GROWTH : GM_SET_MIN_CAP;
|
||||
cbm_graph_mem_hop_t *rows =
|
||||
cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, h->rows, next * sizeof(cbm_graph_mem_hop_t));
|
||||
if (!rows) {
|
||||
return false;
|
||||
}
|
||||
h->rows = rows;
|
||||
h->cap = next;
|
||||
}
|
||||
h->rows[h->count].id = id;
|
||||
h->rows[h->count].hop = hop;
|
||||
h->count++;
|
||||
return true;
|
||||
}
|
||||
|
||||
static int gm_hop_id_cmp(const void *pa, const void *pb) {
|
||||
const cbm_graph_mem_hop_t *a = pa;
|
||||
const cbm_graph_mem_hop_t *b = pb;
|
||||
return (a->id > b->id) - (a->id < b->id);
|
||||
}
|
||||
|
||||
/* One BFS level: expand every frontier node over the masked edges. A node is
|
||||
* first reached at its shortest distance; the start is never re-entered (its
|
||||
* neighbours were all reached at hop 1), only recorded once at the length of
|
||||
* the shortest cycle back to it — the CTE's (start, hop>0) row. */
|
||||
typedef struct {
|
||||
const uint32_t *off;
|
||||
const uint32_t *nbr;
|
||||
const uint8_t *type;
|
||||
cbm_graph_mem_types_t mask;
|
||||
uint32_t start;
|
||||
bool start_back;
|
||||
gm_set_t seen;
|
||||
gm_hops_t out;
|
||||
} gm_walk_t;
|
||||
|
||||
static bool gm_expand_level(gm_walk_t *w, const gm_list_t *frontier, gm_list_t *next, int hop) {
|
||||
for (size_t f = 0; f < frontier->count; f++) {
|
||||
uint32_t u = frontier->ids[f];
|
||||
for (uint32_t e = w->off[u]; e < w->off[u + 1]; e++) {
|
||||
if (((w->mask >> w->type[e]) & 1U) == 0) {
|
||||
continue;
|
||||
}
|
||||
uint32_t v = w->nbr[e];
|
||||
if (v == w->start) {
|
||||
if (!w->start_back) {
|
||||
w->start_back = true;
|
||||
if (!gm_hops_push(&w->out, v, hop)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
int added = gm_set_add(&w->seen, v);
|
||||
if (added < 0) {
|
||||
return false;
|
||||
}
|
||||
if (added > 0 && (!gm_hops_push(&w->out, v, hop) || !gm_list_push(next, v))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool cbm_graph_mem_bfs(const cbm_graph_mem_t *g, int64_t start, bool inbound,
|
||||
cbm_graph_mem_types_t mask, int max_depth, cbm_graph_mem_hop_t **out,
|
||||
int *count) {
|
||||
*out = NULL;
|
||||
*count = 0;
|
||||
if (!g || max_depth < 1 || mask == 0 || start < 0 || start >= (int64_t)g->n_ids) {
|
||||
return true; /* nothing reachable: the CTE yields only the anchor */
|
||||
}
|
||||
gm_walk_t w = {
|
||||
.off = inbound ? g->in_off : g->out_off,
|
||||
.nbr = inbound ? g->in_nbr : g->out_nbr,
|
||||
.type = inbound ? g->in_type : g->out_type,
|
||||
.mask = mask,
|
||||
.start = (uint32_t)start,
|
||||
};
|
||||
gm_list_t frontier = {0};
|
||||
gm_list_t next = {0};
|
||||
bool ok = gm_list_push(&frontier, w.start);
|
||||
for (int hop = 1; ok && hop <= max_depth && frontier.count > 0; hop++) {
|
||||
size_t level_begin = w.out.count;
|
||||
next.count = 0;
|
||||
ok = gm_expand_level(&w, &frontier, &next, hop);
|
||||
if (ok && w.out.count - level_begin > 1) {
|
||||
qsort(w.out.rows + level_begin, w.out.count - level_begin, sizeof(cbm_graph_mem_hop_t),
|
||||
gm_hop_id_cmp);
|
||||
}
|
||||
gm_list_t swap = frontier;
|
||||
frontier = next;
|
||||
next = swap;
|
||||
}
|
||||
gm_free_ptr(frontier.ids);
|
||||
gm_free_ptr(next.ids);
|
||||
gm_free_ptr(w.seen.slots);
|
||||
if (!ok || w.out.count > (size_t)INT32_MAX) {
|
||||
gm_free_ptr(w.out.rows);
|
||||
return false;
|
||||
}
|
||||
*out = w.out.rows;
|
||||
*count = (int)w.out.count;
|
||||
return true;
|
||||
}
|
||||
|
||||
void cbm_graph_mem_hops_free(cbm_graph_mem_hop_t *hops) {
|
||||
gm_free_ptr(hops);
|
||||
}
|
||||
|
||||
/* ── Registry introspection ─────────────────────────────────────── */
|
||||
|
||||
int cbm_graph_mem_cached_count(void) {
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
int n = 0;
|
||||
for (cbm_graph_mem_t *g = g_layers; g; g = g->next) {
|
||||
n++;
|
||||
}
|
||||
sqlite3_mutex_leave(mu);
|
||||
return n;
|
||||
}
|
||||
|
||||
size_t cbm_graph_mem_cached_bytes(void) {
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
if (g_cached_bytes_seam) {
|
||||
return g_cached_bytes_seam;
|
||||
}
|
||||
#endif
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
size_t bytes = g_total_bytes;
|
||||
sqlite3_mutex_leave(mu);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
uint64_t cbm_graph_mem_build_count(void) {
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
uint64_t n = g_builds;
|
||||
sqlite3_mutex_leave(mu);
|
||||
return n;
|
||||
}
|
||||
|
||||
void cbm_graph_mem_drop_idle(void) {
|
||||
sqlite3_mutex *mu = gm_registry_mutex();
|
||||
sqlite3_mutex_enter(mu);
|
||||
cbm_graph_mem_t *g = g_layers;
|
||||
while (g) {
|
||||
cbm_graph_mem_t *next = g->next;
|
||||
if (g->ready && g->refs == 0) {
|
||||
gm_retire(g);
|
||||
}
|
||||
g = next;
|
||||
}
|
||||
sqlite3_mutex_leave(mu);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* graph_mem.h — read-only in-memory adjacency over a published store's edges.
|
||||
*
|
||||
* A graph walk on a kernel-scale index (8.5M nodes, 16.9M edges) spends its
|
||||
* time in SQLite B-tree seeks: trace_path's recursive CTE took 6.6 s and a
|
||||
* degree-filtered search_graph 3.9 s, where the same questions on django take
|
||||
* 20-100 ms. This layer reads the edges ONCE, in one sequential scan of the
|
||||
* covering UNIQUE(source_id, target_id, type, ...) index, into a typed CSR
|
||||
* (node id -> out/in neighbours, each with its interned edge type) and answers
|
||||
* those walks from RAM. Node rows still come from SQLite; only the edge
|
||||
* structure is held.
|
||||
*
|
||||
* Identity: a layer is keyed by (db path, version), where the caller promises
|
||||
* that one version names one edge set. store.c builds the version from the
|
||||
* store generation (every publish stamps a new one) plus the rollback-journal
|
||||
* file state (the header change counter every commit bumps — it also catches
|
||||
* writers that mutate a published DB in place without a new generation), and
|
||||
* admits only read-only query stores of a sealed, non-legacy DB.
|
||||
*
|
||||
* Lifetime: the daemon closes its query stores after every request, so layers
|
||||
* live in a process-wide registry, shared by refcount across request threads.
|
||||
* Every byte is allocated through the memory core (CBM_MEM_CLASS_GRAPH_CACHE)
|
||||
* and the registry is bounded: the layers together stay under a quarter of
|
||||
* the memory budget, unheld ones are evicted oldest-first to make room, and a
|
||||
* layer that still would not fit (or that would push the process over its
|
||||
* budget) is not built — the caller keeps the SQL path. A build is
|
||||
* all-or-nothing: a failed or refused build leaves no partial layer behind.
|
||||
* A layer nobody held for CBM_GRAPH_MEM_IDLE_EVICT_MS is evicted on the next
|
||||
* request (no timer thread), and the daemon subtracts the resident bytes from
|
||||
* what it hands index workers (cbm_graph_mem_cached_bytes).
|
||||
*/
|
||||
#ifndef CBM_GRAPH_MEM_H
|
||||
#define CBM_GRAPH_MEM_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
struct sqlite3;
|
||||
|
||||
typedef struct cbm_graph_mem cbm_graph_mem_t;
|
||||
|
||||
/* A set of interned edge types: bit i = the layer's type i. */
|
||||
typedef uint64_t cbm_graph_mem_types_t;
|
||||
|
||||
/* One reached node: its id and its shortest distance from the start. */
|
||||
typedef struct {
|
||||
int64_t id;
|
||||
int hop;
|
||||
} cbm_graph_mem_hop_t;
|
||||
|
||||
/* The layer for (db_path, version): the cached one, or one built now from
|
||||
* `db`'s edges table. The caller must hold one read transaction across the
|
||||
* version read and this call, so the scan sees that version's edges.
|
||||
* NULL means "use the SQL path": no budget, over the cap, another thread is
|
||||
* building this key right now, a schema or id range the layer cannot hold, or
|
||||
* a failed scan. A non-NULL layer is held until cbm_graph_mem_release. */
|
||||
cbm_graph_mem_t *cbm_graph_mem_acquire(struct sqlite3 *db, const char *db_path,
|
||||
const char *version);
|
||||
void cbm_graph_mem_release(cbm_graph_mem_t *g);
|
||||
|
||||
/* The type set naming `types` (exact byte match, like SQL `type IN (...)`).
|
||||
* NULL entries and types absent from the graph contribute nothing. */
|
||||
cbm_graph_mem_types_t cbm_graph_mem_type_mask(const cbm_graph_mem_t *g, const char *const *types,
|
||||
int count);
|
||||
|
||||
/* Shortest-walk BFS with the exact semantics of store_bfs's recursive CTE:
|
||||
* every node reached by a walk of 1..max_depth edges whose types are in
|
||||
* `mask`, at its MINIMAL such length. The start itself appears only when a
|
||||
* cycle leads back to it (at the cycle's length). Rows come sorted by
|
||||
* (hop, id); ids absent from the nodes table are NOT filtered here.
|
||||
* Returns false only on allocation failure. *out is freed with
|
||||
* cbm_graph_mem_hops_free. */
|
||||
bool cbm_graph_mem_bfs(const cbm_graph_mem_t *g, int64_t start, bool inbound,
|
||||
cbm_graph_mem_types_t mask, int max_depth, cbm_graph_mem_hop_t **out,
|
||||
int *count);
|
||||
void cbm_graph_mem_hops_free(cbm_graph_mem_hop_t *hops);
|
||||
|
||||
/* Number of edges with a type in `mask` ending at (inbound) or leaving
|
||||
* (outbound) node `id` — SQL's COUNT(*) over target_id/source_id = id. */
|
||||
int cbm_graph_mem_degree(const cbm_graph_mem_t *g, int64_t id, bool inbound,
|
||||
cbm_graph_mem_types_t mask);
|
||||
|
||||
/* Bytes this layer holds (all through the memory core). */
|
||||
size_t cbm_graph_mem_bytes(const cbm_graph_mem_t *g);
|
||||
|
||||
/* ── Idle eviction ──────────────────────────────────────────────── */
|
||||
|
||||
/* How long a layer may sit unheld before it goes. Ten minutes outlasts the
|
||||
* gaps between an agent's graph calls within one task (seconds to a few
|
||||
* minutes), so a working session keeps its layer, while a session that moved
|
||||
* on stops pinning it (237 MB for the kernel). A wrong guess costs one rebuild
|
||||
* — about 1.2-1.5 s on the 17 GB kernel index, tens of ms on ordinary repos —
|
||||
* against ten minutes of held memory. */
|
||||
enum { CBM_GRAPH_MEM_IDLE_EVICT_MS = 10 * 60 * 1000 };
|
||||
|
||||
/* Evict every layer unheld for CBM_GRAPH_MEM_IDLE_EVICT_MS. The daemon calls
|
||||
* it at the end of every request and acquire runs it too: a daemon that stops
|
||||
* asking stops paying, with no timer thread. */
|
||||
void cbm_graph_mem_evict_expired(void);
|
||||
|
||||
/* ── Registry introspection (tests, diagnostics) ─────────────────── */
|
||||
|
||||
/* Layers currently held by the registry, and their total bytes — every
|
||||
* resident layer, held or not, which is what index-worker budgets give up. */
|
||||
int cbm_graph_mem_cached_count(void);
|
||||
size_t cbm_graph_mem_cached_bytes(void);
|
||||
/* Builds completed since process start: a test's proof that a call hit the
|
||||
* cache (unchanged) or built (incremented). */
|
||||
uint64_t cbm_graph_mem_build_count(void);
|
||||
/* Drop every unheld layer (refs == 0), however recently used. */
|
||||
void cbm_graph_mem_drop_idle(void);
|
||||
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
/* The clock idle eviction reads; NULL restores cbm_now_ms. Tests move time
|
||||
* instead of waiting for it. */
|
||||
void cbm_graph_mem_set_clock_for_testing(uint64_t (*now_ms)(void));
|
||||
/* Pin cbm_graph_mem_cached_bytes to `bytes`; 0 restores the live reading. */
|
||||
void cbm_graph_mem_set_cached_bytes_for_testing(size_t bytes);
|
||||
#endif
|
||||
|
||||
#endif /* CBM_GRAPH_MEM_H */
|
||||
+828
-75
File diff suppressed because it is too large
Load Diff
@@ -478,6 +478,16 @@ int cbm_store_commit(cbm_store_t *s);
|
||||
/* Rollback the current transaction. */
|
||||
int cbm_store_rollback(cbm_store_t *s);
|
||||
|
||||
/* Read batch: run the reads that follow in one deferred read transaction,
|
||||
* renewed every 100 ms so a writer is never held back longer, instead of one
|
||||
* lock round trip per statement. Begin returns false (and does nothing) when
|
||||
* a transaction is already open. For read-only work only; end it before any
|
||||
* write, transaction or read scope on the same store. Close ends it. */
|
||||
bool cbm_store_read_batch_begin(cbm_store_t *s);
|
||||
void cbm_store_read_batch_end(cbm_store_t *s);
|
||||
/* Test seam: make an open read batch due for renewal at its next statement. */
|
||||
void cbm_store_read_batch_expire_for_test(cbm_store_t *s);
|
||||
|
||||
/* Stable request read scope. Borrows an exclusively owned, single-thread
|
||||
* store and callback context. Refuses a transaction, busy statement or owned
|
||||
* progress handler; idle cached statements are allowed. Saves query_only and
|
||||
@@ -615,6 +625,29 @@ int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const cha
|
||||
int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char *file_path,
|
||||
cbm_node_t **out, int *count);
|
||||
|
||||
/* Streamed node scan: the rows of cbm_store_find_nodes (label NULL) or
|
||||
* cbm_store_find_nodes_by_label, in the same order, a batch at a time. Pages
|
||||
* are keyset queries along the eager query's index, and no read lock is held
|
||||
* between calls. open returns NULL on error. next fills out[0..max) and sets
|
||||
* *count; a count of 0 means the scan is exhausted. The caller frees each
|
||||
* returned node with cbm_node_free_fields. A failed read returns
|
||||
* CBM_STORE_ERR with no rows from that call and ends the scan;
|
||||
* cbm_store_error() names the failure. paged is false when the eager query's
|
||||
* plan is not the one the pages reproduce: the scan then hands out the eager
|
||||
* scan's rows (same order, no memory saving). Close the scan before closing
|
||||
* the store. */
|
||||
typedef struct cbm_store_node_scan cbm_store_node_scan_t;
|
||||
cbm_store_node_scan_t *cbm_store_node_scan_open(cbm_store_t *s, const char *project,
|
||||
const char *label);
|
||||
int cbm_store_node_scan_next(cbm_store_node_scan_t *scan, cbm_node_t *out, int max, int *count);
|
||||
bool cbm_store_node_scan_paged(const cbm_store_node_scan_t *scan);
|
||||
void cbm_store_node_scan_close(cbm_store_node_scan_t *scan);
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
/* One-shot fault: after this many successful pages (of any scan), the next
|
||||
* page fails like a SQLite step error; -1 disables the seam. */
|
||||
void cbm_store_node_scan_test_fail_page_after(int successful_pages);
|
||||
#endif
|
||||
|
||||
/* Return a stable, paginated outline for one exact repository-relative file.
|
||||
* File/folder/container nodes are excluded. labels may be NULL when
|
||||
* label_count is zero; otherwise labels are exact-match filters. The query is
|
||||
@@ -1210,6 +1243,13 @@ char *cbm_glob_to_like(const char *pattern);
|
||||
* Each out[i] is malloc'd — caller must free each string. */
|
||||
int cbm_extract_like_hints(const char *pattern, char **out, int max_out);
|
||||
|
||||
/* Literal prefix every match of an anchored POSIX ERE must begin with: the
|
||||
* plain printable-ASCII run after a leading '^', minus its last character when
|
||||
* a quantifier or escape follows it. Returns its length (written NUL-terminated
|
||||
* to out); 0 when the pattern is not anchored, contains '|', or starts with an
|
||||
* operator. Case is kept as written. */
|
||||
int cbm_regex_anchored_prefix(const char *pattern, char *out, int out_sz);
|
||||
|
||||
/* Prepend (?i) to a regex pattern if not already present.
|
||||
* Returns a static buffer — do NOT free. */
|
||||
const char *cbm_ensure_case_insensitive(const char *pattern);
|
||||
|
||||
@@ -5580,6 +5580,518 @@ TEST(cypher_fifty_union_branches) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── ORDER BY, DISTINCT and grouping against a reference ───────────
|
||||
* These were bubble sorts and linear duplicate/group scans — seconds to
|
||||
* minutes on a 30k-row result. The results must stay exactly the old ones: a
|
||||
* stable sort of the unsorted rows, first-occurrence DISTINCT, first-seen
|
||||
* group order. Each case derives that reference from the engine's own
|
||||
* unsorted rows, on a graph built for ties, duplicates and large groups. */
|
||||
|
||||
enum { REF_NODES = 400, REF_NAMES = 97 };
|
||||
|
||||
static cbm_store_t *setup_reference_store(void) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
int64_t ids[REF_NODES];
|
||||
for (int i = 0; i < REF_NODES; i++) {
|
||||
char name[32];
|
||||
char qn[32];
|
||||
char file[32];
|
||||
snprintf(name, sizeof(name), "fn_%02d", (i * 7) % REF_NAMES); /* repeated names */
|
||||
snprintf(qn, sizeof(qn), "test.n%d", i);
|
||||
snprintf(file, sizeof(file), "f%02d.c", i % 13);
|
||||
cbm_node_t n = {.project = "test",
|
||||
.label = "Function",
|
||||
.name = name,
|
||||
.qualified_name = qn,
|
||||
.file_path = file,
|
||||
.start_line = (i * 31) % 50, /* tied lines */
|
||||
.end_line = 60};
|
||||
ids[i] = cbm_store_upsert_node(s, &n);
|
||||
}
|
||||
/* Every node calls both hubs (n0, n1: ~400 callers, 97 distinct names,
|
||||
* past the hashed COUNT(DISTINCT) threshold) and one more node. */
|
||||
for (int i = 0; i < REF_NODES; i++) {
|
||||
int targets[3] = {0, 1, (i * i + 3) % REF_NODES};
|
||||
for (int t = 0; t < 3; t++) {
|
||||
if (targets[t] != i) {
|
||||
cbm_edge_t e = {.project = "test",
|
||||
.source_id = ids[i],
|
||||
.target_id = ids[targets[t]],
|
||||
.type = "CALLS"};
|
||||
cbm_store_insert_edge(s, &e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
typedef int (*ref_cmp_fn)(const cbm_cypher_result_t *r, int a, int b);
|
||||
|
||||
/* The reference order: an insertion sort that moves a row before an earlier
|
||||
* one only when the earlier one compares greater — stable, like the old
|
||||
* bubble sorts. */
|
||||
static void ref_stable_order(const cbm_cypher_result_t *r, int *order, ref_cmp_fn cmp) {
|
||||
for (int i = 0; i < r->row_count; i++) {
|
||||
order[i] = i;
|
||||
}
|
||||
for (int i = 1; i < r->row_count; i++) {
|
||||
for (int j = i; j > 0 && cmp(r, order[j - 1], order[j]) > 0; j--) {
|
||||
int swap = order[j];
|
||||
order[j] = order[j - 1];
|
||||
order[j - 1] = swap;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Columns 0 (name) and 1 (line): line descending, then name ascending. */
|
||||
static int ref_line_desc_name(const cbm_cypher_result_t *r, int a, int b) {
|
||||
long la = strtol(r->rows[a][1], NULL, 10);
|
||||
long lb = strtol(r->rows[b][1], NULL, 10);
|
||||
int cmp = (lb > la) - (lb < la);
|
||||
return cmp != 0 ? cmp : strcmp(r->rows[a][0], r->rows[b][0]);
|
||||
}
|
||||
|
||||
static int ref_all_equal(const cbm_cypher_result_t *r, int a, int b) {
|
||||
(void)r;
|
||||
(void)a;
|
||||
(void)b;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* `sorted` must hold the rows of `unsorted` in the reference order. */
|
||||
static bool ref_rows_in_order(const cbm_cypher_result_t *sorted,
|
||||
const cbm_cypher_result_t *unsorted, ref_cmp_fn cmp) {
|
||||
if (sorted->row_count != unsorted->row_count || sorted->col_count != unsorted->col_count ||
|
||||
unsorted->row_count < REF_NODES) {
|
||||
return false;
|
||||
}
|
||||
int order[REF_NODES * 4];
|
||||
ref_stable_order(unsorted, order, cmp);
|
||||
for (int i = 0; i < sorted->row_count; i++) {
|
||||
for (int c = 0; c < sorted->col_count; c++) {
|
||||
if (strcmp(sorted->rows[i][c], unsorted->rows[order[i]][c]) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ref_query(cbm_store_t *s, const char *query, cbm_cypher_result_t *out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
return cbm_cypher_execute(s, query, "test", 0, out) == 0 && !out->truncated;
|
||||
}
|
||||
|
||||
TEST(cypher_order_by_is_the_stable_reference_order) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t plain;
|
||||
cbm_cypher_result_t sorted;
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.start_line", &plain));
|
||||
|
||||
/* RETURN: numeric key descending, ties broken by the next key. */
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (f:Function) RETURN f.name, f.start_line ORDER BY f.start_line DESC, f.name",
|
||||
&sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
|
||||
/* WITH: the same keys through a projection. */
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (f:Function) WITH f.name AS n, f.start_line AS l "
|
||||
"ORDER BY l DESC, n RETURN n, l",
|
||||
&sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
cbm_cypher_result_free(&plain);
|
||||
|
||||
/* All keys tied: both sorts keep the unsorted order. */
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.label", &plain));
|
||||
ASSERT_TRUE(
|
||||
ref_query(s, "MATCH (f:Function) RETURN f.name, f.label ORDER BY f.label", &sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (f:Function) WITH f.name AS n, f.label AS l ORDER BY l RETURN n, l", &sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
cbm_cypher_result_free(&plain);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* `got` must be the first occurrence of every value in column 0 of `all`. */
|
||||
static bool ref_first_occurrences(const cbm_cypher_result_t *got, const cbm_cypher_result_t *all) {
|
||||
int kept = 0;
|
||||
for (int i = 0; i < all->row_count; i++) {
|
||||
bool seen = false;
|
||||
for (int j = 0; j < i && !seen; j++) {
|
||||
seen = strcmp(all->rows[i][0], all->rows[j][0]) == 0;
|
||||
}
|
||||
if (seen) {
|
||||
continue;
|
||||
}
|
||||
if (kept >= got->row_count || strcmp(got->rows[kept][0], all->rows[i][0]) != 0) {
|
||||
return false;
|
||||
}
|
||||
kept++;
|
||||
}
|
||||
return kept == got->row_count;
|
||||
}
|
||||
|
||||
TEST(cypher_distinct_keeps_first_occurrences) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t all;
|
||||
cbm_cypher_result_t got;
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name", &all));
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN DISTINCT f.name", &got));
|
||||
ASSERT_EQ(got.row_count, REF_NAMES);
|
||||
ASSERT_TRUE(ref_first_occurrences(&got, &all));
|
||||
cbm_cypher_result_free(&got);
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) WITH DISTINCT f.name AS n RETURN n", &got));
|
||||
ASSERT_EQ(got.row_count, REF_NAMES);
|
||||
ASSERT_TRUE(ref_first_occurrences(&got, &all));
|
||||
cbm_cypher_result_free(&got);
|
||||
cbm_cypher_result_free(&all);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* `got` (column 0 = callee, column 1 = count) must list the callees of
|
||||
* `pairs` (column 0 = callee, column 1 = caller name) in first-seen order,
|
||||
* each with its caller count — of distinct names when `distinct`. */
|
||||
static bool ref_groups_match(const cbm_cypher_result_t *got, const cbm_cypher_result_t *pairs,
|
||||
bool distinct) {
|
||||
enum { REF_PAIRS_MAX = REF_NODES * 4 };
|
||||
if (pairs->row_count > REF_PAIRS_MAX) {
|
||||
return false;
|
||||
}
|
||||
bool first_callee[REF_PAIRS_MAX];
|
||||
bool first_pair[REF_PAIRS_MAX];
|
||||
for (int k = 0; k < pairs->row_count; k++) {
|
||||
first_callee[k] = true;
|
||||
first_pair[k] = true;
|
||||
for (int m = 0; m < k; m++) {
|
||||
if (strcmp(pairs->rows[m][0], pairs->rows[k][0]) == 0) {
|
||||
first_callee[k] = false;
|
||||
first_pair[k] = first_pair[k] && strcmp(pairs->rows[m][1], pairs->rows[k][1]) != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
int groups = 0;
|
||||
for (int i = 0; i < pairs->row_count; i++) {
|
||||
if (!first_callee[i]) {
|
||||
continue;
|
||||
}
|
||||
int count = 0;
|
||||
for (int k = 0; k < pairs->row_count; k++) {
|
||||
if (strcmp(pairs->rows[k][0], pairs->rows[i][0]) == 0) {
|
||||
count += !distinct || first_pair[k] ? 1 : 0;
|
||||
}
|
||||
}
|
||||
char want[16];
|
||||
snprintf(want, sizeof(want), "%d", count);
|
||||
if (groups >= got->row_count || strcmp(got->rows[groups][0], pairs->rows[i][0]) != 0 ||
|
||||
strcmp(got->rows[groups][1], want) != 0) {
|
||||
return false;
|
||||
}
|
||||
groups++;
|
||||
}
|
||||
return groups == got->row_count;
|
||||
}
|
||||
|
||||
TEST(cypher_grouping_keeps_first_seen_groups) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t pairs;
|
||||
cbm_cypher_result_t got;
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, a.name", &pairs));
|
||||
ASSERT_GT(pairs.row_count, REF_NODES * 2);
|
||||
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, count(a) AS c", &got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, false));
|
||||
cbm_cypher_result_free(&got);
|
||||
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (a)-[:CALLS]->(b:Function) "
|
||||
"RETURN b.qualified_name, count(DISTINCT a.name) AS c",
|
||||
&got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, true));
|
||||
cbm_cypher_result_free(&got);
|
||||
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (a)-[:CALLS]->(b:Function) WITH b.qualified_name AS q, "
|
||||
"count(DISTINCT a.name) AS c RETURN q, c",
|
||||
&got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, true));
|
||||
cbm_cypher_result_free(&got);
|
||||
cbm_cypher_result_free(&pairs);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Streamed execution: chunks of start nodes give the single pass ──
|
||||
*
|
||||
* execute_single runs the start nodes through the pipeline in chunks rather
|
||||
* than materialising every binding first (kernel count(*): 18 GB RSS). The
|
||||
* chunked run must be indistinguishable from one pass over the whole start
|
||||
* set: same rows in the same order, first-seen group order, ORDER BY ties,
|
||||
* DISTINCT, LIMIT / max_rows truncation and OPTIONAL fallback rows. Each query
|
||||
* runs as one chunk (the reference) and in chunks of 1, 7, 64 and the
|
||||
* production default, so chunk boundaries fall inside groups and inside
|
||||
* every start node's expansion order. */
|
||||
|
||||
static int stream_run(cbm_store_t *s, const char *query, int max_rows, int chunk,
|
||||
cbm_cypher_result_t *out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
cbm_cypher_test_set_stream_chunk(chunk);
|
||||
int rc = cbm_cypher_execute(s, query, "test", max_rows, out);
|
||||
cbm_cypher_test_set_stream_chunk(0);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static bool stream_same_str(const char *a, const char *b) {
|
||||
return (!a && !b) || (a && b && strcmp(a, b) == 0);
|
||||
}
|
||||
|
||||
static bool stream_results_equal(const cbm_cypher_result_t *a, const cbm_cypher_result_t *b) {
|
||||
if (a->col_count != b->col_count || a->row_count != b->row_count ||
|
||||
a->truncated != b->truncated || !stream_same_str(a->warning, b->warning) ||
|
||||
!stream_same_str(a->error, b->error)) {
|
||||
return false;
|
||||
}
|
||||
for (int c = 0; c < a->col_count; c++) {
|
||||
if (strcmp(a->columns[c], b->columns[c]) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int r = 0; r < a->row_count; r++) {
|
||||
for (int c = 0; c < a->col_count; c++) {
|
||||
if (strcmp(a->rows[r][c], b->rows[r][c]) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* True when every chunk size reproduces the single-pass result, which is
|
||||
* left in *ref for the caller's own checks (caller frees). */
|
||||
static bool stream_matches_single_pass(cbm_store_t *s, const char *query, int max_rows,
|
||||
cbm_cypher_result_t *ref) {
|
||||
static const int chunks[] = {1, 7, 64, 0 /* production default */};
|
||||
int ref_rc = stream_run(s, query, max_rows, INT_MAX, ref);
|
||||
bool same = ref_rc == 0;
|
||||
for (size_t i = 0; same && i < sizeof(chunks) / sizeof(chunks[0]); i++) {
|
||||
cbm_cypher_result_t got;
|
||||
int rc = stream_run(s, query, max_rows, chunks[i], &got);
|
||||
same = rc == ref_rc && stream_results_equal(ref, &got);
|
||||
if (!same) {
|
||||
printf(" chunk %d differs from the single pass: %s\n", chunks[i], query);
|
||||
}
|
||||
cbm_cypher_result_free(&got);
|
||||
}
|
||||
return same;
|
||||
}
|
||||
|
||||
/* Every pipeline consumer across chunk boundaries: unsorted rows, global and
|
||||
* grouped aggregates (groups fed from many chunks), DISTINCT, late WHERE,
|
||||
* collect, WITH aggregation and WITH sorting, RETURN * with and without a
|
||||
* WITH, the default projection, edge variables and variable-length hops. */
|
||||
TEST(cypher_stream_chunks_match_single_pass) {
|
||||
static const char *const queries[] = {
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN a.name, b.qualified_name",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN count(*)",
|
||||
"MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, count(a) AS c ORDER BY c DESC",
|
||||
"MATCH (a:Function)-[:CALLS]->(b) RETURN b.name, count(DISTINCT a.name) AS c",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name",
|
||||
"MATCH (a)-[:CALLS]->(b) WHERE b.start_line > 20 RETURN a.name, b.name, b.start_line",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN b.name, collect(a.name) AS cs",
|
||||
("MATCH (a:Function)-[:CALLS]->(b) WITH b, count(a) AS c WHERE c > 1 "
|
||||
"RETURN b.name, c ORDER BY c DESC LIMIT 20"),
|
||||
"MATCH (a)-[:CALLS]->(b) WITH a.name AS n, b.start_line AS l ORDER BY l DESC RETURN n, l",
|
||||
"MATCH (a)-[:CALLS]->(b) WITH DISTINCT b.name AS n RETURN n",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN *",
|
||||
"MATCH (a)-[:CALLS]->(b) WITH a.name AS n RETURN *",
|
||||
"MATCH (a)-[:CALLS]->(b)",
|
||||
"MATCH (a)-[r:CALLS]->(b) RETURN type(r), a.name ORDER BY a.name",
|
||||
"MATCH (a)-[:CALLS*1..2]->(b) RETURN a.name, b.name",
|
||||
};
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
for (size_t i = 0; i < sizeof(queries) / sizeof(queries[0]); i++) {
|
||||
cbm_cypher_result_t ref;
|
||||
ASSERT_TRUE(stream_matches_single_pass(s, queries[i], 0, &ref));
|
||||
ASSERT_GT(ref.row_count, 0);
|
||||
ASSERT_FALSE(ref.truncated);
|
||||
cbm_cypher_result_free(&ref);
|
||||
}
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* OPTIONAL fallback rows (a start node with no match keeps its row, target
|
||||
* unbound) sit between the expanded rows of their neighbours in every chunk
|
||||
* layout; the multi-pattern form keeps its single pass. */
|
||||
TEST(cypher_stream_optional_across_chunks) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t ref;
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.name, b.name", 0, &ref));
|
||||
int fallback = 0;
|
||||
for (int r = 0; r < ref.row_count; r++) {
|
||||
fallback += ref.rows[r][1][0] == '\0' ? 1 : 0;
|
||||
}
|
||||
ASSERT_GT(fallback, 0); /* uncalled nodes keep a row */
|
||||
ASSERT_GT(ref.row_count, fallback); /* and called nodes expand */
|
||||
cbm_cypher_result_free(&ref);
|
||||
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.qualified_name, count(b) AS c", 0, &ref));
|
||||
ASSERT_EQ(ref.row_count, REF_NODES);
|
||||
cbm_cypher_result_free(&ref);
|
||||
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "MATCH (a:Function) OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.name, count(b)", 0, &ref));
|
||||
ASSERT_GT(ref.row_count, 0);
|
||||
cbm_cypher_result_free(&ref);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A LIMIT or the max_rows budget fills up in one chunk while later chunks
|
||||
* still arrive: they must add nothing, and truncation is reported exactly
|
||||
* when the single pass reports it (engine budget yes, explicit LIMIT no). */
|
||||
TEST(cypher_stream_limit_and_max_rows_across_chunks) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t ref;
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name LIMIT 25", 0, &ref));
|
||||
ASSERT_EQ(ref.row_count, 25);
|
||||
ASSERT_FALSE(ref.truncated);
|
||||
cbm_cypher_result_free(&ref);
|
||||
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name LIMIT 30", 0, &ref));
|
||||
ASSERT_EQ(ref.row_count, 30);
|
||||
cbm_cypher_result_free(&ref);
|
||||
|
||||
ASSERT_TRUE(stream_matches_single_pass(
|
||||
s, "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name SKIP 5 LIMIT 10", 0, &ref));
|
||||
ASSERT_EQ(ref.row_count, 10);
|
||||
cbm_cypher_result_free(&ref);
|
||||
|
||||
static const char *const budgeted[] = {
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name ORDER BY b.name",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN *",
|
||||
"MATCH (a)-[:CALLS]->(b)",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(budgeted) / sizeof(budgeted[0]); i++) {
|
||||
ASSERT_TRUE(stream_matches_single_pass(s, budgeted[i], 30, &ref));
|
||||
ASSERT_EQ(ref.row_count, 30);
|
||||
ASSERT_TRUE(ref.truncated);
|
||||
cbm_cypher_result_free(&ref);
|
||||
}
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A start-node scan that fails part-way fails the whole query: the rows read
|
||||
* before the failure must not answer it as if they were all of them. Pages
|
||||
* hold 7 rows at chunk 7; the store seam fails a page after earlier pages
|
||||
* delivered rows (the all-nodes scan's first page is its empty NULL
|
||||
* file_path pass, hence one more page before the fault). */
|
||||
TEST(cypher_stream_scan_failure_fails_query) {
|
||||
static const struct {
|
||||
const char *query;
|
||||
int pages_before_fault;
|
||||
} cases[] = {
|
||||
{"MATCH (a:Function)-[:CALLS]->(b) RETURN a.name, b.name", 1},
|
||||
{"MATCH (a:Function)-[:CALLS]->(b) RETURN count(*)", 1},
|
||||
{"MATCH (n) RETURN n.name", 2},
|
||||
};
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t r;
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
ASSERT_EQ(stream_run(s, cases[i].query, 0, 7, &r), 0);
|
||||
ASSERT_GT(r.row_count, 0);
|
||||
cbm_cypher_result_free(&r);
|
||||
|
||||
cbm_store_node_scan_test_fail_page_after(cases[i].pages_before_fault);
|
||||
int rc = stream_run(s, cases[i].query, 0, 7, &r);
|
||||
cbm_store_node_scan_test_fail_page_after(-1);
|
||||
ASSERT_TRUE(rc != 0);
|
||||
ASSERT_NOT_NULL(r.error);
|
||||
ASSERT_TRUE(strstr(r.error, "reading the graph failed") != NULL);
|
||||
ASSERT_TRUE(strstr(r.error, "injected page failure") != NULL);
|
||||
ASSERT_EQ(r.row_count, 0);
|
||||
cbm_cypher_result_free(&r);
|
||||
}
|
||||
/* The failure does not outlive its query. */
|
||||
ASSERT_EQ(stream_run(s, cases[0].query, 0, 7, &r), 0);
|
||||
ASSERT_GT(r.row_count, 0);
|
||||
ASSERT_NULL(r.error);
|
||||
cbm_cypher_result_free(&r);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The production chunk boundary itself: a start set of 2.5 chunks, a ring of
|
||||
* calls plus five hubs whose caller groups straddle both boundaries. */
|
||||
TEST(cypher_stream_default_chunk_boundary) {
|
||||
enum { STREAM_NODES = 2600, STREAM_HUBS = 5 };
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
static int64_t ids[STREAM_NODES];
|
||||
for (int i = 0; i < STREAM_NODES; i++) {
|
||||
char name[32];
|
||||
char qn[32];
|
||||
snprintf(name, sizeof(name), "fn_%d", i % 211);
|
||||
snprintf(qn, sizeof(qn), "test.s%d", i);
|
||||
cbm_node_t n = {.project = "test",
|
||||
.label = "Function",
|
||||
.name = name,
|
||||
.qualified_name = qn,
|
||||
.file_path = "ring.c",
|
||||
.start_line = i % 37};
|
||||
ids[i] = cbm_store_upsert_node(s, &n);
|
||||
}
|
||||
for (int i = 0; i < STREAM_NODES; i++) {
|
||||
cbm_edge_t ring = {.project = "test",
|
||||
.source_id = ids[i],
|
||||
.target_id = ids[(i + 1) % STREAM_NODES],
|
||||
.type = "CALLS"};
|
||||
cbm_edge_t hub = {.project = "test",
|
||||
.source_id = ids[i],
|
||||
.target_id = ids[i % STREAM_HUBS],
|
||||
.type = "CALLS"};
|
||||
cbm_store_insert_edge(s, &ring);
|
||||
if (i >= STREAM_HUBS) { /* no hub self-loops */
|
||||
cbm_store_insert_edge(s, &hub);
|
||||
}
|
||||
}
|
||||
static const char *const queries[] = {
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN count(*)",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN a.qualified_name, b.qualified_name",
|
||||
"MATCH (a:Function)-[:CALLS]->(b) RETURN b.qualified_name, count(a) AS c ORDER BY c DESC",
|
||||
"MATCH (a)-[:CALLS]->(b) WITH b.name AS n, count(DISTINCT a.name) AS c RETURN n, c",
|
||||
"MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name LIMIT 300",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(queries) / sizeof(queries[0]); i++) {
|
||||
cbm_cypher_result_t ref;
|
||||
ASSERT_TRUE(stream_matches_single_pass(s, queries[i], 0, &ref));
|
||||
ASSERT_GT(ref.row_count, 0);
|
||||
cbm_cypher_result_free(&ref);
|
||||
}
|
||||
cbm_cypher_result_t ref;
|
||||
ASSERT_TRUE(stream_matches_single_pass(s, queries[0], 0, &ref));
|
||||
ASSERT_STR_EQ(ref.rows[0][0], "5195"); /* 2600 ring + 2595 hub edges */
|
||||
cbm_cypher_result_free(&ref);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
SUITE(cypher) {
|
||||
@@ -5823,4 +6335,13 @@ SUITE(cypher) {
|
||||
RUN_TEST(cypher_chain_with_unbound_alias_keeps_seed);
|
||||
RUN_TEST(cypher_many_union_branches);
|
||||
RUN_TEST(cypher_fifty_union_branches);
|
||||
RUN_TEST(cypher_order_by_is_the_stable_reference_order);
|
||||
RUN_TEST(cypher_distinct_keeps_first_occurrences);
|
||||
RUN_TEST(cypher_grouping_keeps_first_seen_groups);
|
||||
/* Streamed execution: chunked start set == single pass */
|
||||
RUN_TEST(cypher_stream_chunks_match_single_pass);
|
||||
RUN_TEST(cypher_stream_optional_across_chunks);
|
||||
RUN_TEST(cypher_stream_limit_and_max_rows_across_chunks);
|
||||
RUN_TEST(cypher_stream_default_chunk_boundary);
|
||||
RUN_TEST(cypher_stream_scan_failure_fails_query);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "mcp/mcp.h"
|
||||
#include "mcp/mcp_internal.h"
|
||||
#include "pipeline/pipeline.h"
|
||||
#include "store/graph_mem.h"
|
||||
#include "store/store.h"
|
||||
#include "ui/config.h"
|
||||
#include "watcher/watcher.h"
|
||||
@@ -5889,6 +5890,32 @@ TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The daemon's resident adjacency layers are its own memory: a worker's slice
|
||||
* is the aggregate minus them, and never "no cap" (0) when they exceed it. The
|
||||
* held bytes come through the graph_mem seam, so no layer has to be built. */
|
||||
TEST(daemon_application_worker_slice_leaves_room_for_graph_cache) {
|
||||
const size_t mib = (size_t)1024U * 1024U;
|
||||
const size_t aggregate = 1000U * mib;
|
||||
cbm_daemon_application_config_t config = {.aggregate_memory_budget_bytes = aggregate};
|
||||
cbm_graph_mem_drop_idle();
|
||||
cbm_daemon_application_t *application = cbm_daemon_application_new(&config);
|
||||
size_t without_layers = cbm_daemon_application_worker_memory_budget_bytes(application);
|
||||
cbm_graph_mem_set_cached_bytes_for_testing(200U * mib);
|
||||
size_t with_layers = cbm_daemon_application_worker_memory_budget_bytes(application);
|
||||
cbm_graph_mem_set_cached_bytes_for_testing(aggregate + 5U);
|
||||
size_t layers_over_aggregate = cbm_daemon_application_worker_memory_budget_bytes(application);
|
||||
cbm_graph_mem_set_cached_bytes_for_testing(0);
|
||||
bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS);
|
||||
cbm_daemon_application_free(application);
|
||||
|
||||
ASSERT_NOT_NULL(application);
|
||||
ASSERT_TRUE(stopped);
|
||||
ASSERT_EQ(without_layers, aggregate);
|
||||
ASSERT_EQ(with_layers, aggregate - 200U * mib);
|
||||
ASSERT_EQ(layers_over_aggregate, 1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Decision A (#1997 #832): the over-budget verdict travels as a CLEAN worker
|
||||
* exit carrying an error response. The daemon passes that response through
|
||||
* verbatim on the first attempt — no crash/hang recovery loop, no quarantine
|
||||
@@ -6960,6 +6987,7 @@ SUITE(daemon_application) {
|
||||
RUN_TEST(daemon_application_queues_explicit_index_behind_physical_job_limit);
|
||||
RUN_TEST(daemon_application_default_limit_admits_four_and_rejects_fifth);
|
||||
RUN_TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs);
|
||||
RUN_TEST(daemon_application_worker_slice_leaves_room_for_graph_cache);
|
||||
RUN_TEST(daemon_application_over_budget_response_passes_through_without_recovery);
|
||||
RUN_TEST(daemon_application_free_reports_retained_live_ownership);
|
||||
RUN_TEST(daemon_application_tool_error_is_failed_job_even_after_clean_worker_exit);
|
||||
|
||||
@@ -440,9 +440,65 @@ TEST(githistory_ignores_inherited_repo_env) {
|
||||
#endif /* _WIN32 */
|
||||
}
|
||||
|
||||
/* ── Branch-only resolve (list_projects) ───────────────────────────
|
||||
* The listing prints only the branch; cbm_git_branch_resolve gets it from the
|
||||
* two git calls cbm_git_context_resolve uses for it, so both must agree. */
|
||||
|
||||
#ifndef _WIN32
|
||||
/* The branch resolver against the full context at `path`; NULL `want` = not git. */
|
||||
static bool branch_resolve_agrees(const char *path, const char *want) {
|
||||
cbm_git_context_t ctx = {0};
|
||||
(void)cbm_git_context_resolve(path, &ctx);
|
||||
bool is_git = false;
|
||||
char branch[CBM_GIT_BRANCH_MAX];
|
||||
(void)cbm_git_branch_resolve(path, &is_git, branch, sizeof(branch));
|
||||
bool agrees = is_git == ctx.is_git && is_git == (want != NULL) &&
|
||||
(want ? (ctx.branch && strcmp(branch, want) == 0 && strcmp(ctx.branch, want) == 0)
|
||||
: branch[0] == '\0');
|
||||
cbm_git_context_free(&ctx);
|
||||
return agrees;
|
||||
}
|
||||
#endif /* _WIN32 */
|
||||
|
||||
TEST(git_branch_resolve_matches_full_context) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("git-based branch test not supported on Windows CI");
|
||||
#else
|
||||
char plain[512];
|
||||
char *tmp = th_mktempdir("cbm_gitbranch_plain");
|
||||
if (!tmp) FAIL("th_mktempdir returned NULL");
|
||||
snprintf(plain, sizeof(plain), "%s", tmp);
|
||||
tmp = th_mktempdir("cbm_gitbranch");
|
||||
if (!tmp) FAIL("th_mktempdir returned NULL");
|
||||
if (make_git_repo(tmp) != 0) {
|
||||
th_rmtree(tmp);
|
||||
th_rmtree(plain);
|
||||
SKIP_PLATFORM("git not available to init a repo");
|
||||
}
|
||||
char sub[600];
|
||||
snprintf(sub, sizeof(sub), "%s/sub", tmp);
|
||||
ASSERT_EQ(th_mkdir_p(sub), 0);
|
||||
char missing[600];
|
||||
snprintf(missing, sizeof(missing), "%s/missing", plain);
|
||||
|
||||
ASSERT_TRUE(branch_resolve_agrees(plain, NULL));
|
||||
ASSERT_TRUE(branch_resolve_agrees(missing, NULL));
|
||||
ASSERT_EQ(git_run(tmp, "checkout -q -b feat/x"), 0);
|
||||
ASSERT_TRUE(branch_resolve_agrees(tmp, "feat/x"));
|
||||
ASSERT_TRUE(branch_resolve_agrees(sub, "feat/x"));
|
||||
ASSERT_EQ(git_run(tmp, "checkout -q --detach"), 0);
|
||||
ASSERT_TRUE(branch_resolve_agrees(tmp, "DETACHED"));
|
||||
|
||||
th_rmtree(tmp);
|
||||
th_rmtree(plain);
|
||||
PASS();
|
||||
#endif /* _WIN32 */
|
||||
}
|
||||
|
||||
/* ── Suite ──────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(git_context) {
|
||||
RUN_TEST(git_branch_resolve_matches_full_context);
|
||||
RUN_TEST(root_exists_non_ascii_directory);
|
||||
RUN_TEST(canonical_root_repo_root);
|
||||
RUN_TEST(canonical_root_subdir);
|
||||
|
||||
@@ -0,0 +1,739 @@
|
||||
/*
|
||||
* test_graph_mem.c — the in-memory adjacency layer (store/graph_mem.h).
|
||||
*
|
||||
* The contract is byte-identical output: a read-only query store answers
|
||||
* BFS and degree filters from the layer, a writable store of the same file
|
||||
* keeps the SQL path, and every row, order, hop, degree, total and flag must
|
||||
* match. The fixture mixes a seeded random graph with the shapes that break a
|
||||
* naive BFS: ties at one hop, a cycle back to the start, a self-loop, a
|
||||
* diamond, duplicate IMPORTS edges, a cross-project edge and edgeless nodes.
|
||||
*/
|
||||
#include "test_framework.h"
|
||||
#include "test_helpers.h"
|
||||
#include <foundation/compat_thread.h>
|
||||
#include <foundation/mem.h>
|
||||
#include <foundation/mem_core.h>
|
||||
#include <mcp/mcp.h>
|
||||
#include <store/graph_mem.h>
|
||||
#include <store/store.h>
|
||||
#include <sqlite3.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
enum {
|
||||
GMT_NODES = 40,
|
||||
GMT_RANDOM_EDGES = 140,
|
||||
GMT_THREADS = 4,
|
||||
GMT_SEED = 20261014,
|
||||
GMT_PATH_MAX = 512,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
char dir[GMT_PATH_MAX];
|
||||
char db[GMT_PATH_MAX];
|
||||
int64_t ids[GMT_NODES];
|
||||
int64_t other_id; /* the cross-project node */
|
||||
} gmt_fixture_t;
|
||||
|
||||
static const char *const gmt_types[] = {"CALLS", "USAGE", "DATA_FLOWS", "IMPORTS"};
|
||||
|
||||
static uint32_t gmt_next(uint32_t *state) {
|
||||
*state = *state * 1103515245U + 12345U;
|
||||
return *state >> 8;
|
||||
}
|
||||
|
||||
static int64_t gmt_node(cbm_store_t *s, const char *project, int i, const char *name) {
|
||||
char qn[64];
|
||||
char file[32];
|
||||
snprintf(qn, sizeof(qn), "%s.mod%d.n%02d", project, i % 5, i);
|
||||
snprintf(file, sizeof(file), "src/m%d.c", i % 5);
|
||||
cbm_node_t n = {.project = project,
|
||||
.label = (i % 7 == 3) ? "Method" : "Function",
|
||||
.name = name,
|
||||
.qualified_name = qn,
|
||||
.file_path = file,
|
||||
.start_line = i,
|
||||
.end_line = i + 3};
|
||||
return cbm_store_upsert_node(s, &n);
|
||||
}
|
||||
|
||||
static void gmt_edge(cbm_store_t *s, int64_t src, int64_t dst, const char *type,
|
||||
const char *props) {
|
||||
cbm_edge_t e = {
|
||||
.project = "p", .source_id = src, .target_id = dst, .type = type, .properties_json = props};
|
||||
(void)cbm_store_insert_edge(s, &e);
|
||||
}
|
||||
|
||||
/* Build the fixture file and stamp a generation, the way a publish does. */
|
||||
static bool gmt_fixture_build(gmt_fixture_t *f) {
|
||||
char *tmp = th_mktempdir("cbm_graph_mem");
|
||||
if (!tmp) {
|
||||
return false;
|
||||
}
|
||||
snprintf(f->dir, sizeof(f->dir), "%s", tmp);
|
||||
snprintf(f->db, sizeof(f->db), "%s/graph.db", f->dir);
|
||||
cbm_store_t *s = cbm_store_open_path(f->db);
|
||||
if (!s || cbm_store_upsert_project(s, "p", "/p") != CBM_STORE_OK ||
|
||||
cbm_store_upsert_project(s, "q", "/q") != CBM_STORE_OK) {
|
||||
cbm_store_close(s);
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < GMT_NODES; i++) {
|
||||
char name[16];
|
||||
/* Every fifth node shares one name: search's (name, id) order has ties. */
|
||||
snprintf(name, sizeof(name), (i % 5 == 4) ? "dup" : "n%02d", i);
|
||||
f->ids[i] = gmt_node(s, "p", i, name);
|
||||
}
|
||||
f->other_id = gmt_node(s, "q", 99, "other");
|
||||
const int64_t *id = f->ids;
|
||||
/* chain with a cycle back to the start (0 at hop 3), a self-loop, a diamond */
|
||||
gmt_edge(s, id[0], id[1], "CALLS", NULL);
|
||||
gmt_edge(s, id[1], id[2], "CALLS", NULL);
|
||||
gmt_edge(s, id[2], id[0], "CALLS", NULL);
|
||||
gmt_edge(s, id[2], id[3], "CALLS", NULL);
|
||||
gmt_edge(s, id[5], id[5], "CALLS", NULL);
|
||||
gmt_edge(s, id[0], id[6], "CALLS", NULL);
|
||||
gmt_edge(s, id[0], id[7], "CALLS", NULL);
|
||||
gmt_edge(s, id[6], id[8], "CALLS", NULL);
|
||||
gmt_edge(s, id[7], id[8], "CALLS", NULL);
|
||||
/* two IMPORTS rows for one (source, target): distinct local names */
|
||||
gmt_edge(s, id[3], id[4], "IMPORTS", "{\"local_name\":\"a\"}");
|
||||
gmt_edge(s, id[3], id[4], "IMPORTS", "{\"local_name\":\"b\"}");
|
||||
/* a cross-project hop: BFS does not filter by project */
|
||||
gmt_edge(s, id[8], f->other_id, "CALLS", NULL);
|
||||
gmt_edge(s, f->other_id, id[9], "CALLS", NULL);
|
||||
/* a hub for the degree filters */
|
||||
for (int i = 10; i < 30; i++) {
|
||||
gmt_edge(s, id[i], id[1], (i % 2) ? "CALLS" : "USAGE", NULL);
|
||||
}
|
||||
/* seeded random edges over the first 34 nodes; 34..39 stay edgeless */
|
||||
uint32_t state = GMT_SEED;
|
||||
for (int k = 0; k < GMT_RANDOM_EDGES; k++) {
|
||||
int a = (int)(gmt_next(&state) % 34U);
|
||||
int b = (int)(gmt_next(&state) % 34U);
|
||||
const char *type = gmt_types[gmt_next(&state) % 4U];
|
||||
gmt_edge(s, id[a], id[b], type, "{\"w\":1}");
|
||||
}
|
||||
/* Stamp and seal, as a publish does: a sealed rollback-journal file is
|
||||
* the only kind of DB the layer serves. */
|
||||
bool ok = cbm_store_generation_advance(s) == CBM_STORE_OK &&
|
||||
cbm_store_prepare_for_publish(s) == CBM_STORE_OK;
|
||||
cbm_store_close(s);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool gmt_reseal(const gmt_fixture_t *f) {
|
||||
cbm_store_t *w = cbm_store_open_path(f->db);
|
||||
bool ok = w && cbm_store_prepare_for_publish(w) == CBM_STORE_OK;
|
||||
cbm_store_close(w);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static void gmt_fixture_free(gmt_fixture_t *f) {
|
||||
cbm_graph_mem_drop_idle();
|
||||
th_rmtree(f->dir);
|
||||
}
|
||||
|
||||
static bool gmt_str_eq(const char *a, const char *b) {
|
||||
return (a == NULL && b == NULL) || (a && b && strcmp(a, b) == 0);
|
||||
}
|
||||
|
||||
static bool gmt_node_eq(const cbm_node_t *a, const cbm_node_t *b) {
|
||||
return a->id == b->id && gmt_str_eq(a->project, b->project) && gmt_str_eq(a->label, b->label) &&
|
||||
gmt_str_eq(a->name, b->name) && gmt_str_eq(a->qualified_name, b->qualified_name) &&
|
||||
gmt_str_eq(a->file_path, b->file_path) && a->start_line == b->start_line &&
|
||||
a->end_line == b->end_line && gmt_str_eq(a->properties_json, b->properties_json);
|
||||
}
|
||||
|
||||
static bool gmt_edge_eq(const cbm_edge_info_t *a, const cbm_edge_info_t *b) {
|
||||
return gmt_str_eq(a->from_name, b->from_name) && gmt_str_eq(a->to_name, b->to_name) &&
|
||||
gmt_str_eq(a->type, b->type) && a->confidence == b->confidence &&
|
||||
a->source_id == b->source_id && a->target_id == b->target_id &&
|
||||
gmt_str_eq(a->properties_json, b->properties_json);
|
||||
}
|
||||
|
||||
static bool gmt_traverse_eq(const cbm_traverse_result_t *a, const cbm_traverse_result_t *b) {
|
||||
if (!gmt_node_eq(&a->root, &b->root) || a->visited_count != b->visited_count ||
|
||||
a->edge_count != b->edge_count || a->truncated != b->truncated ||
|
||||
a->edges_truncated != b->edges_truncated) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < a->visited_count; i++) {
|
||||
if (a->visited[i].hop != b->visited[i].hop ||
|
||||
!gmt_node_eq(&a->visited[i].node, &b->visited[i].node)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < a->edge_count; i++) {
|
||||
if (!gmt_edge_eq(&a->edges[i], &b->edges[i])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
const char *types[3];
|
||||
int count;
|
||||
} gmt_typeset_t;
|
||||
|
||||
static const gmt_typeset_t gmt_typesets[] = {
|
||||
{{NULL}, 0}, /* the default: CALLS */
|
||||
{{"CALLS", "USAGE"}, 2},
|
||||
{{"DATA_FLOWS"}, 1},
|
||||
{{"IMPORTS"}, 1},
|
||||
{{"NO_SUCH_TYPE", "CALLS"}, 2},
|
||||
};
|
||||
|
||||
/* One BFS shape on both stores; prints the shape that diverged. */
|
||||
static bool gmt_bfs_same(cbm_store_t *sql, cbm_store_t *mem, int64_t start, const char *dir,
|
||||
const gmt_typeset_t *ts, int depth, int max_results, int max_edges) {
|
||||
cbm_traverse_result_t a = {0};
|
||||
cbm_traverse_result_t b = {0};
|
||||
const char **types = (const char **)ts->types;
|
||||
int rca = cbm_store_bfs_with_edge_limit(sql, start, dir, types, ts->count, depth, max_results,
|
||||
max_edges, &a);
|
||||
int rcb = cbm_store_bfs_with_edge_limit(mem, start, dir, types, ts->count, depth, max_results,
|
||||
max_edges, &b);
|
||||
bool same = rca == rcb && gmt_traverse_eq(&a, &b);
|
||||
if (!same) {
|
||||
(void)fprintf(stderr,
|
||||
" graph_mem divergence: start=%lld dir=%s types=%d depth=%d max=%d "
|
||||
"edges=%d rc=%d/%d visited=%d/%d\n",
|
||||
(long long)start, dir, ts->count, depth, max_results, max_edges, rca, rcb,
|
||||
a.visited_count, b.visited_count);
|
||||
}
|
||||
cbm_store_traverse_free(&a);
|
||||
cbm_store_traverse_free(&b);
|
||||
return same;
|
||||
}
|
||||
|
||||
/* The runner initializes no memory budget unless the mem suite ran first, and
|
||||
* without one no layer is ever built. Each test pins a budget the 3 KB fixture
|
||||
* layer fits, collects its observations, and restores the budget BEFORE
|
||||
* asserting (the hook's contract: a failed assert must not leak the budget). */
|
||||
static size_t gmt_budget_pin(size_t bytes) {
|
||||
size_t saved = cbm_mem_budget();
|
||||
cbm_mem_set_budget_for_tests(bytes);
|
||||
return saved;
|
||||
}
|
||||
|
||||
static size_t gmt_budget_roomy(void) {
|
||||
return cbm_mem_charged() + ((size_t)1 << 30);
|
||||
}
|
||||
|
||||
static int gmt_visited(cbm_store_t *s, int64_t start) {
|
||||
cbm_traverse_result_t tr = {0};
|
||||
int n = cbm_store_bfs(s, start, "outbound", NULL, 0, 3, 100, &tr) == CBM_STORE_OK
|
||||
? tr.visited_count
|
||||
: -1;
|
||||
cbm_store_traverse_free(&tr);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* One request: open a query store, walk from `start`, close. */
|
||||
static int gmt_request(const char *db, int64_t start) {
|
||||
cbm_store_t *q = cbm_store_open_path_query(db);
|
||||
int n = q ? gmt_visited(q, start) : -1;
|
||||
cbm_store_close(q);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* The SQL reference: a read-only store like the one under test (a writable
|
||||
* open would switch the sealed file to WAL), pinned to the SQL path because
|
||||
* its one layer attempt runs under a budget no layer fits. */
|
||||
static cbm_store_t *gmt_open_sql_reference(const gmt_fixture_t *f) {
|
||||
size_t saved = gmt_budget_pin(64);
|
||||
cbm_store_t *s = cbm_store_open_path_query(f->db);
|
||||
if (s) {
|
||||
(void)gmt_visited(s, f->ids[0]);
|
||||
}
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
return s;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int shapes;
|
||||
int diverged;
|
||||
uint64_t builds;
|
||||
bool start_back; /* the cycle 0->1->2->0 returns node 0 at hop 3 */
|
||||
} gmt_bfs_run_t;
|
||||
|
||||
static void gmt_bfs_all_shapes(const gmt_fixture_t *f, cbm_store_t *sql, cbm_store_t *mem,
|
||||
gmt_bfs_run_t *run) {
|
||||
static const char *const dirs[] = {"outbound", "inbound"};
|
||||
static const int depths[] = {0, 1, 2, 3, 6};
|
||||
for (int n = 0; n <= GMT_NODES; n++) {
|
||||
int64_t start = n < GMT_NODES ? f->ids[n] : f->other_id;
|
||||
for (int d = 0; d < 2; d++) {
|
||||
for (size_t t = 0; t < sizeof(gmt_typesets) / sizeof(gmt_typesets[0]); t++) {
|
||||
for (size_t k = 0; k < sizeof(depths) / sizeof(depths[0]); k++) {
|
||||
const gmt_typeset_t *ts = &gmt_typesets[t];
|
||||
/* lean, with edge data, and saturated (probe row + edge ceiling) */
|
||||
run->diverged += !gmt_bfs_same(sql, mem, start, dirs[d], ts, depths[k], 100, 0);
|
||||
run->diverged +=
|
||||
!gmt_bfs_same(sql, mem, start, dirs[d], ts, depths[k], 100, 1000);
|
||||
run->diverged += !gmt_bfs_same(sql, mem, start, dirs[d], ts, depths[k], 3, 4);
|
||||
run->shapes += 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cbm_traverse_result_t tr = {0};
|
||||
if (cbm_store_bfs(mem, f->ids[0], "outbound", NULL, 0, 3, 100, &tr) == CBM_STORE_OK) {
|
||||
for (int i = 0; i < tr.visited_count; i++) {
|
||||
if (tr.visited[i].node.id == f->ids[0]) {
|
||||
run->start_back = tr.visited[i].hop == 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
cbm_store_traverse_free(&tr);
|
||||
}
|
||||
|
||||
TEST(graph_mem_bfs_matches_sql_on_every_shape) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
cbm_store_t *sql = gmt_open_sql_reference(&f);
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
cbm_store_t *mem = cbm_store_open_path_query(f.db);
|
||||
gmt_bfs_run_t run = {0};
|
||||
if (sql && mem) {
|
||||
gmt_bfs_all_shapes(&f, sql, mem, &run);
|
||||
}
|
||||
run.builds = cbm_graph_mem_build_count() - builds;
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
cbm_store_close(sql);
|
||||
cbm_store_close(mem);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_GT(run.shapes, 0);
|
||||
ASSERT_EQ(run.diverged, 0);
|
||||
/* The layer really answered: one build, for the store under test only. */
|
||||
ASSERT_EQ(run.builds, 1);
|
||||
ASSERT_TRUE(run.start_back);
|
||||
PASS();
|
||||
}
|
||||
|
||||
static bool gmt_search_same(cbm_store_t *sql, cbm_store_t *mem, cbm_search_params_t p) {
|
||||
cbm_search_output_t a = {0};
|
||||
cbm_search_output_t b = {0};
|
||||
int rca = cbm_store_search(sql, &p, &a);
|
||||
int rcb = cbm_store_search(mem, &p, &b);
|
||||
bool same = rca == rcb && a.total == b.total && a.count == b.count;
|
||||
for (int i = 0; same && i < a.count; i++) {
|
||||
same = gmt_node_eq(&a.results[i].node, &b.results[i].node) &&
|
||||
a.results[i].in_degree == b.results[i].in_degree &&
|
||||
a.results[i].out_degree == b.results[i].out_degree;
|
||||
}
|
||||
if (!same) {
|
||||
(void)fprintf(stderr,
|
||||
" graph_mem search divergence: min=%d max=%d limit=%d offset=%d "
|
||||
"total=%d/%d count=%d/%d\n",
|
||||
p.min_degree, p.max_degree, p.limit, p.offset, a.total, b.total, a.count,
|
||||
b.count);
|
||||
}
|
||||
cbm_store_search_free(&a);
|
||||
cbm_store_search_free(&b);
|
||||
return same;
|
||||
}
|
||||
|
||||
static int gmt_search_all_shapes(cbm_store_t *sql, cbm_store_t *mem) {
|
||||
static const int ranges[][2] = {{1, -1}, {3, -1}, {10, -1}, {0, 0}, {-1, 0},
|
||||
{-1, 2}, {2, 5}, {0, -1}, {50, -1}};
|
||||
static const char *const labels[] = {NULL, "Function"};
|
||||
int diverged = 0;
|
||||
for (size_t r = 0; r < sizeof(ranges) / sizeof(ranges[0]); r++) {
|
||||
for (int l = 0; l < 2; l++) {
|
||||
for (int page = 0; page < 3; page++) {
|
||||
cbm_search_params_t p = {.project = "p",
|
||||
.label = labels[l],
|
||||
.min_degree = ranges[r][0],
|
||||
.max_degree = ranges[r][1],
|
||||
.limit = page == 2 ? 100 : 4,
|
||||
.offset = page == 1 ? 3 : 0};
|
||||
diverged += !gmt_search_same(sql, mem, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
cbm_search_params_t named = {
|
||||
.project = "p", .name_pattern = "dup", .min_degree = 1, .max_degree = -1, .limit = 10};
|
||||
diverged += !gmt_search_same(sql, mem, named);
|
||||
return diverged;
|
||||
}
|
||||
|
||||
TEST(graph_mem_degree_search_matches_sql) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
cbm_store_t *sql = gmt_open_sql_reference(&f);
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
|
||||
/* No degree filter: the layer is neither needed nor built. */
|
||||
cbm_store_t *plain = cbm_store_open_path_query(f.db);
|
||||
cbm_search_params_t nofilter = {.project = "p", .min_degree = -1, .max_degree = -1};
|
||||
cbm_search_output_t out = {0};
|
||||
int plain_rc = plain ? cbm_store_search(plain, &nofilter, &out) : CBM_STORE_ERR;
|
||||
cbm_store_search_free(&out);
|
||||
cbm_store_close(plain);
|
||||
uint64_t plain_builds = cbm_graph_mem_build_count() - builds;
|
||||
|
||||
cbm_store_t *mem = cbm_store_open_path_query(f.db);
|
||||
int diverged = sql && mem ? gmt_search_all_shapes(sql, mem) : -1;
|
||||
uint64_t filter_builds = cbm_graph_mem_build_count() - builds;
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
cbm_store_close(sql);
|
||||
cbm_store_close(mem);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_EQ(plain_rc, CBM_STORE_OK);
|
||||
ASSERT_EQ(plain_builds, 0);
|
||||
ASSERT_EQ(diverged, 0);
|
||||
ASSERT_EQ(filter_builds, 1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(graph_mem_generation_change_invalidates) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
int64_t leaf = f.ids[36]; /* edgeless */
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
|
||||
int first = gmt_request(f.db, leaf);
|
||||
uint64_t first_builds = cbm_graph_mem_build_count() - builds;
|
||||
/* A later request store on the same version reuses the layer. */
|
||||
int second = gmt_request(f.db, leaf);
|
||||
uint64_t second_builds = cbm_graph_mem_build_count() - builds;
|
||||
|
||||
/* A mutation stamped with a new generation and sealed: a publish. */
|
||||
cbm_store_t *w = cbm_store_open_path(f.db);
|
||||
if (w) {
|
||||
gmt_edge(w, leaf, f.ids[37], "CALLS", NULL);
|
||||
}
|
||||
bool published = w && cbm_store_generation_advance(w) == CBM_STORE_OK &&
|
||||
cbm_store_prepare_for_publish(w) == CBM_STORE_OK;
|
||||
cbm_store_close(w);
|
||||
|
||||
cbm_store_t *q3 = cbm_store_open_path_query(f.db);
|
||||
int third = q3 ? gmt_visited(q3, leaf) : -1;
|
||||
cbm_store_t *sql = gmt_open_sql_reference(&f);
|
||||
int third_sql = sql ? gmt_visited(sql, leaf) : -1;
|
||||
bool same =
|
||||
sql && q3 && gmt_bfs_same(sql, q3, f.ids[0], "outbound", &gmt_typesets[1], 6, 100, 1000);
|
||||
uint64_t third_builds = cbm_graph_mem_build_count() - builds;
|
||||
/* The old version's layer is retired: one layer for this file. */
|
||||
int cached = cbm_graph_mem_cached_count();
|
||||
cbm_store_close(sql);
|
||||
cbm_store_close(q3);
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
int cached_after = cbm_graph_mem_cached_count();
|
||||
|
||||
ASSERT_EQ(first, 0);
|
||||
ASSERT_EQ(first_builds, 1);
|
||||
ASSERT_EQ(second, 0);
|
||||
ASSERT_EQ(second_builds, 1);
|
||||
ASSERT_TRUE(published);
|
||||
ASSERT_EQ(third, 1);
|
||||
ASSERT_EQ(third_sql, 1);
|
||||
ASSERT_TRUE(same);
|
||||
ASSERT_EQ(third_builds, 2);
|
||||
ASSERT_EQ(cached, 1);
|
||||
ASSERT_EQ(cached_after, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The cross-repo shape: edges written straight into a published file, with
|
||||
* NO new generation. A layer of the old content must never answer for it —
|
||||
* not while the file is in WAL mode, and not after it is sealed again. */
|
||||
TEST(graph_mem_in_place_write_never_serves_stale) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
int64_t leaf = f.ids[36]; /* edgeless */
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
|
||||
int before = gmt_request(f.db, leaf);
|
||||
uint64_t before_builds = cbm_graph_mem_build_count() - builds;
|
||||
|
||||
/* The writer stays open across two commits, so no checkpoint lands
|
||||
* between them: the main file — header included — is the same for both.
|
||||
* Only refusing WAL mode keeps the second request from a stale answer. */
|
||||
cbm_store_t *w = cbm_store_open_path(f.db);
|
||||
if (w) {
|
||||
gmt_edge(w, leaf, f.ids[37], "CALLS", NULL);
|
||||
}
|
||||
int wal_one = gmt_request(f.db, leaf);
|
||||
if (w) {
|
||||
gmt_edge(w, leaf, f.ids[38], "CALLS", NULL);
|
||||
}
|
||||
int wal_two = gmt_request(f.db, leaf);
|
||||
uint64_t wal_builds = cbm_graph_mem_build_count() - builds;
|
||||
cbm_store_close(w);
|
||||
|
||||
bool resealed = gmt_reseal(&f);
|
||||
int after = gmt_request(f.db, leaf);
|
||||
uint64_t after_builds = cbm_graph_mem_build_count() - builds;
|
||||
int cached = cbm_graph_mem_cached_count();
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_NOT_NULL(w);
|
||||
ASSERT_EQ(before, 0);
|
||||
ASSERT_EQ(before_builds, 1);
|
||||
/* WAL mode: the SQL path answers, with each new edge. */
|
||||
ASSERT_EQ(wal_one, 1);
|
||||
ASSERT_EQ(wal_two, 2);
|
||||
ASSERT_EQ(wal_builds, 1);
|
||||
/* Sealed again under the SAME generation: a new version, a new layer. */
|
||||
ASSERT_TRUE(resealed);
|
||||
ASSERT_EQ(after, 2);
|
||||
ASSERT_EQ(after_builds, 2);
|
||||
ASSERT_EQ(cached, 1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(graph_mem_over_budget_keeps_sql_path) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
cbm_store_t *sql = gmt_open_sql_reference(&f);
|
||||
/* A budget whose quarter cannot hold the layer: no build, SQL answers. The
|
||||
* file stays sealed, so the budget is the only reason to refuse. */
|
||||
size_t saved = gmt_budget_pin(64);
|
||||
cbm_store_t *mem = cbm_store_open_path_query(f.db);
|
||||
bool same =
|
||||
sql && mem && gmt_bfs_same(sql, mem, f.ids[0], "outbound", &gmt_typesets[1], 3, 100, 1000);
|
||||
cbm_search_params_t p = {.project = "p", .min_degree = 3, .max_degree = -1, .limit = 50};
|
||||
bool same_search = sql && mem && gmt_search_same(sql, mem, p);
|
||||
uint64_t builds_after = cbm_graph_mem_build_count();
|
||||
int cached = cbm_graph_mem_cached_count();
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
cbm_store_close(sql);
|
||||
cbm_store_close(mem);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_TRUE(same);
|
||||
ASSERT_TRUE(same_search);
|
||||
ASSERT_EQ(builds_after, builds);
|
||||
ASSERT_EQ(cached, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(graph_mem_legacy_generation_keeps_sql_path) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
/* A DB that predates generation tracking cannot detect staleness. The raw
|
||||
* connection leaves the file in rollback mode: legacy is the only reason
|
||||
* to refuse. */
|
||||
sqlite3 *raw = NULL;
|
||||
ASSERT_EQ(sqlite3_open(f.db, &raw), SQLITE_OK);
|
||||
ASSERT_EQ(sqlite3_exec(raw, "DROP TABLE store_meta;", NULL, NULL, NULL), SQLITE_OK);
|
||||
sqlite3_close(raw);
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
int visited = gmt_request(f.db, f.ids[0]);
|
||||
uint64_t builds_after = cbm_graph_mem_build_count();
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_GT(visited, 0);
|
||||
ASSERT_EQ(builds_after, builds);
|
||||
PASS();
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
const char *db;
|
||||
int64_t start;
|
||||
const cbm_traverse_result_t *expect;
|
||||
bool same;
|
||||
} gmt_thread_arg_t;
|
||||
|
||||
static void *gmt_thread_main(void *arg) {
|
||||
gmt_thread_arg_t *t = arg;
|
||||
cbm_store_t *s = cbm_store_open_path_query(t->db);
|
||||
cbm_traverse_result_t got = {0};
|
||||
t->same = s && cbm_store_bfs(s, t->start, "inbound", NULL, 0, 6, 100, &got) == CBM_STORE_OK &&
|
||||
gmt_traverse_eq(t->expect, &got);
|
||||
cbm_store_traverse_free(&got);
|
||||
cbm_store_close(s);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
TEST(graph_mem_concurrent_requests_build_once) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
cbm_store_t *sql = gmt_open_sql_reference(&f);
|
||||
cbm_traverse_result_t expect = {0};
|
||||
int expect_rc =
|
||||
sql ? cbm_store_bfs(sql, f.ids[1], "inbound", NULL, 0, 6, 100, &expect) : CBM_STORE_ERR;
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
cbm_thread_t threads[GMT_THREADS];
|
||||
gmt_thread_arg_t args[GMT_THREADS];
|
||||
int started = 0;
|
||||
for (int i = 0; i < GMT_THREADS; i++) {
|
||||
args[i] = (gmt_thread_arg_t){.db = f.db, .start = f.ids[1], .expect = &expect};
|
||||
started += cbm_thread_create(&threads[i], 0, gmt_thread_main, &args[i]) == 0;
|
||||
}
|
||||
int joined = 0;
|
||||
for (int i = 0; i < started; i++) {
|
||||
joined += cbm_thread_join(&threads[i]) == 0;
|
||||
}
|
||||
uint64_t built = cbm_graph_mem_build_count() - builds;
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
int same = 0;
|
||||
for (int i = 0; i < started; i++) {
|
||||
same += args[i].same;
|
||||
}
|
||||
cbm_store_traverse_free(&expect);
|
||||
cbm_store_close(sql);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_EQ(expect_rc, CBM_STORE_OK);
|
||||
ASSERT_EQ(started, GMT_THREADS);
|
||||
ASSERT_EQ(joined, GMT_THREADS);
|
||||
ASSERT_EQ(same, GMT_THREADS);
|
||||
/* A request meeting another's build takes SQL; nobody builds twice. */
|
||||
ASSERT_EQ(built, 1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Idle eviction reads this clock through the seam: tests move time instead
|
||||
* of waiting for it. */
|
||||
static uint64_t g_gmt_now_ms;
|
||||
|
||||
static uint64_t gmt_fake_clock(void) {
|
||||
return g_gmt_now_ms;
|
||||
}
|
||||
|
||||
static void gmt_clock_pin(uint64_t now_ms) {
|
||||
g_gmt_now_ms = now_ms;
|
||||
cbm_graph_mem_set_clock_for_testing(gmt_fake_clock);
|
||||
}
|
||||
|
||||
TEST(graph_mem_idle_layer_evicted_after_idle_time) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
uint64_t builds = cbm_graph_mem_build_count();
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
const uint64_t t0 = 1000000;
|
||||
gmt_clock_pin(t0);
|
||||
cbm_mem_class_flush_thread();
|
||||
size_t class_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GRAPH_CACHE);
|
||||
|
||||
int first = gmt_request(f.db, f.ids[0]); /* built and released at t0 */
|
||||
cbm_mem_class_flush_thread();
|
||||
size_t class_cached = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GRAPH_CACHE);
|
||||
g_gmt_now_ms = t0 + CBM_GRAPH_MEM_IDLE_EVICT_MS - 1;
|
||||
cbm_graph_mem_evict_expired();
|
||||
int cached_just_before = cbm_graph_mem_cached_count();
|
||||
g_gmt_now_ms = t0 + CBM_GRAPH_MEM_IDLE_EVICT_MS;
|
||||
cbm_graph_mem_evict_expired();
|
||||
int cached_at_expiry = cbm_graph_mem_cached_count();
|
||||
size_t bytes_at_expiry = cbm_graph_mem_cached_bytes();
|
||||
cbm_mem_class_flush_thread();
|
||||
size_t class_evicted = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GRAPH_CACHE);
|
||||
/* The next walk builds again: eviction is a cache miss, not an error. */
|
||||
int again = gmt_request(f.db, f.ids[0]);
|
||||
uint64_t built = cbm_graph_mem_build_count() - builds;
|
||||
|
||||
cbm_graph_mem_set_clock_for_testing(NULL);
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_GT(first, 0);
|
||||
/* The layer is charged to its own class and given back in full. */
|
||||
ASSERT_GT(class_cached, class_before);
|
||||
ASSERT_EQ(class_evicted, class_before);
|
||||
ASSERT_EQ(cached_just_before, 1);
|
||||
ASSERT_EQ(cached_at_expiry, 0);
|
||||
ASSERT_EQ(bytes_at_expiry, 0);
|
||||
ASSERT_EQ(again, first);
|
||||
ASSERT_EQ(built, 2);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Idle time counts from the LAST release: a layer a request still holds is
|
||||
* never evicted, however old its build. */
|
||||
TEST(graph_mem_held_layer_is_not_idle) {
|
||||
gmt_fixture_t f;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
const uint64_t t0 = 5000000;
|
||||
gmt_clock_pin(t0);
|
||||
|
||||
cbm_store_t *held = cbm_store_open_path_query(f.db);
|
||||
int walked = held ? gmt_visited(held, f.ids[0]) : -1; /* acquired at t0, still held */
|
||||
g_gmt_now_ms = t0 + 3 * (uint64_t)CBM_GRAPH_MEM_IDLE_EVICT_MS;
|
||||
cbm_graph_mem_evict_expired();
|
||||
int cached_while_held = cbm_graph_mem_cached_count();
|
||||
const uint64_t released_at = g_gmt_now_ms;
|
||||
cbm_store_close(held); /* released now: idle from here */
|
||||
cbm_graph_mem_evict_expired();
|
||||
int cached_after_release = cbm_graph_mem_cached_count();
|
||||
g_gmt_now_ms = released_at + CBM_GRAPH_MEM_IDLE_EVICT_MS;
|
||||
cbm_graph_mem_evict_expired();
|
||||
int cached_after_idle = cbm_graph_mem_cached_count();
|
||||
|
||||
cbm_graph_mem_set_clock_for_testing(NULL);
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
|
||||
ASSERT_GT(walked, 0);
|
||||
ASSERT_EQ(cached_while_held, 1);
|
||||
ASSERT_EQ(cached_after_release, 1);
|
||||
ASSERT_EQ(cached_after_idle, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The daemon has no timer: the next request is the clock. Any MCP request —
|
||||
* here one for a tool that does not exist — sweeps expired layers on its way
|
||||
* out, and a walk on another DB sweeps them on its way in. */
|
||||
TEST(graph_mem_next_request_evicts_expired_layers) {
|
||||
gmt_fixture_t f;
|
||||
gmt_fixture_t g;
|
||||
ASSERT_TRUE(gmt_fixture_build(&f));
|
||||
ASSERT_TRUE(gmt_fixture_build(&g));
|
||||
size_t saved = gmt_budget_pin(gmt_budget_roomy());
|
||||
const uint64_t t0 = 9000000;
|
||||
gmt_clock_pin(t0);
|
||||
|
||||
(void)gmt_request(f.db, f.ids[0]);
|
||||
g_gmt_now_ms = t0 + CBM_GRAPH_MEM_IDLE_EVICT_MS;
|
||||
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
|
||||
char *reply = srv ? cbm_mcp_handle_tool(srv, "no_such_tool", "{}") : NULL;
|
||||
int cached_after_mcp = cbm_graph_mem_cached_count();
|
||||
free(reply);
|
||||
cbm_mcp_server_free(srv);
|
||||
|
||||
(void)gmt_request(f.db, f.ids[0]); /* f rebuilt at t0 + idle */
|
||||
g_gmt_now_ms += CBM_GRAPH_MEM_IDLE_EVICT_MS;
|
||||
(void)gmt_request(g.db, g.ids[0]); /* acquiring g retires the expired f */
|
||||
int cached_after_walk = cbm_graph_mem_cached_count();
|
||||
|
||||
cbm_graph_mem_set_clock_for_testing(NULL);
|
||||
cbm_mem_set_budget_for_tests(saved);
|
||||
gmt_fixture_free(&f);
|
||||
gmt_fixture_free(&g);
|
||||
|
||||
ASSERT_NOT_NULL(srv);
|
||||
ASSERT_NOT_NULL(reply);
|
||||
ASSERT_EQ(cached_after_mcp, 0);
|
||||
ASSERT_EQ(cached_after_walk, 1); /* only g */
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(graph_mem) {
|
||||
RUN_TEST(graph_mem_bfs_matches_sql_on_every_shape);
|
||||
RUN_TEST(graph_mem_degree_search_matches_sql);
|
||||
RUN_TEST(graph_mem_generation_change_invalidates);
|
||||
RUN_TEST(graph_mem_in_place_write_never_serves_stale);
|
||||
RUN_TEST(graph_mem_over_budget_keeps_sql_path);
|
||||
RUN_TEST(graph_mem_legacy_generation_keeps_sql_path);
|
||||
RUN_TEST(graph_mem_concurrent_requests_build_once);
|
||||
RUN_TEST(graph_mem_idle_layer_evicted_after_idle_time);
|
||||
RUN_TEST(graph_mem_held_layer_is_not_idle);
|
||||
RUN_TEST(graph_mem_next_request_evicts_expired_layers);
|
||||
}
|
||||
@@ -1732,6 +1732,7 @@ extern void suite_store_edges(void);
|
||||
extern void suite_store_impact(void);
|
||||
extern void suite_store_scope(void);
|
||||
extern void suite_store_graph_digest(void);
|
||||
extern void suite_graph_mem(void);
|
||||
extern void suite_test_impact(void);
|
||||
extern void suite_test_impact_profiles(void);
|
||||
extern void suite_test_impact_changes(void);
|
||||
@@ -2194,6 +2195,7 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(store_impact);
|
||||
RUN_SELECTED_SUITE(store_scope);
|
||||
RUN_SELECTED_SUITE(store_graph_digest);
|
||||
RUN_SELECTED_SUITE(graph_mem);
|
||||
RUN_SELECTED_SUITE(test_impact);
|
||||
RUN_SELECTED_SUITE(test_impact_profiles);
|
||||
RUN_SELECTED_SUITE(test_impact_changes);
|
||||
|
||||
@@ -12205,6 +12205,16 @@ TEST(search_code_windows_scope_prefilter_removes_pipeline_filter) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/* An indexed project's scan runs in process; the supervised runner now serves
|
||||
* the unindexed recursive fallback and a Windows file_pattern it cannot
|
||||
* prefilter. The supervision tests below pin that runner through the command
|
||||
* override, with a command that does nothing: their probes act at spawn. */
|
||||
#ifdef _WIN32
|
||||
static const char MCP_SUPERVISED_SCAN_NOOP[] = "cd .";
|
||||
#else
|
||||
static const char MCP_SUPERVISED_SCAN_NOOP[] = ":";
|
||||
#endif
|
||||
|
||||
TEST(search_code_cancel_cleans_supervised_scan) {
|
||||
mcp_search_cache_t cache;
|
||||
ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-cancel"));
|
||||
@@ -12213,6 +12223,7 @@ TEST(search_code_cancel_cleans_supervised_scan) {
|
||||
cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path),
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {
|
||||
.server = srv,
|
||||
.cancel_on_call = true,
|
||||
@@ -12490,6 +12501,7 @@ TEST(search_code_scan_deadline_precedes_output_limit) {
|
||||
cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path),
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {.fill_output_directory = logs, .delay_ms = 1100};
|
||||
cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe);
|
||||
cbm_mcp_server_set_search_scan_timeout_for_test(srv, 1000, true);
|
||||
@@ -12521,6 +12533,7 @@ TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up) {
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scratch_dir_for_test(srv, cache.path);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {.reject = true, .scratch_directory = cache.path};
|
||||
cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe);
|
||||
|
||||
@@ -24546,6 +24559,272 @@ TEST(search_code_drops_walker_hits_that_resolve_outside_root) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ── In-process search_code scan ───────────────────────────────────
|
||||
* The scoped scan used to shell out to grep (POSIX) or Select-String
|
||||
* (Windows) and now runs in process. These cases pin its records to the
|
||||
* `grep -Hn` shape the subprocess wrote: path:line:content on POSIX, the
|
||||
* same fields TAB-separated on Windows. */
|
||||
|
||||
#ifdef _WIN32
|
||||
#define SCAN_FS "\t"
|
||||
#define SCAN_RS '\n'
|
||||
#else
|
||||
#define SCAN_FS ":"
|
||||
#define SCAN_RS '\0'
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
char dir[256];
|
||||
char list[512];
|
||||
char out[512];
|
||||
char files[4][512];
|
||||
int count;
|
||||
} scan_fixture_t;
|
||||
|
||||
static bool scan_fixture_open(scan_fixture_t *fx) {
|
||||
memset(fx, 0, sizeof(*fx));
|
||||
const char *tmp = th_mktempdir("cbm_scan");
|
||||
if (!tmp) {
|
||||
return false;
|
||||
}
|
||||
snprintf(fx->dir, sizeof(fx->dir), "%s", tmp);
|
||||
snprintf(fx->list, sizeof(fx->list), "%s/list", tmp);
|
||||
snprintf(fx->out, sizeof(fx->out), "%s/out", tmp);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a file holding exactly `length` bytes (NUL bytes allowed). */
|
||||
static bool scan_fixture_add(scan_fixture_t *fx, const char *name, const char *data,
|
||||
size_t length) {
|
||||
if (fx->count >= 4) {
|
||||
return false;
|
||||
}
|
||||
/* Format from a copy: GCC's -Werror=restrict rejects a source and a
|
||||
* destination inside the same struct. */
|
||||
char dir[sizeof(fx->dir)];
|
||||
memcpy(dir, fx->dir, sizeof(dir));
|
||||
char *path = fx->files[fx->count];
|
||||
snprintf(path, sizeof(fx->files[0]), "%s/%s", dir, name);
|
||||
FILE *fp = cbm_fopen(path, "wb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
bool ok = fwrite(data, 1, length, fp) == length;
|
||||
ok = fclose(fp) == 0 && ok;
|
||||
fx->count += ok ? 1 : 0;
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Write the scoped file list as search_code does (NUL records on POSIX,
|
||||
* lines on Windows); `extra` names one more path that need not exist. */
|
||||
static bool scan_fixture_list(const scan_fixture_t *fx, const char *extra) {
|
||||
FILE *fp = cbm_fopen(fx->list, "wb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
bool ok = true;
|
||||
for (int i = 0; i < fx->count; i++) {
|
||||
ok = ok && fputs(fx->files[i], fp) != EOF && fputc(SCAN_RS, fp) != EOF;
|
||||
}
|
||||
if (extra) {
|
||||
ok = ok && fputs(extra, fp) != EOF && fputc(SCAN_RS, fp) != EOF;
|
||||
}
|
||||
return fclose(fp) == 0 && ok;
|
||||
}
|
||||
|
||||
/* Run the scan; returns the records it wrote (caller frees), CRs kept. */
|
||||
static char *scan_fixture_run(const scan_fixture_t *fx, const char *pattern, bool use_regex,
|
||||
size_t output_limit, int *cause) {
|
||||
*cause = cbm_mcp_search_scan_for_test(NULL, fx->list, pattern, use_regex, fx->out,
|
||||
output_limit, UINT64_MAX);
|
||||
FILE *fp = cbm_fopen(fx->out, "rb");
|
||||
if (!fp) {
|
||||
return NULL;
|
||||
}
|
||||
char *text = calloc(CBM_SZ_64K, 1);
|
||||
size_t length = text ? fread(text, 1, CBM_SZ_64K - 1, fp) : 0;
|
||||
(void)fclose(fp);
|
||||
if (text) {
|
||||
text[length] = '\0';
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/* Append one expected record to `buf`. */
|
||||
static void scan_expect(char *buf, size_t cap, const char *path, int line, const char *content) {
|
||||
size_t used = strlen(buf);
|
||||
snprintf(buf + used, cap - used, "%s" SCAN_FS "%d" SCAN_FS "%s\n", path, line, content);
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_writes_grep_records) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
/* The last line has no newline and line 3 is empty. */
|
||||
static const char a[] = "alpha\nbeta gamma\n\nalpha beta\nlast alpha";
|
||||
static const char b[] = "nothing\nalpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "a.txt", a, sizeof(a) - 1));
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "b.txt", b, sizeof(b) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
const char *pa = fx.files[0];
|
||||
const char *pb = fx.files[1];
|
||||
char want[4096];
|
||||
int cause = -1;
|
||||
|
||||
/* Literal: every matching line, file order, then line order. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* A multi-line pattern is one pattern per line, as with grep -f. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 2, "beta gamma");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
got = scan_fixture_run(&fx, "gamma\nbeta", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* An empty pattern matches every line, the empty one included. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 2, "beta gamma");
|
||||
scan_expect(want, sizeof(want), pa, 3, "");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 1, "nothing");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
got = scan_fixture_run(&fx, "", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* Extended regex, anchored per line. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
got = scan_fixture_run(&fx, "^(alpha|last)", true, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* No match: success with no records. */
|
||||
got = scan_fixture_run(&fx, "zzqqxx", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, "");
|
||||
free(got);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_binary_and_crlf) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
/* A NUL in the first 32 KiB marks the whole file binary: no records. */
|
||||
static const char bin[] = "alpha\n\0alpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "bin.dat", bin, sizeof(bin) - 1));
|
||||
/* A later NUL ends the text at the last complete line before it. */
|
||||
enum { FILLER_LINES = 4000 }; /* 10 bytes each: past the 32 KiB probe */
|
||||
size_t late_cap = (size_t)FILLER_LINES * 10 + 64;
|
||||
char *late = malloc(late_cap);
|
||||
ASSERT_NOT_NULL(late);
|
||||
size_t used = 0;
|
||||
for (int i = 0; i < FILLER_LINES; i++) {
|
||||
memcpy(late + used, "xxxxxxxxx\n", 10);
|
||||
used += 10;
|
||||
}
|
||||
static const char tail[] = "alpha one\nalpha two\0\nalpha three\n";
|
||||
memcpy(late + used, tail, sizeof(tail) - 1);
|
||||
used += sizeof(tail) - 1;
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "late.txt", late, used));
|
||||
free(late);
|
||||
static const char crlf[] = "alpha\r\nbeta\r\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "crlf.txt", crlf, sizeof(crlf) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
char want[4096];
|
||||
int cause = -1;
|
||||
|
||||
/* The record keeps the line's CR; the result parser trims it. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), fx.files[1], FILLER_LINES + 1, "alpha one");
|
||||
scan_expect(want, sizeof(want), fx.files[2], 1, "alpha\r");
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* `$` before a CR: grep on POSIX sees the CR and does not match;
|
||||
* Select-String's Line on Windows had no CR and did. */
|
||||
want[0] = '\0';
|
||||
#ifdef _WIN32
|
||||
scan_expect(want, sizeof(want), fx.files[2], 2, "beta\r");
|
||||
#endif
|
||||
got = scan_fixture_run(&fx, "^beta$", true, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_limits_and_failures) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
static const char a[] = "alpha\nalpha\nalpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "a.txt", a, sizeof(a) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
int cause = -1;
|
||||
|
||||
/* Past the output limit: the output-limit cause (3), not a failure. */
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 8, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 3);
|
||||
|
||||
/* An invalid regex is a command failure (1), as grep's exit 2 was. */
|
||||
got = scan_fixture_run(&fx, "(", true, 0, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 1);
|
||||
|
||||
/* A passed deadline stops the scan before the next file (4); a request
|
||||
* cancelled under the server stops it too, and takes precedence (5). */
|
||||
ASSERT_EQ(cbm_mcp_search_scan_for_test(NULL, fx.list, "alpha", false, fx.out, 0, 0), 4);
|
||||
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
|
||||
ASSERT_NOT_NULL(srv);
|
||||
ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv));
|
||||
ASSERT_TRUE(cbm_mcp_server_cancel_active(srv));
|
||||
cause = cbm_mcp_search_scan_for_test(srv, fx.list, "alpha", false, fx.out, 0, 0);
|
||||
cbm_mcp_server_request_scope_end(srv);
|
||||
cbm_mcp_server_free(srv);
|
||||
ASSERT_EQ(cause, 5);
|
||||
|
||||
/* A listed file that cannot be read is a command failure (1) as well. */
|
||||
char missing[600];
|
||||
snprintf(missing, sizeof(missing), "%s/missing.txt", fx.dir);
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, missing));
|
||||
got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 1);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
#undef SCAN_FS
|
||||
#undef SCAN_RS
|
||||
|
||||
SUITE(mcp) {
|
||||
RUN_TEST(index_repository_async_and_status_refused_without_daemon_issue2144);
|
||||
RUN_TEST(index_repository_schema_documents_async_polling_issue2144);
|
||||
@@ -24813,6 +25092,9 @@ SUITE(mcp) {
|
||||
RUN_TEST(search_code_scan_deadline_override_is_per_server);
|
||||
RUN_TEST(search_code_scan_setup_failures_respect_cause_precedence);
|
||||
RUN_TEST(search_code_scan_live_child_deadline_is_bounded_and_fails_closed);
|
||||
RUN_TEST(search_code_scan_in_process_writes_grep_records);
|
||||
RUN_TEST(search_code_scan_in_process_binary_and_crlf);
|
||||
RUN_TEST(search_code_scan_in_process_limits_and_failures);
|
||||
RUN_TEST(search_code_scan_cancellation_precedes_zero_deadline);
|
||||
RUN_TEST(search_code_scan_deadline_precedes_output_limit);
|
||||
RUN_TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up);
|
||||
|
||||
@@ -1715,6 +1715,7 @@ TEST(mem_core_class_names_are_total) {
|
||||
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_STORE), "store") == 0);
|
||||
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_HASH_TABLE), "hash_table") == 0);
|
||||
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_DYN_ARRAY), "dyn_array") == 0);
|
||||
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_GRAPH_CACHE), "graph_cache") == 0);
|
||||
/* Out of range must still answer, so a log line never takes a NULL. */
|
||||
ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)(CBM_MEM_CLASS_COUNT + 5)) != NULL);
|
||||
ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)-1) != NULL);
|
||||
|
||||
+180
-6
@@ -6153,6 +6153,177 @@ TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
static int live_noop_run(const char *repo, const char *db_path, cbm_incremental_route_t *route) {
|
||||
cbm_pipeline_incremental_test_reset_faults();
|
||||
*route = CBM_INCREMENTAL_ROUTE_NONE;
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL);
|
||||
if (!p) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
int rc = cbm_pipeline_run(p);
|
||||
*route = cbm_pipeline_incremental_test_last_route();
|
||||
cbm_pipeline_free(p);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* An unchanged re-index proves the exact no-op on the LIVE generation. It
|
||||
* never copies the database into a stage, seals the copy and renames it back
|
||||
* (on a kernel-scale index that copy alone was a 19 GB write per call): the
|
||||
* live file keeps its identity, size and mtime, no stage artifact remains,
|
||||
* and the graph still serves. */
|
||||
TEST(pipeline_unchanged_reindex_leaves_live_generation_untouched) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_live_noop_XXXXXX");
|
||||
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
|
||||
write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n");
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp);
|
||||
char *project = cbm_project_name_from_path(tmp);
|
||||
ASSERT_NOT_NULL(project);
|
||||
|
||||
cbm_incremental_route_t first_route;
|
||||
int first_rc = live_noop_run(tmp, db_path, &first_route);
|
||||
cbm_path_info_t before = {0};
|
||||
struct stat before_st;
|
||||
bool before_ok =
|
||||
cbm_path_info_utf8(db_path, &before) == CBM_PATH_INFO_OK && stat(db_path, &before_st) == 0;
|
||||
|
||||
cbm_incremental_route_t noop_route;
|
||||
int noop_rc = live_noop_run(tmp, db_path, &noop_route);
|
||||
cbm_path_info_t after = {0};
|
||||
struct stat after_st;
|
||||
bool after_ok =
|
||||
cbm_path_info_utf8(db_path, &after) == CBM_PATH_INFO_OK && stat(db_path, &after_st) == 0;
|
||||
int defined_count = -1;
|
||||
int absent_count = -1;
|
||||
observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &defined_count,
|
||||
&absent_count);
|
||||
int stage_count = count_generation_stage_artifacts(tmp, "generation.db");
|
||||
cbm_pipeline_incremental_test_reset_faults();
|
||||
free(project);
|
||||
th_rmtree(tmp);
|
||||
|
||||
ASSERT_EQ(first_rc, 0);
|
||||
ASSERT_TRUE(before_ok);
|
||||
ASSERT_EQ(noop_rc, 0);
|
||||
ASSERT_EQ(noop_route, CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_TRUE(after_ok);
|
||||
ASSERT_TRUE(after_st.st_ino == before_st.st_ino);
|
||||
ASSERT_TRUE(after.mtime_ns == before.mtime_ns);
|
||||
ASSERT_TRUE(after.size == before.size);
|
||||
ASSERT_EQ(stage_count, 0);
|
||||
ASSERT_EQ(defined_count, 1);
|
||||
ASSERT_EQ(absent_count, 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* The live proof answers only for an exact manifest. Every real change -- a
|
||||
* same-size edit that keeps the original mtime, an addition, a deletion and
|
||||
* a rename -- still routes past it into a rebuild that the graph shows, and
|
||||
* only the unchanged re-index after them takes the no-op. */
|
||||
TEST(pipeline_live_noop_proof_never_hides_a_change) {
|
||||
char tmp[256];
|
||||
snprintf(tmp, sizeof(tmp), "/tmp/cbm_live_noop_change_XXXXXX");
|
||||
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
|
||||
write_temp_file(tmp, "alpha.py", "def AlphaDefined():\n return 1\n");
|
||||
write_temp_file(tmp, "beta.py", "def BetaDefined():\n return 2\n");
|
||||
char db_path[512];
|
||||
char alpha_path[512];
|
||||
char beta_path[512];
|
||||
char gamma_path[512];
|
||||
char delta_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp);
|
||||
snprintf(alpha_path, sizeof(alpha_path), "%s/alpha.py", tmp);
|
||||
snprintf(beta_path, sizeof(beta_path), "%s/beta.py", tmp);
|
||||
snprintf(gamma_path, sizeof(gamma_path), "%s/gamma.py", tmp);
|
||||
snprintf(delta_path, sizeof(delta_path), "%s/delta.py", tmp);
|
||||
char *project = cbm_project_name_from_path(tmp);
|
||||
ASSERT_NOT_NULL(project);
|
||||
|
||||
cbm_incremental_route_t initial_route;
|
||||
int initial_rc = live_noop_run(tmp, db_path, &initial_route);
|
||||
|
||||
/* Same size, original mtime restored: only the bytes differ. */
|
||||
cbm_path_info_t alpha_info = {0};
|
||||
bool alpha_stat_ok = cbm_path_info_utf8(alpha_path, &alpha_info) == CBM_PATH_INFO_OK;
|
||||
write_temp_file(tmp, "alpha.py", "def AlphbDefined():\n return 1\n");
|
||||
int mtime_rc = pipeline_test_set_mtime(alpha_path, (time_t)(alpha_info.mtime_ns / 1000000000),
|
||||
(long)(alpha_info.mtime_ns % 1000000000));
|
||||
cbm_path_info_t edited_info = {0};
|
||||
bool edited_same_stat = cbm_path_info_utf8(alpha_path, &edited_info) == CBM_PATH_INFO_OK &&
|
||||
edited_info.size == alpha_info.size &&
|
||||
edited_info.mtime_ns == alpha_info.mtime_ns;
|
||||
cbm_incremental_route_t modify_route;
|
||||
int modify_rc = live_noop_run(tmp, db_path, &modify_route);
|
||||
int old_alpha = -1;
|
||||
int new_alpha = -1;
|
||||
observe_named_generation(db_path, project, "AlphaDefined", "AlphbDefined", &old_alpha,
|
||||
&new_alpha);
|
||||
|
||||
write_temp_file(tmp, "gamma.py", "def GammaDefined():\n return 3\n");
|
||||
cbm_incremental_route_t add_route;
|
||||
int add_rc = live_noop_run(tmp, db_path, &add_route);
|
||||
int gamma_added = -1;
|
||||
int unused = -1;
|
||||
observe_named_generation(db_path, project, "GammaDefined", "NeverDefined", &gamma_added,
|
||||
&unused);
|
||||
|
||||
int unlink_rc = cbm_unlink(beta_path);
|
||||
cbm_incremental_route_t delete_route;
|
||||
int delete_rc = live_noop_run(tmp, db_path, &delete_route);
|
||||
int beta_left = -1;
|
||||
observe_named_generation(db_path, project, "BetaDefined", "NeverDefined", &beta_left, &unused);
|
||||
|
||||
int rename_rc = cbm_rename_replace(gamma_path, delta_path);
|
||||
cbm_incremental_route_t rename_route;
|
||||
int renamed_rc = live_noop_run(tmp, db_path, &rename_route);
|
||||
int gamma_renamed = -1;
|
||||
observe_named_generation(db_path, project, "GammaDefined", "NeverDefined", &gamma_renamed,
|
||||
&unused);
|
||||
int old_path_hash = CBM_STORE_OK;
|
||||
int new_path_hash = CBM_STORE_NOT_FOUND;
|
||||
cbm_store_t *store = cbm_store_open_path_query(db_path);
|
||||
if (store) {
|
||||
cbm_file_hash_t hash = {0};
|
||||
old_path_hash = cbm_store_get_file_hash(store, project, "gamma.py", &hash);
|
||||
cbm_store_clear_file_hash(&hash);
|
||||
new_path_hash = cbm_store_get_file_hash(store, project, "delta.py", &hash);
|
||||
cbm_store_clear_file_hash(&hash);
|
||||
cbm_store_close(store);
|
||||
}
|
||||
|
||||
cbm_incremental_route_t final_route;
|
||||
int final_rc = live_noop_run(tmp, db_path, &final_route);
|
||||
cbm_pipeline_incremental_test_reset_faults();
|
||||
free(project);
|
||||
th_rmtree(tmp);
|
||||
|
||||
ASSERT_EQ(initial_rc, 0);
|
||||
ASSERT_TRUE(alpha_stat_ok);
|
||||
ASSERT_EQ(mtime_rc, 0);
|
||||
ASSERT_TRUE(edited_same_stat);
|
||||
ASSERT_EQ(modify_rc, 0);
|
||||
ASSERT_TRUE(modify_route != CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(old_alpha, 0);
|
||||
ASSERT_EQ(new_alpha, 1);
|
||||
ASSERT_EQ(add_rc, 0);
|
||||
ASSERT_TRUE(add_route != CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(gamma_added, 1);
|
||||
ASSERT_EQ(unlink_rc, 0);
|
||||
ASSERT_EQ(delete_rc, 0);
|
||||
ASSERT_TRUE(delete_route != CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(beta_left, 0);
|
||||
ASSERT_EQ(rename_rc, 0);
|
||||
ASSERT_EQ(renamed_rc, 0);
|
||||
ASSERT_TRUE(rename_route != CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(gamma_renamed, 1);
|
||||
ASSERT_EQ(old_path_hash, CBM_STORE_NOT_FOUND);
|
||||
ASSERT_EQ(new_path_hash, CBM_STORE_OK);
|
||||
ASSERT_EQ(final_rc, 0);
|
||||
ASSERT_EQ(final_route, CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
PASS();
|
||||
}
|
||||
|
||||
static char g_route_log_capture[8192];
|
||||
static atomic_flag g_route_log_spin = ATOMIC_FLAG_INIT;
|
||||
|
||||
@@ -6762,15 +6933,15 @@ TEST(pipeline_existing_artifact_refreshes_after_default_forced_full_reindex) {
|
||||
ASSERT_EQ(reindex_observer.rename_calls, 1);
|
||||
ASSERT_EQ(reindex_observer.exports_before_publish, 0);
|
||||
ASSERT_EQ(reindex_observer.export_count, 1);
|
||||
/* The no-ops are proven on the live generation and publish nothing; the
|
||||
* artifact is still refreshed, from that already published database. */
|
||||
ASSERT_EQ(explicit_rc, 0);
|
||||
ASSERT_EQ(explicit_route, CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(explicit_observer.rename_calls, 1);
|
||||
ASSERT_EQ(explicit_observer.exports_before_publish, 0);
|
||||
ASSERT_EQ(explicit_observer.rename_calls, 0);
|
||||
ASSERT_EQ(explicit_observer.export_count, 1);
|
||||
ASSERT_EQ(unchanged_rc, 0);
|
||||
ASSERT_EQ(unchanged_route, CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_EQ(observer.rename_calls, 1);
|
||||
ASSERT_EQ(observer.exports_before_publish, 0);
|
||||
ASSERT_EQ(observer.rename_calls, 0);
|
||||
ASSERT_EQ(observer.export_count, 1);
|
||||
ASSERT_TRUE(journal_ok);
|
||||
ASSERT_STR_EQ(journal_mode, "delete");
|
||||
@@ -19247,8 +19418,9 @@ TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delet
|
||||
ASSERT_EQ(noop.rc, 0);
|
||||
ASSERT_EQ(noop.route, CBM_INCREMENTAL_ROUTE_NOOP);
|
||||
ASSERT_TRUE(noop.tools_excluded);
|
||||
ASSERT_EQ(noop.publish.rename_calls, 1);
|
||||
ASSERT_EQ(noop.publish.exports_before_publish, 0);
|
||||
/* Proven on the live generation: nothing is published, the artifact is
|
||||
* still refreshed from the live database. */
|
||||
ASSERT_EQ(noop.publish.rename_calls, 0);
|
||||
ASSERT_EQ(noop.publish.export_count, 1);
|
||||
|
||||
ASSERT_EQ(deleted.rc, 0);
|
||||
@@ -23753,6 +23925,8 @@ SUITE(pipeline_semantic_manifest_repro) {
|
||||
RUN_TEST(pipeline_sweep_reclaims_orphans_of_other_projects_and_keeps_their_live_stages);
|
||||
RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage);
|
||||
RUN_TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage);
|
||||
RUN_TEST(pipeline_unchanged_reindex_leaves_live_generation_untouched);
|
||||
RUN_TEST(pipeline_live_noop_proof_never_hides_a_change);
|
||||
RUN_TEST(pipeline_fresh_index_never_reports_invalid_existing_db);
|
||||
RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation);
|
||||
RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation);
|
||||
|
||||
@@ -298,6 +298,45 @@ TEST(arch_path_scoping) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* packages: count DESC, tied counts in name (BINARY) order. The aggregation
|
||||
* runs off the label index now; a sorted GROUP BY follows the ORDER BY
|
||||
* direction, which reversed the ties until the name tie-break was explicit. */
|
||||
TEST(arch_packages_tied_counts_in_name_order) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
ASSERT_EQ(cbm_store_upsert_project(s, "test", "/tmp/test"), CBM_STORE_OK);
|
||||
static const char *const pkg_names[] = {"gamma", "alpha", "Zeta", "beta",
|
||||
"Alpha", "gamma", "beta"};
|
||||
enum { PKG_NODES = 7, PKG_GROUPS = 5 };
|
||||
for (int i = 0; i < PKG_NODES; i++) {
|
||||
char qn[64];
|
||||
snprintf(qn, sizeof(qn), "test.pkg%d.%s", i, pkg_names[i]);
|
||||
cbm_node_t n = {.project = "test",
|
||||
.label = "Package",
|
||||
.name = pkg_names[i],
|
||||
.qualified_name = qn,
|
||||
.file_path = "pkg/manifest.toml"};
|
||||
ASSERT_TRUE(cbm_store_upsert_node(s, &n) > 0);
|
||||
}
|
||||
static const char *const want_names[] = {"beta", "gamma", "Alpha", "Zeta", "alpha"};
|
||||
static const int want_counts[] = {2, 2, 1, 1, 1};
|
||||
const char *aspects[] = {"packages"};
|
||||
const char *paths[] = {NULL, "pkg"}; /* unscoped and path-scoped SQL */
|
||||
for (int p = 0; p < 2; p++) {
|
||||
cbm_architecture_info_t info;
|
||||
ASSERT_EQ(cbm_store_get_architecture(s, "test", paths[p], aspects, 1, &info),
|
||||
CBM_STORE_OK);
|
||||
ASSERT_EQ(info.package_count, PKG_GROUPS);
|
||||
for (int i = 0; i < PKG_GROUPS; i++) {
|
||||
ASSERT_STR_EQ(info.packages[i].name, want_names[i]);
|
||||
ASSERT_EQ(info.packages[i].node_count, want_counts[i]);
|
||||
}
|
||||
cbm_store_architecture_free(&info);
|
||||
}
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(arch_empty_project) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
@@ -1859,6 +1898,7 @@ SUITE(store_arch) {
|
||||
RUN_TEST(arch_hotspots_exclude_tests);
|
||||
RUN_TEST(arch_specific_aspects);
|
||||
RUN_TEST(arch_path_scoping);
|
||||
RUN_TEST(arch_packages_tied_counts_in_name_order);
|
||||
RUN_TEST(arch_empty_project);
|
||||
RUN_TEST(arch_languages);
|
||||
RUN_TEST(arch_routes);
|
||||
|
||||
@@ -402,6 +402,122 @@ TEST(store_node_find_by_label) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
static bool scan_str_same(const char *a, const char *b) {
|
||||
return (!a && !b) || (a && b && strcmp(a, b) == 0);
|
||||
}
|
||||
|
||||
static bool scan_node_same(const cbm_node_t *a, const cbm_node_t *b) {
|
||||
return a->id == b->id && scan_str_same(a->project, b->project) &&
|
||||
scan_str_same(a->label, b->label) && scan_str_same(a->name, b->name) &&
|
||||
scan_str_same(a->qualified_name, b->qualified_name) &&
|
||||
scan_str_same(a->file_path, b->file_path) && a->start_line == b->start_line &&
|
||||
a->end_line == b->end_line && scan_str_same(a->properties_json, b->properties_json);
|
||||
}
|
||||
|
||||
/* The streamed scan gives the eager scan's rows, field for field and in the
|
||||
* same order, at any batch size, while other lookups (including the eager
|
||||
* scans' own cached statements) run between batches. Returns the row count,
|
||||
* or -1 on the first difference; *paged tells whether the scan paged. */
|
||||
static int scan_matches_eager(cbm_store_t *s, const char *project, const char *label, int batch,
|
||||
bool *paged) {
|
||||
cbm_node_t *eager = NULL;
|
||||
int n = 0;
|
||||
int rc = label ? cbm_store_find_nodes_by_label(s, project, label, &eager, &n)
|
||||
: cbm_store_find_nodes(s, project, &eager, &n);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
return -1;
|
||||
}
|
||||
cbm_store_node_scan_t *scan = cbm_store_node_scan_open(s, project, label);
|
||||
*paged = cbm_store_node_scan_paged(scan);
|
||||
cbm_node_t buf[64];
|
||||
int seen = 0;
|
||||
bool same = scan != NULL && batch <= 64;
|
||||
int got = 0;
|
||||
while (same && cbm_store_node_scan_next(scan, buf, batch, &got) == CBM_STORE_OK && got > 0) {
|
||||
for (int i = 0; i < got; i++, seen++) {
|
||||
same = same && seen < n && scan_node_same(&buf[i], &eager[seen]);
|
||||
cbm_node_free_fields(&buf[i]);
|
||||
}
|
||||
cbm_node_t *other = NULL;
|
||||
int other_n = 0;
|
||||
(void)cbm_store_find_nodes(s, project, &other, &other_n);
|
||||
cbm_store_free_nodes(other, other_n);
|
||||
(void)cbm_store_find_nodes_by_label(s, project, "Class", &other, &other_n);
|
||||
cbm_store_free_nodes(other, other_n);
|
||||
}
|
||||
/* An exhausted scan stays exhausted. */
|
||||
same = same && cbm_store_node_scan_next(scan, buf, batch, &got) == CBM_STORE_OK && got == 0;
|
||||
cbm_store_node_scan_close(scan);
|
||||
cbm_store_free_nodes(eager, n);
|
||||
return same && seen == n ? n : -1;
|
||||
}
|
||||
|
||||
/* Pages of 1, 3 and 64 rows end inside files, at file boundaries and inside
|
||||
* the NULL file_path rows, which the eager all-nodes order puts first. The
|
||||
* upsert API stores "" for a missing path; other writers may store NULL. */
|
||||
TEST(store_node_scan_streams_eager_scan_order) {
|
||||
enum { SCAN_NODES = 60 };
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
cbm_store_upsert_project(s, "other", "/tmp/other");
|
||||
static const char *const labels[] = {"Function", "Class", "Method"};
|
||||
static const char *const files[] = {"b.c", "a.c", "", "c.c"};
|
||||
int in_test = 0;
|
||||
int test_functions = 0;
|
||||
int other_methods = 0;
|
||||
for (int i = 0; i < SCAN_NODES; i++) {
|
||||
bool other = i % 4 == 3;
|
||||
char name[32];
|
||||
char qn[64];
|
||||
char props[64];
|
||||
snprintf(name, sizeof(name), "n%d", (i * 17) % 40);
|
||||
snprintf(qn, sizeof(qn), "%s.n%d", other ? "other" : "test", i);
|
||||
snprintf(props, sizeof(props), "{\"complexity\":%d}", i);
|
||||
cbm_node_t n = {.project = other ? "other" : "test",
|
||||
.label = labels[i % 3],
|
||||
.name = name,
|
||||
.qualified_name = qn,
|
||||
.file_path = files[(i / 2) % 4],
|
||||
.start_line = i,
|
||||
.end_line = i + 5,
|
||||
.properties_json = i % 5 ? props : NULL};
|
||||
ASSERT_GT(cbm_store_upsert_node(s, &n), 0);
|
||||
in_test += other ? 0 : 1;
|
||||
test_functions += !other && i % 3 == 0 ? 1 : 0;
|
||||
other_methods += other && i % 3 == 2 ? 1 : 0;
|
||||
}
|
||||
ASSERT_EQ(cbm_store_exec(s, "UPDATE nodes SET file_path = NULL WHERE start_line % 7 = 0;"), 0);
|
||||
cbm_node_t *eager = NULL;
|
||||
int eager_n = 0;
|
||||
ASSERT_EQ(cbm_store_find_nodes(s, "test", &eager, &eager_n), CBM_STORE_OK);
|
||||
ASSERT_TRUE(eager_n > 0 && eager[0].file_path == NULL); /* NULL rows come first */
|
||||
cbm_store_free_nodes(eager, eager_n);
|
||||
|
||||
static const int batches[] = {1, 3, 64};
|
||||
bool paged = false;
|
||||
for (size_t b = 0; b < sizeof(batches) / sizeof(batches[0]); b++) {
|
||||
ASSERT_EQ(scan_matches_eager(s, "test", NULL, batches[b], &paged), in_test);
|
||||
ASSERT_TRUE(paged);
|
||||
ASSERT_EQ(scan_matches_eager(s, "test", "Function", batches[b], &paged), test_functions);
|
||||
ASSERT_TRUE(paged);
|
||||
ASSERT_EQ(scan_matches_eager(s, "other", NULL, batches[b], &paged),
|
||||
SCAN_NODES - in_test); /* project filter */
|
||||
ASSERT_EQ(scan_matches_eager(s, "other", "Method", batches[b], &paged), other_methods);
|
||||
ASSERT_EQ(scan_matches_eager(s, "test", "NoSuchLabel", batches[b], &paged), 0);
|
||||
ASSERT_EQ(scan_matches_eager(s, "missing", NULL, batches[b], &paged), 0);
|
||||
}
|
||||
|
||||
/* Without the node indexes the eager queries take other plans, which the
|
||||
* pages would not reproduce: the scan hands out the eager rows instead. */
|
||||
ASSERT_EQ(cbm_store_drop_indexes(s), 0);
|
||||
ASSERT_EQ(scan_matches_eager(s, "test", NULL, 3, &paged), in_test);
|
||||
ASSERT_FALSE(paged);
|
||||
ASSERT_EQ(scan_matches_eager(s, "test", "Function", 3, &paged), test_functions);
|
||||
ASSERT_FALSE(paged);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(store_node_find_by_file) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
@@ -2516,6 +2632,7 @@ SUITE(store_nodes) {
|
||||
RUN_TEST(store_node_crud);
|
||||
RUN_TEST(store_node_dedup);
|
||||
RUN_TEST(store_node_find_by_label);
|
||||
RUN_TEST(store_node_scan_streams_eager_scan_order);
|
||||
RUN_TEST(store_node_find_by_file);
|
||||
RUN_TEST(store_file_outline_is_filtered_stable_bounded_and_cancellable_issue469);
|
||||
RUN_TEST(store_file_outline_fails_closed_on_text_budget_issue469);
|
||||
|
||||
@@ -345,7 +345,75 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A read tool's statements share one read batch instead of taking the file
|
||||
* lock once per statement. The batch reads one snapshot until it renews
|
||||
* (every 100 ms, forced here through the seam), then sees later commits. */
|
||||
static char *read_batch_name(cbm_store_t *s) {
|
||||
cbm_node_t found = {0};
|
||||
if (cbm_store_find_node_by_qn(s, "p", "p.a", &found) != CBM_STORE_OK) {
|
||||
return NULL;
|
||||
}
|
||||
char *name = found.name ? strdup(found.name) : NULL;
|
||||
cbm_node_free_fields(&found);
|
||||
return name;
|
||||
}
|
||||
|
||||
static bool read_batch_rename(cbm_store_t *writer, const char *name) {
|
||||
cbm_node_t node = {
|
||||
.project = "p", .label = "Function", .name = name, .qualified_name = "p.a", .file_path = "a.c"};
|
||||
return cbm_store_upsert_node(writer, &node) > 0;
|
||||
}
|
||||
|
||||
TEST(store_read_batch_snapshot_renews) {
|
||||
char *td = th_mktempdir("cbm_read_batch");
|
||||
ASSERT_NOT_NULL(td);
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/batch.db", td);
|
||||
cbm_store_t *writer = cbm_store_open_path(db_path); /* WAL: commits beside a reader */
|
||||
ASSERT_NOT_NULL(writer);
|
||||
ASSERT_EQ(cbm_store_upsert_project(writer, "p", td), CBM_STORE_OK);
|
||||
ASSERT_TRUE(read_batch_rename(writer, "before"));
|
||||
cbm_store_t *reader = cbm_store_open_path_query(db_path);
|
||||
ASSERT_NOT_NULL(reader);
|
||||
|
||||
ASSERT_TRUE(cbm_store_read_batch_begin(reader));
|
||||
ASSERT_FALSE(cbm_store_read_batch_begin(reader)); /* one batch at a time */
|
||||
char *name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "before");
|
||||
free(name);
|
||||
|
||||
/* Inside the batch the reader keeps its snapshot... */
|
||||
ASSERT_TRUE(read_batch_rename(writer, "after"));
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "before");
|
||||
free(name);
|
||||
|
||||
/* ...until the batch renews at its next statement. */
|
||||
cbm_store_read_batch_expire_for_test(reader);
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "after");
|
||||
free(name);
|
||||
|
||||
/* Ended, every statement reads the latest commit again. */
|
||||
cbm_store_read_batch_end(reader);
|
||||
ASSERT_TRUE(read_batch_rename(writer, "latest"));
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "latest");
|
||||
free(name);
|
||||
|
||||
/* Begin refuses while another transaction is open; close ends a batch. */
|
||||
ASSERT_EQ(cbm_store_begin(writer), CBM_STORE_OK);
|
||||
ASSERT_FALSE(cbm_store_read_batch_begin(writer));
|
||||
ASSERT_EQ(cbm_store_rollback(writer), CBM_STORE_OK);
|
||||
ASSERT_TRUE(cbm_store_read_batch_begin(reader));
|
||||
cbm_store_close(reader);
|
||||
cbm_store_close(writer);
|
||||
th_rmtree(td);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(store_pragmas) {
|
||||
RUN_TEST(store_read_batch_snapshot_renews);
|
||||
RUN_TEST(journal_size_limit_bounds_wal_issue1083);
|
||||
RUN_TEST(store_role_pragmas_issue1419);
|
||||
RUN_TEST(store_generation_tracks_mutations);
|
||||
|
||||
@@ -621,6 +621,56 @@ TEST(store_schema_info) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Label and edge-type counts: count DESC, tied counts in BINARY name order —
|
||||
* the order the GROUP BY scan produced and the distinct-value walk keeps.
|
||||
* Another project's rows must not be counted. */
|
||||
TEST(store_schema_counts_ties_in_name_order) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
cbm_store_upsert_project(s, "other", "/tmp/other");
|
||||
static const char *const labels[] = {"gamma", "Delta", "alpha", "Zeta", "Beta",
|
||||
"Delta", "Zeta", "Alpha", "gamma", "Delta"};
|
||||
enum { NODE_COUNT = 10, LABEL_GROUPS = 6, TYPE_GROUPS = 4, EDGE_COUNT = 8 };
|
||||
int64_t ids[NODE_COUNT];
|
||||
for (int i = 0; i < NODE_COUNT; i++) {
|
||||
char qn[32];
|
||||
snprintf(qn, sizeof(qn), "test.n%d", i);
|
||||
cbm_node_t n = {.project = "test", .label = labels[i], .name = qn, .qualified_name = qn};
|
||||
ids[i] = cbm_store_upsert_node(s, &n);
|
||||
ASSERT_TRUE(ids[i] > 0);
|
||||
snprintf(qn, sizeof(qn), "other.n%d", i);
|
||||
cbm_node_t o = {.project = "other", .label = "Alpha", .name = qn, .qualified_name = qn};
|
||||
ASSERT_TRUE(cbm_store_upsert_node(s, &o) > 0);
|
||||
}
|
||||
static const char *const types[] = {"USAGE", "DEFINES", "CALLS", "IMPORTS",
|
||||
"CALLS", "DEFINES", "USAGE", "DEFINES"};
|
||||
for (int i = 0; i < EDGE_COUNT; i++) {
|
||||
cbm_edge_t e = {
|
||||
.project = "test", .source_id = ids[i], .target_id = ids[i + 1], .type = types[i]};
|
||||
ASSERT_TRUE(cbm_store_insert_edge(s, &e) > 0);
|
||||
}
|
||||
cbm_schema_info_t schema = {0};
|
||||
ASSERT_EQ(cbm_store_get_schema_counts(s, "test", &schema), CBM_STORE_OK);
|
||||
static const char *const want_labels[] = {"Delta", "Zeta", "gamma", "Alpha", "Beta", "alpha"};
|
||||
static const int want_label_counts[] = {3, 2, 2, 1, 1, 1};
|
||||
ASSERT_EQ(schema.node_label_count, LABEL_GROUPS);
|
||||
for (int i = 0; i < LABEL_GROUPS; i++) {
|
||||
ASSERT_STR_EQ(schema.node_labels[i].label, want_labels[i]);
|
||||
ASSERT_EQ(schema.node_labels[i].count, want_label_counts[i]);
|
||||
}
|
||||
static const char *const want_types[] = {"DEFINES", "CALLS", "USAGE", "IMPORTS"};
|
||||
static const int want_type_counts[] = {3, 2, 2, 1};
|
||||
ASSERT_EQ(schema.edge_type_count, TYPE_GROUPS);
|
||||
for (int i = 0; i < TYPE_GROUPS; i++) {
|
||||
ASSERT_STR_EQ(schema.edge_types[i].type, want_types[i]);
|
||||
ASSERT_EQ(schema.edge_types[i].count, want_type_counts[i]);
|
||||
}
|
||||
cbm_store_schema_free(&schema);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Search with exclude_labels ─────────────────────────────────── */
|
||||
|
||||
TEST(store_search_exclude_labels) {
|
||||
@@ -859,6 +909,173 @@ TEST(store_search_case_insensitive) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Anchored name patterns: the idx_nodes_name range path ─────── */
|
||||
|
||||
TEST(store_regex_anchored_prefix) {
|
||||
char out[64];
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^schedule$", out, sizeof(out)), 8);
|
||||
ASSERT_STR_EQ(out, "schedule");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^kmalloc", out, sizeof(out)), 7);
|
||||
ASSERT_STR_EQ(out, "kmalloc");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^get_", out, sizeof(out)), 4);
|
||||
ASSERT_STR_EQ(out, "get_");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^o'neil", out, sizeof(out)), 6);
|
||||
ASSERT_STR_EQ(out, "o'neil");
|
||||
/* The run stops at the first operator; the prefix before it holds. */
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^foo.bar", out, sizeof(out)), 3);
|
||||
ASSERT_STR_EQ(out, "foo");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^foo[0-9]", out, sizeof(out)), 3);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^foo(bar)?", out, sizeof(out)), 3);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^foo$", out, sizeof(out)), 3);
|
||||
/* A quantifier or escape after the run drops its last character. */
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc*", out, sizeof(out)), 2);
|
||||
ASSERT_STR_EQ(out, "ab");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc?", out, sizeof(out)), 2);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc+?", out, sizeof(out)), 2);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc{0,1}", out, sizeof(out)), 2);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc\\.d", out, sizeof(out)), 2);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^a*", out, sizeof(out)), 0);
|
||||
/* No provable prefix: unanchored, alternation, leading operator,
|
||||
* non-ASCII. */
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("schedule", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix(".*schedule", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^foo|bar", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^(foo|bar)x", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^(foo)", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^[Ff]oo", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^.foo", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^\\x", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^\xc3\xbc"
|
||||
"ber",
|
||||
out, sizeof(out)),
|
||||
0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^ab\xc3\xbc", out, sizeof(out)), 2);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("(?i)^foo", out, sizeof(out)), 0);
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix(NULL, out, sizeof(out)), 0);
|
||||
/* Bounded by the buffer; a quantifier right after the cut still drops. */
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abcdef", out, 4), 3);
|
||||
ASSERT_STR_EQ(out, "abc");
|
||||
ASSERT_EQ(cbm_regex_anchored_prefix("^abc*", out, 4), 2);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* Statements a search ran with the name-range predicate (SQLITE_TRACE_STMT). */
|
||||
static int anchored_trace_cb(unsigned type, void *ctx, void *stmt, void *sql) {
|
||||
(void)stmt;
|
||||
if (type == SQLITE_TRACE_STMT && sql && strstr((const char *)sql, "n.name >= ?")) {
|
||||
(*(int *)ctx)++;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* "name#label;" per row plus the total, for comparing two searches. */
|
||||
static void anchored_search_rows(cbm_store_t *s, const cbm_search_params_t *p, int *rc,
|
||||
char *buf, size_t buf_sz) {
|
||||
cbm_search_output_t out = {0};
|
||||
*rc = cbm_store_search(s, p, &out);
|
||||
int len = snprintf(buf, buf_sz, "total=%d:", out.total);
|
||||
for (int i = 0; i < out.count && len > 0 && (size_t)len < buf_sz; i++) {
|
||||
len += snprintf(buf + len, buf_sz - (size_t)len, "%s#%s#%lld;", out.results[i].node.name,
|
||||
out.results[i].node.label, (long long)out.results[i].node.id);
|
||||
}
|
||||
cbm_store_search_free(&out);
|
||||
}
|
||||
|
||||
/* An anchored literal pattern must return exactly what the same regex returns
|
||||
* without the range path. Wrapping it in a group, "(^schedule$)", keeps the
|
||||
* regex semantics but has no provable prefix, so it runs the plain scan. The
|
||||
* names cover every letter case of the prefix, prefix-vs-substring, and the
|
||||
* byte order the UNION ALL arms are merged in. */
|
||||
TEST(store_search_anchored_name_matches_plain_scan) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
cbm_store_upsert_project(s, "p", "/tmp/p");
|
||||
static const char *const names[] = {
|
||||
"schedule", "Schedule", "SCHEDULE", "sCHEDULE", "scheduler", "schedule_work",
|
||||
"reschedule", "sched", "Sched_x", "get_name", "Get_Name", "getname",
|
||||
"o'neil", "ScHeDuLe", "schedule", "zabc"};
|
||||
enum { NAME_COUNT = 16 };
|
||||
for (int i = 0; i < NAME_COUNT; i++) {
|
||||
char qn[64];
|
||||
snprintf(qn, sizeof(qn), "p.m%d.%s", i, names[i]);
|
||||
cbm_node_t n = {.project = "p",
|
||||
.label = (i % 3 == 1) ? "Class" : "Function",
|
||||
.name = names[i],
|
||||
.qualified_name = qn,
|
||||
.file_path = "m.c"};
|
||||
ASSERT_TRUE(cbm_store_upsert_node(s, &n) > 0);
|
||||
}
|
||||
int ranged = 0;
|
||||
sqlite3_trace_v2(cbm_store_get_db(s), SQLITE_TRACE_STMT, anchored_trace_cb, &ranged);
|
||||
|
||||
struct {
|
||||
const char *pattern;
|
||||
const char *label;
|
||||
bool case_sensitive;
|
||||
int limit;
|
||||
int offset;
|
||||
int want_total;
|
||||
} cases[] = {
|
||||
{"^schedule$", NULL, false, 0, 0, 6}, {"^sched", NULL, false, 0, 0, 10},
|
||||
{"^sched", "Function", false, 0, 0, 6}, {"^SCHED", NULL, true, 0, 0, 1},
|
||||
{"^Sched", NULL, true, 0, 0, 2}, {"^sche+dule$", NULL, false, 0, 0, 6},
|
||||
{"^s.hedule$", NULL, false, 0, 0, 6}, {"^get_", NULL, false, 0, 0, 2},
|
||||
{"^o'n", NULL, false, 0, 0, 1}, {"^zzz", NULL, false, 0, 0, 0},
|
||||
{"^sched", NULL, false, 3, 2, 10}, {"^sched", NULL, false, 4, 8, 10},
|
||||
};
|
||||
char fast[4096];
|
||||
char plain[4096];
|
||||
char twin[64];
|
||||
for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {
|
||||
cbm_search_params_t p = {.project = "p",
|
||||
.label = cases[c].label,
|
||||
.name_pattern = cases[c].pattern,
|
||||
.case_sensitive = cases[c].case_sensitive,
|
||||
.limit = cases[c].limit,
|
||||
.offset = cases[c].offset,
|
||||
.min_degree = -1,
|
||||
.max_degree = -1};
|
||||
int rc_fast = 0;
|
||||
int rc_plain = 0;
|
||||
int before = ranged;
|
||||
anchored_search_rows(s, &p, &rc_fast, fast, sizeof(fast));
|
||||
ASSERT_EQ(ranged - before, 2); /* count + page both took the range path */
|
||||
snprintf(twin, sizeof(twin), "(%s)", cases[c].pattern);
|
||||
p.name_pattern = twin;
|
||||
before = ranged;
|
||||
anchored_search_rows(s, &p, &rc_plain, plain, sizeof(plain));
|
||||
ASSERT_EQ(ranged - before, 0);
|
||||
ASSERT_EQ(rc_fast, CBM_STORE_OK);
|
||||
ASSERT_EQ(rc_plain, CBM_STORE_OK);
|
||||
ASSERT_STR_EQ(fast, plain);
|
||||
char want_total[32];
|
||||
snprintf(want_total, sizeof(want_total), "total=%d:", cases[c].want_total);
|
||||
ASSERT_EQ(strncmp(fast, want_total, strlen(want_total)), 0);
|
||||
}
|
||||
sqlite3_trace_v2(cbm_store_get_db(s), 0, NULL, NULL);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A regex the engine rejects fails the plain query as soon as one row reaches
|
||||
* it ("zabc" passes the LIKE '%abc%' hint). The range path would never reach
|
||||
* the regex (no name starts with "abc") and answer an empty page instead, so a
|
||||
* pattern that does not compile keeps the plain query and its error. */
|
||||
TEST(store_search_anchored_invalid_regex_keeps_error) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
ASSERT_NOT_NULL(s);
|
||||
cbm_store_upsert_project(s, "p", "/tmp/p");
|
||||
cbm_node_t n = {.project = "p", .label = "Function", .name = "zabc", .qualified_name = "p.zabc"};
|
||||
ASSERT_TRUE(cbm_store_upsert_node(s, &n) > 0);
|
||||
cbm_search_params_t p = {
|
||||
.project = "p", .name_pattern = "^abc(", .min_degree = -1, .max_degree = -1};
|
||||
cbm_search_output_t out = {0};
|
||||
ASSERT_EQ(cbm_store_search(s, &p, &out), CBM_STORE_ERR);
|
||||
cbm_store_search_free(&out);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── Impact: HopToRisk ─────────────────────────────────────────── */
|
||||
|
||||
TEST(store_hop_to_risk) {
|
||||
@@ -1873,6 +2090,9 @@ SUITE(store_search) {
|
||||
RUN_TEST(store_search_all);
|
||||
RUN_TEST(store_search_exclude_labels);
|
||||
RUN_TEST(store_search_case_insensitive);
|
||||
RUN_TEST(store_regex_anchored_prefix);
|
||||
RUN_TEST(store_search_anchored_name_matches_plain_scan);
|
||||
RUN_TEST(store_search_anchored_invalid_regex_keeps_error);
|
||||
RUN_TEST(store_bfs_outbound);
|
||||
RUN_TEST(store_bfs_inbound);
|
||||
RUN_TEST(store_bfs_cross_service);
|
||||
@@ -1881,6 +2101,7 @@ SUITE(store_search) {
|
||||
RUN_TEST(store_transaction_rollback);
|
||||
RUN_TEST(store_bulk_write_mode);
|
||||
RUN_TEST(store_schema_info);
|
||||
RUN_TEST(store_schema_counts_ties_in_name_order);
|
||||
RUN_TEST(store_dump_to_file);
|
||||
RUN_TEST(store_hop_to_risk);
|
||||
RUN_TEST(store_build_impact_summary);
|
||||
|
||||
Reference in New Issue
Block a user