mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-25 06:33:13 +08:00
perf(mcp): fast query tools on every platform
search_code - Scan the indexed files in process instead of through grep (POSIX) or a PowerShell Select-String pipeline (Windows), with the same grep -f semantics: one pattern per line, ERE regex, binary-file and NUL handling, -Hn records. The subprocess runner stays for the unindexed recursive fallback and a Windows file_pattern it cannot prefilter. - Windows now matches like macOS and Linux: Select-String matched case-insensitively, used .NET regex syntax and read NUL-containing files as text. A trailing CR is still ignored there, so `foo$` keeps matching CRLF checkouts. - Result merge, root containment and source reads no longer scale with hits x results: a node-id index, one canonical root per request with a per-file verdict cache, and fread spans instead of per-byte fgetc. query_graph - ORDER BY (RETURN and WITH) was a bubble sort. A stable merge sort keeps the same comparisons and so the same order; WITH keys are resolved once per row instead of twice per comparison. - GROUP BY, DISTINCT (RETURN and WITH) and COUNT(DISTINCT) looked every row up in a linear list. They hash the same keys now. Store reads - Read tools run their statements in one deferred read transaction, renewed every 100 ms, instead of taking the database file lock once per statement. list_projects - Resolve only the branch the listing prints: 2 git processes per project instead of 7. On macOS and Linux every response is byte-identical to before: search_code 30/30 (django), 28/28 (this repo) and 28/28 (Linux, GNU grep); all tools and the Cypher cases 86/86 on shared django and redis indexes. On django: search_code 908 -> 355 ms, WITH ... ORDER BY 41 s -> 62 ms, count(*) 508 -> 184 ms, list_projects 94 -> 40 ms. On the Windows ARM VM: search_code 1.6-3.4 s -> 0.2-0.5 s. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
+427
-81
@@ -3716,6 +3716,113 @@ static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Sorting and key lookup ────────────────────────────────────────
|
||||
* ORDER BY, GROUP BY and DISTINCT used to compare every row with every other
|
||||
* row (bubble sorts, linear group and duplicate scans). These helpers keep the
|
||||
* exact same comparisons and keys, in O(n log n) and O(n). */
|
||||
|
||||
/* Compare items a and b; positive means a sorts after b. */
|
||||
typedef int (*cypher_index_cmp_fn)(const void *ctx, int a, int b);
|
||||
|
||||
/* Stable sort of idx[0..n). Both this and the bubble sorts it replaces are
|
||||
* stable, so for any consistent ordering they give the same permutation. An
|
||||
* item moves before an earlier one only when cmp(earlier, item) > 0, which is
|
||||
* the bubble sorts' swap test. Without scratch memory it degrades to an
|
||||
* insertion sort with the same test and result. */
|
||||
static void cypher_stable_sort(int *idx, int n, cypher_index_cmp_fn cmp, const void *ctx) {
|
||||
if (n < PAIR_LEN) {
|
||||
return;
|
||||
}
|
||||
int *tmp = malloc((size_t)n * sizeof(int));
|
||||
if (!tmp) {
|
||||
for (int i = SKIP_ONE; i < n; i++) {
|
||||
for (int j = i; j > 0 && cmp(ctx, idx[j - SKIP_ONE], idx[j]) > 0; j--) {
|
||||
int swap = idx[j];
|
||||
idx[j] = idx[j - SKIP_ONE];
|
||||
idx[j - SKIP_ONE] = swap;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int64_t width = SKIP_ONE; width < n; width *= PAIR_LEN) {
|
||||
for (int64_t lo = 0; lo < n; lo += PAIR_LEN * width) {
|
||||
int64_t mid = lo + width < n ? lo + width : n;
|
||||
int64_t hi = mid + width < n ? mid + width : n;
|
||||
int64_t i = lo;
|
||||
int64_t j = mid;
|
||||
int64_t k = lo;
|
||||
while (i < mid && j < hi) {
|
||||
tmp[k++] = cmp(ctx, idx[i], idx[j]) > 0 ? idx[j++] : idx[i++];
|
||||
}
|
||||
while (i < mid) {
|
||||
tmp[k++] = idx[i++];
|
||||
}
|
||||
while (j < hi) {
|
||||
tmp[k++] = idx[j++];
|
||||
}
|
||||
}
|
||||
memcpy(idx, tmp, (size_t)n * sizeof(int));
|
||||
}
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
/* String key → the index it was first recorded with. Keys are copied: the
|
||||
* arrays they mirror are reallocated as they grow. */
|
||||
typedef struct {
|
||||
CBMHashTable *table;
|
||||
char **keys;
|
||||
int count;
|
||||
int cap;
|
||||
bool failed; /* an allocation failed: callers fall back to their scans */
|
||||
} cypher_key_index_t;
|
||||
|
||||
/* The recorded index, CYP_FOUND_NONE, or KEY_INDEX_UNKNOWN after a failed
|
||||
* allocation (the caller then scans, so a result never depends on memory). */
|
||||
enum { KEY_INDEX_UNKNOWN = -2 };
|
||||
|
||||
static int key_index_find(const cypher_key_index_t *ix, const char *key) {
|
||||
if (ix->failed) {
|
||||
return KEY_INDEX_UNKNOWN;
|
||||
}
|
||||
void *value = ix->table ? cbm_ht_get(ix->table, key) : NULL;
|
||||
return value ? (int)((intptr_t)value - SKIP_ONE) : CYP_FOUND_NONE;
|
||||
}
|
||||
|
||||
static void key_index_add(cypher_key_index_t *ix, const char *key, int index) {
|
||||
if (ix->failed) {
|
||||
return;
|
||||
}
|
||||
if (!ix->table) {
|
||||
ix->table = cbm_ht_create(CBM_SZ_256);
|
||||
}
|
||||
if (ix->table && ix->count >= ix->cap) {
|
||||
int cap = ix->cap ? ix->cap * PAIR_LEN : CBM_SZ_256;
|
||||
char **keys = realloc(ix->keys, (size_t)cap * sizeof(char *));
|
||||
if (keys) {
|
||||
ix->keys = keys;
|
||||
ix->cap = cap;
|
||||
}
|
||||
}
|
||||
char *copy = ix->table && ix->count < ix->cap ? heap_strdup(key) : NULL;
|
||||
if (!copy) {
|
||||
ix->failed = true;
|
||||
return;
|
||||
}
|
||||
ix->keys[ix->count++] = copy;
|
||||
(void)cbm_ht_set(ix->table, copy, (void *)(intptr_t)(index + SKIP_ONE));
|
||||
}
|
||||
|
||||
static void key_index_free(cypher_key_index_t *ix) {
|
||||
if (ix->table) {
|
||||
cbm_ht_free(ix->table);
|
||||
}
|
||||
for (int i = 0; i < ix->count; i++) {
|
||||
free(ix->keys[i]);
|
||||
}
|
||||
free(ix->keys);
|
||||
memset(ix, 0, sizeof(*ix));
|
||||
}
|
||||
|
||||
/* ── Result postprocessing helpers ─────────────────────────────── */
|
||||
|
||||
/* Find the column index for one ORDER BY key, checking both column names and
|
||||
@@ -3755,6 +3862,43 @@ static bool rb_is_numeric_column(const result_builder_t *rb, int col) {
|
||||
return false;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
const int *cols;
|
||||
const bool *numeric;
|
||||
const bool *descs;
|
||||
int keys;
|
||||
const char ***rows; /* the rows in their pre-sort order */
|
||||
const int *values; /* numeric keys parsed once: values[row * keys + k], or NULL */
|
||||
} rb_order_ctx_t;
|
||||
|
||||
/* One ORDER BY comparison: numeric columns by their (int)strtol value,
|
||||
* others by strcmp, a later key breaking ties; positive = a after b.
|
||||
* va/vb are the rows' parsed numeric keys, or NULL to parse here. */
|
||||
static int rb_order_compare_rows(const rb_order_ctx_t *ctx, const char **ra, const char **rb_row,
|
||||
const int *va, const int *vb) {
|
||||
int cmp = 0;
|
||||
for (int k = 0; k < ctx->keys && cmp == 0; k++) {
|
||||
if (ctx->numeric[k]) {
|
||||
int a = va ? va[k] : (int)strtol(ra[ctx->cols[k]], NULL, CBM_DECIMAL_BASE);
|
||||
int b = vb ? vb[k] : (int)strtol(rb_row[ctx->cols[k]], NULL, CBM_DECIMAL_BASE);
|
||||
cmp = (a > b) - (a < b);
|
||||
} else {
|
||||
cmp = strcmp(ra[ctx->cols[k]], rb_row[ctx->cols[k]]);
|
||||
}
|
||||
if (ctx->descs[k]) {
|
||||
cmp = -cmp;
|
||||
}
|
||||
}
|
||||
return cmp;
|
||||
}
|
||||
|
||||
static int rb_order_compare(const void *context, int a, int b) {
|
||||
const rb_order_ctx_t *ctx = context;
|
||||
const int *va = ctx->values ? ctx->values + (size_t)a * (size_t)ctx->keys : NULL;
|
||||
const int *vb = ctx->values ? ctx->values + (size_t)b * (size_t)ctx->keys : NULL;
|
||||
return rb_order_compare_rows(ctx, ctx->rows[a], ctx->rows[b], va, vb);
|
||||
}
|
||||
|
||||
static void rb_apply_order_by(result_builder_t *rb, const cbm_return_clause_t *ret) {
|
||||
if (ret->order_key_count == 0) {
|
||||
return;
|
||||
@@ -3776,30 +3920,46 @@ static void rb_apply_order_by(result_builder_t *rb, const cbm_return_clause_t *r
|
||||
descs[keys] = ret->order_descs[k];
|
||||
keys++;
|
||||
}
|
||||
if (keys == 0) {
|
||||
if (keys == 0 || rb->row_count < PAIR_LEN) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < rb->row_count - SKIP_ONE; i++) {
|
||||
for (int j = 0; j < rb->row_count - i - SKIP_ONE; j++) {
|
||||
int cmp = 0;
|
||||
for (int k = 0; k < keys && cmp == 0; k++) {
|
||||
if (numeric[k]) {
|
||||
cmp = (int)strtol(rb->rows[j][cols[k]], NULL, CBM_DECIMAL_BASE) -
|
||||
(int)strtol(rb->rows[j + SKIP_ONE][cols[k]], NULL, CBM_DECIMAL_BASE);
|
||||
} else {
|
||||
cmp = strcmp(rb->rows[j][cols[k]], rb->rows[j + SKIP_ONE][cols[k]]);
|
||||
}
|
||||
if (descs[k]) {
|
||||
cmp = -cmp;
|
||||
}
|
||||
}
|
||||
if (cmp > 0) {
|
||||
int n = rb->row_count;
|
||||
rb_order_ctx_t ctx = {.cols = cols, .numeric = numeric, .descs = descs, .keys = keys};
|
||||
int *idx = malloc((size_t)n * sizeof(int));
|
||||
const char ***rows = malloc((size_t)n * sizeof(const char **));
|
||||
if (!idx || !rows) {
|
||||
/* No scratch memory: insertion sort in place, same comparisons. */
|
||||
free(idx);
|
||||
free(rows);
|
||||
for (int i = SKIP_ONE; i < n; i++) {
|
||||
for (int j = i; j > 0 && rb_order_compare_rows(&ctx, rb->rows[j - SKIP_ONE],
|
||||
rb->rows[j], NULL, NULL) > 0;
|
||||
j--) {
|
||||
const char **tmp = rb->rows[j];
|
||||
rb->rows[j] = rb->rows[j + SKIP_ONE];
|
||||
rb->rows[j + SKIP_ONE] = tmp;
|
||||
rb->rows[j] = rb->rows[j - SKIP_ONE];
|
||||
rb->rows[j - SKIP_ONE] = tmp;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
int *values = malloc((size_t)n * (size_t)keys * sizeof(int));
|
||||
for (int r = 0; r < n; r++) {
|
||||
idx[r] = r;
|
||||
rows[r] = rb->rows[r];
|
||||
for (int k = 0; values && k < keys; k++) {
|
||||
values[(size_t)r * (size_t)keys + (size_t)k] =
|
||||
numeric[k] ? (int)strtol(rb->rows[r][cols[k]], NULL, CBM_DECIMAL_BASE) : 0;
|
||||
}
|
||||
}
|
||||
ctx.rows = rows;
|
||||
ctx.values = values;
|
||||
cypher_stable_sort(idx, n, rb_order_compare, &ctx);
|
||||
for (int r = 0; r < n; r++) {
|
||||
rb->rows[r] = rows[idx[r]];
|
||||
}
|
||||
free(values);
|
||||
free(rows);
|
||||
free(idx);
|
||||
}
|
||||
|
||||
static void rb_apply_skip_limit(result_builder_t *rb, int skip_n, int limit,
|
||||
@@ -3838,25 +3998,60 @@ static void rb_apply_skip_limit(result_builder_t *rb, int skip_n, int limit,
|
||||
}
|
||||
}
|
||||
|
||||
/* A row's DISTINCT identity: every column, length-prefixed so no value can
|
||||
* run into the next — equal keys exactly when every column is equal. Built in
|
||||
* *buf (grown as needed); NULL when out of memory. */
|
||||
static const char *rb_row_key(const result_builder_t *rb, const char **row, char **buf,
|
||||
size_t *cap) {
|
||||
size_t need = SKIP_ONE;
|
||||
for (int c = 0; c < rb->col_count; c++) {
|
||||
need += strlen(row[c]) + CBM_SZ_32;
|
||||
}
|
||||
if (need > *cap) {
|
||||
char *grown = realloc(*buf, need);
|
||||
if (!grown) {
|
||||
return NULL;
|
||||
}
|
||||
*buf = grown;
|
||||
*cap = need;
|
||||
}
|
||||
size_t used = 0;
|
||||
(*buf)[0] = '\0';
|
||||
for (int c = 0; c < rb->col_count; c++) {
|
||||
used += (size_t)snprintf(*buf + used, *cap - used, "%zu:%s", strlen(row[c]), row[c]);
|
||||
}
|
||||
return *buf;
|
||||
}
|
||||
|
||||
/* Whether row i equals one of the first `kept` rows, column by column. */
|
||||
static bool rb_row_seen_before(const result_builder_t *rb, int i, int kept) {
|
||||
for (int j = 0; j < kept; j++) {
|
||||
bool same = true;
|
||||
for (int c = 0; c < rb->col_count && same; c++) {
|
||||
same = strcmp(rb->rows[i][c], rb->rows[j][c]) == 0;
|
||||
}
|
||||
if (same) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void rb_apply_distinct(result_builder_t *rb) {
|
||||
if (rb->row_count <= SKIP_ONE) {
|
||||
return;
|
||||
}
|
||||
int kept = SKIP_ONE;
|
||||
for (int i = SKIP_ONE; i < rb->row_count; i++) {
|
||||
bool dup = false;
|
||||
for (int j = 0; j < kept && !dup; j++) {
|
||||
bool same = true;
|
||||
for (int c = 0; c < rb->col_count && same; c++) {
|
||||
if (strcmp(rb->rows[i][c], rb->rows[j][c]) != 0) {
|
||||
same = false;
|
||||
}
|
||||
}
|
||||
if (same) {
|
||||
dup = true;
|
||||
}
|
||||
}
|
||||
cypher_key_index_t seen = {0};
|
||||
char *key_buf = NULL;
|
||||
size_t key_cap = 0;
|
||||
int kept = 0;
|
||||
for (int i = 0; i < rb->row_count; i++) {
|
||||
const char *key = rb_row_key(rb, rb->rows[i], &key_buf, &key_cap);
|
||||
seen.failed = seen.failed || !key;
|
||||
int found = key_index_find(&seen, key);
|
||||
bool dup = found == KEY_INDEX_UNKNOWN ? rb_row_seen_before(rb, i, kept) : found >= 0;
|
||||
if (!dup) {
|
||||
key_index_add(&seen, key, kept);
|
||||
if (kept != i) {
|
||||
rb->rows[kept] = rb->rows[i];
|
||||
}
|
||||
@@ -3869,6 +4064,8 @@ static void rb_apply_distinct(result_builder_t *rb) {
|
||||
}
|
||||
}
|
||||
rb->row_count = kept;
|
||||
key_index_free(&seen);
|
||||
free(key_buf);
|
||||
}
|
||||
|
||||
static void rb_free(result_builder_t *rb) {
|
||||
@@ -4113,17 +4310,57 @@ static bool is_aggregate_func(const char *func) {
|
||||
strcmp(func, "MIN") == 0 || strcmp(func, "MAX") == 0 || strcmp(func, "COLLECT") == 0);
|
||||
}
|
||||
|
||||
/* Grow a string list for one more entry at `idx`: the capacity doubles at
|
||||
* every power of two, so n appends cost O(n) copying, not O(n^2). */
|
||||
static char **string_list_reserve(char **list, int idx) {
|
||||
if (idx > 0 && (idx & (idx - SKIP_ONE)) != 0) {
|
||||
return list; /* capacity is the next power of two above idx */
|
||||
}
|
||||
return safe_realloc(list, (size_t)(idx ? idx * PAIR_LEN : SKIP_ONE) * sizeof(char *));
|
||||
}
|
||||
|
||||
enum { DISTINCT_SET_MIN = 16 }; /* below this a set is scanned, not hashed */
|
||||
|
||||
/* Append `val` to a string list only if not already present — i.e. maintain a
|
||||
* set of distinct values. Used by COUNT(DISTINCT x) (#239). */
|
||||
static void distinct_list_add(char ***list, int *count, const char *val) {
|
||||
for (int i = 0; i < *count; i++) {
|
||||
if (strcmp((*list)[i], val) == 0) {
|
||||
* set of distinct values. Used by COUNT(DISTINCT x) (#239). Past
|
||||
* DISTINCT_SET_MIN values the membership test goes through *set, whose keys
|
||||
* are the list's own strings; a set that cannot be built stays a scan. */
|
||||
static void distinct_list_add(char ***list, int *count, CBMHashTable **set, const char *val) {
|
||||
if (*set) {
|
||||
if (cbm_ht_has(*set, val)) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < *count; i++) {
|
||||
if (strcmp((*list)[i], val) == 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
int idx = *count;
|
||||
*list = string_list_reserve(*list, idx);
|
||||
char *copy = *list ? heap_strdup(val) : NULL;
|
||||
if (!copy) {
|
||||
/* Out of memory: keep a consistent (possibly emptied) set. */
|
||||
if (!*list) {
|
||||
*count = 0;
|
||||
if (*set) {
|
||||
cbm_ht_free(*set);
|
||||
*set = NULL;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
(*list)[idx] = copy;
|
||||
(*count)++;
|
||||
if (*set) {
|
||||
(void)cbm_ht_set(*set, (*list)[idx], (void *)SKIP_ONE);
|
||||
} else if (*count >= DISTINCT_SET_MIN) {
|
||||
*set = cbm_ht_create(CBM_SZ_64);
|
||||
for (int i = 0; *set && i < *count; i++) {
|
||||
(void)cbm_ht_set(*set, (*list)[i], (void *)SKIP_ONE);
|
||||
}
|
||||
}
|
||||
int idx = (*count)++;
|
||||
*list = safe_realloc(*list, (size_t)(idx + SKIP_ONE) * sizeof(char *));
|
||||
(*list)[idx] = heap_strdup(val);
|
||||
}
|
||||
|
||||
/* Resolve one WITH ORDER BY key on a projected binding. The key is either a
|
||||
@@ -4150,31 +4387,109 @@ static const char *order_key_value(binding_t *b, const char *key) {
|
||||
return binding_get_virtual(b, var, dot + SKIP_ONE);
|
||||
}
|
||||
|
||||
/* Sort bindings by the ORDER BY key list (virtual variables) using bubble
|
||||
* sort; later keys break ties, direction is per key (#1334). */
|
||||
/* One WITH ORDER BY key comparison: numerically when both values start with a
|
||||
* number, else by strcmp; positive = a after b. */
|
||||
static int with_order_compare_value(const char *va, const char *vb) {
|
||||
char *ea = NULL;
|
||||
char *eb = NULL;
|
||||
double da = strtod(va, &ea);
|
||||
double db = strtod(vb, &eb);
|
||||
return (ea != va && eb != vb) ? ((da > db) - (da < db)) : strcmp(va, vb);
|
||||
}
|
||||
|
||||
/* The full comparison over the ORDER BY key list: later keys break ties,
|
||||
* direction is per key (#1334). */
|
||||
static int with_order_compare_bindings(const cbm_return_clause_t *wc, binding_t *a, binding_t *b) {
|
||||
int cmp = 0;
|
||||
for (int k = 0; k < wc->order_key_count && cmp == 0; k++) {
|
||||
cmp = with_order_compare_value(order_key_value(a, wc->order_keys[k]),
|
||||
order_key_value(b, wc->order_keys[k]));
|
||||
if (wc->order_descs[k]) {
|
||||
cmp = -cmp;
|
||||
}
|
||||
}
|
||||
return cmp;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
const cbm_return_clause_t *wc;
|
||||
char **values; /* values[i * keys + k]: owned copy of binding i's key k */
|
||||
bool *numbered; /* whether strtod consumed a number from that value */
|
||||
double *numbers;
|
||||
} with_order_ctx_t;
|
||||
|
||||
/* The same comparison on values resolved once per binding. order_key_value
|
||||
* may return a rotating thread-local buffer, so the values are copies. */
|
||||
static int with_order_compare(const void *context, int a, int b) {
|
||||
const with_order_ctx_t *ctx = context;
|
||||
size_t keys = (size_t)ctx->wc->order_key_count;
|
||||
int cmp = 0;
|
||||
for (size_t k = 0; k < keys && cmp == 0; k++) {
|
||||
size_t ia = (size_t)a * keys + k;
|
||||
size_t ib = (size_t)b * keys + k;
|
||||
if (ctx->numbered[ia] && ctx->numbered[ib]) {
|
||||
cmp = (ctx->numbers[ia] > ctx->numbers[ib]) - (ctx->numbers[ia] < ctx->numbers[ib]);
|
||||
} else {
|
||||
cmp = strcmp(ctx->values[ia], ctx->values[ib]);
|
||||
}
|
||||
if (ctx->wc->order_descs[k]) {
|
||||
cmp = -cmp;
|
||||
}
|
||||
}
|
||||
return cmp;
|
||||
}
|
||||
|
||||
/* Sort bindings by the ORDER BY key list (virtual variables), stably. */
|
||||
static void sort_bindings(binding_t *vbindings, int count, const cbm_return_clause_t *wc) {
|
||||
for (int i = 0; i < count - SKIP_ONE; i++) {
|
||||
for (int j = 0; j < count - i - SKIP_ONE; j++) {
|
||||
int cmp = 0;
|
||||
for (int k = 0; k < wc->order_key_count && cmp == 0; k++) {
|
||||
const char *va = order_key_value(&vbindings[j], wc->order_keys[k]);
|
||||
const char *vb2 = order_key_value(&vbindings[j + SKIP_ONE], wc->order_keys[k]);
|
||||
char *ea = NULL;
|
||||
char *eb = NULL;
|
||||
double da = strtod(va, &ea);
|
||||
double db = strtod(vb2, &eb);
|
||||
cmp = (ea != va && eb != vb2) ? ((da > db) - (da < db)) : strcmp(va, vb2);
|
||||
if (wc->order_descs[k]) {
|
||||
cmp = -cmp;
|
||||
}
|
||||
}
|
||||
if (cmp > 0) {
|
||||
if (count < PAIR_LEN || wc->order_key_count <= 0) {
|
||||
return;
|
||||
}
|
||||
size_t keys = (size_t)wc->order_key_count;
|
||||
size_t cells = (size_t)count * keys;
|
||||
with_order_ctx_t ctx = {.wc = wc,
|
||||
.values = calloc(cells, sizeof(char *)),
|
||||
.numbered = calloc(cells, sizeof(bool)),
|
||||
.numbers = calloc(cells, sizeof(double))};
|
||||
int *idx = malloc((size_t)count * sizeof(int));
|
||||
binding_t *sorted = malloc((size_t)count * sizeof(binding_t));
|
||||
bool ready = ctx.values && ctx.numbered && ctx.numbers && idx && sorted;
|
||||
for (int i = 0; ready && i < count; i++) {
|
||||
idx[i] = i;
|
||||
for (size_t k = 0; ready && k < keys; k++) {
|
||||
size_t cell = (size_t)i * keys + k;
|
||||
ctx.values[cell] = heap_strdup(order_key_value(&vbindings[i], wc->order_keys[k]));
|
||||
ready = ctx.values[cell] != NULL;
|
||||
char *end = NULL;
|
||||
ctx.numbers[cell] = ready ? strtod(ctx.values[cell], &end) : 0;
|
||||
ctx.numbered[cell] = ready && end != ctx.values[cell];
|
||||
}
|
||||
}
|
||||
if (ready) {
|
||||
cypher_stable_sort(idx, count, with_order_compare, &ctx);
|
||||
for (int i = 0; i < count; i++) {
|
||||
sorted[i] = vbindings[idx[i]];
|
||||
}
|
||||
memcpy(vbindings, sorted, (size_t)count * sizeof(binding_t));
|
||||
} else {
|
||||
/* No scratch memory: insertion sort in place, same comparisons. */
|
||||
for (int i = SKIP_ONE; i < count; i++) {
|
||||
for (int j = i; j > 0 && with_order_compare_bindings(wc, &vbindings[j - SKIP_ONE],
|
||||
&vbindings[j]) > 0;
|
||||
j--) {
|
||||
binding_t tmp = vbindings[j];
|
||||
vbindings[j] = vbindings[j + SKIP_ONE];
|
||||
vbindings[j + SKIP_ONE] = tmp;
|
||||
vbindings[j] = vbindings[j - SKIP_ONE];
|
||||
vbindings[j - SKIP_ONE] = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t c = 0; ctx.values && c < cells; c++) {
|
||||
free(ctx.values[c]);
|
||||
}
|
||||
free(ctx.values);
|
||||
free(ctx.numbered);
|
||||
free(ctx.numbers);
|
||||
free(idx);
|
||||
free(sorted);
|
||||
}
|
||||
|
||||
/* Apply skip and limit to a binding array, freeing discarded entries */
|
||||
@@ -4230,9 +4545,10 @@ typedef struct {
|
||||
double *sums;
|
||||
int *counts;
|
||||
double *mins, *maxs;
|
||||
char ***distinct_lists; /* per-item set of seen values for COUNT(DISTINCT) */
|
||||
int *distinct_n; /* per-item distinct count (#239) */
|
||||
cbm_node_t *group_nodes; /* per-item carried node (deep copy; id 0 = not a node) */
|
||||
char ***distinct_lists; /* per-item set of seen values for COUNT(DISTINCT) */
|
||||
int *distinct_n; /* per-item distinct count (#239) */
|
||||
CBMHashTable **distinct_sets; /* per-item hash over distinct_lists (lazy) */
|
||||
cbm_node_t *group_nodes; /* per-item carried node (deep copy; id 0 = not a node) */
|
||||
} with_agg_t;
|
||||
|
||||
/* A WITH item that is a bare, bound node variable (`WITH f`, `WITH f AS g`)
|
||||
@@ -4278,18 +4594,24 @@ static int with_agg_build_key(cbm_return_clause_t *wc, binding_t *b, char *key,
|
||||
|
||||
/* Find or create an aggregation group. Returns index. */
|
||||
static int with_agg_find_or_create(with_agg_t **aggs, int *agg_cnt, int *agg_cap,
|
||||
cbm_return_clause_t *wc, binding_t *b, const char *key) {
|
||||
for (int a = 0; a < *agg_cnt; a++) {
|
||||
cypher_key_index_t *groups, cbm_return_clause_t *wc,
|
||||
binding_t *b, const char *key) {
|
||||
int existing = key_index_find(groups, key);
|
||||
for (int a = 0; existing == KEY_INDEX_UNKNOWN && a < *agg_cnt; a++) {
|
||||
if (strcmp((*aggs)[a].group_key, key) == 0) {
|
||||
return a;
|
||||
existing = a;
|
||||
}
|
||||
}
|
||||
if (existing >= 0) {
|
||||
return existing;
|
||||
}
|
||||
if (*agg_cnt >= *agg_cap) {
|
||||
*agg_cap *= PAIR_LEN;
|
||||
*aggs = safe_realloc(*aggs, *agg_cap * sizeof(with_agg_t));
|
||||
}
|
||||
int found = (*agg_cnt)++;
|
||||
snprintf((*aggs)[found].group_key, sizeof((*aggs)[found].group_key), "%s", key);
|
||||
key_index_add(groups, (*aggs)[found].group_key, found);
|
||||
(*aggs)[found].group_vals = calloc(wc->count, sizeof(const char *));
|
||||
(*aggs)[found].sums = calloc(wc->count, sizeof(double));
|
||||
(*aggs)[found].counts = calloc(wc->count, sizeof(int));
|
||||
@@ -4297,6 +4619,7 @@ static int with_agg_find_or_create(with_agg_t **aggs, int *agg_cnt, int *agg_cap
|
||||
(*aggs)[found].maxs = calloc(wc->count, sizeof(double));
|
||||
(*aggs)[found].distinct_lists = calloc(wc->count, sizeof(char **));
|
||||
(*aggs)[found].distinct_n = calloc(wc->count, sizeof(int));
|
||||
(*aggs)[found].distinct_sets = calloc(wc->count, sizeof(CBMHashTable *));
|
||||
(*aggs)[found].group_nodes = calloc(wc->count, sizeof(cbm_node_t));
|
||||
for (int ci = 0; ci < wc->count; ci++) {
|
||||
(*aggs)[found].mins[ci] = CYP_DBL_MAX;
|
||||
@@ -4327,7 +4650,8 @@ static void with_agg_accumulate(with_agg_t *agg, cbm_return_clause_t *wc, bindin
|
||||
agg->counts[ci]++;
|
||||
const char *raw = binding_get_virtual(b, wc->items[ci].variable, wc->items[ci].property);
|
||||
if (wc->items[ci].distinct && strcmp(wc->items[ci].func, "COUNT") == 0) {
|
||||
distinct_list_add(&agg->distinct_lists[ci], &agg->distinct_n[ci], raw);
|
||||
distinct_list_add(&agg->distinct_lists[ci], &agg->distinct_n[ci],
|
||||
&agg->distinct_sets[ci], raw);
|
||||
}
|
||||
double dv = strtod(raw, NULL);
|
||||
agg->sums[ci] += dv;
|
||||
@@ -4373,6 +4697,9 @@ static void with_agg_free(with_agg_t *aggs, int agg_cnt, int item_count) {
|
||||
for (int a = 0; a < agg_cnt; a++) {
|
||||
for (int ci = 0; ci < item_count; ci++) {
|
||||
safe_str_free(&aggs[a].group_vals[ci]);
|
||||
if (aggs[a].distinct_sets && aggs[a].distinct_sets[ci]) {
|
||||
cbm_ht_free(aggs[a].distinct_sets[ci]);
|
||||
}
|
||||
if (aggs[a].distinct_lists && aggs[a].distinct_lists[ci]) {
|
||||
for (int j = 0; j < aggs[a].distinct_n[ci]; j++) {
|
||||
free(aggs[a].distinct_lists[ci][j]);
|
||||
@@ -4387,6 +4714,7 @@ static void with_agg_free(with_agg_t *aggs, int agg_cnt, int item_count) {
|
||||
free(aggs[a].maxs);
|
||||
free(aggs[a].distinct_lists);
|
||||
free(aggs[a].distinct_n);
|
||||
free(aggs[a].distinct_sets);
|
||||
if (aggs[a].group_nodes) {
|
||||
for (int ci = 0; ci < item_count; ci++) {
|
||||
node_fields_free(&aggs[a].group_nodes[ci]);
|
||||
@@ -4403,13 +4731,16 @@ static void execute_with_aggregate(cbm_return_clause_t *wc, binding_t *bindings,
|
||||
int agg_cap = CBM_SZ_256;
|
||||
with_agg_t *aggs = calloc(agg_cap, sizeof(with_agg_t));
|
||||
int agg_cnt = 0;
|
||||
cypher_key_index_t groups = {0};
|
||||
|
||||
for (int bi = 0; bi < bind_count; bi++) {
|
||||
char key[CBM_SZ_1K] = "";
|
||||
with_agg_build_key(wc, &bindings[bi], key, sizeof(key));
|
||||
int found = with_agg_find_or_create(&aggs, &agg_cnt, &agg_cap, wc, &bindings[bi], key);
|
||||
int found =
|
||||
with_agg_find_or_create(&aggs, &agg_cnt, &agg_cap, &groups, wc, &bindings[bi], key);
|
||||
with_agg_accumulate(&aggs[found], wc, &bindings[bi]);
|
||||
}
|
||||
key_index_free(&groups);
|
||||
|
||||
*vbindings = safe_realloc(*vbindings, (agg_cnt + SKIP_ONE) * sizeof(binding_t));
|
||||
if (!*vbindings) {
|
||||
@@ -4505,22 +4836,23 @@ static void with_proj_key(cbm_return_clause_t *wc, binding_t *b, char *key, size
|
||||
/* Apply WITH DISTINCT: drop projected rows whose value tuple duplicates an
|
||||
* earlier one, keeping first occurrence (#238 — previously silently ignored). */
|
||||
static void with_apply_distinct(cbm_return_clause_t *wc, binding_t *vbindings, int *vcount) {
|
||||
cypher_key_index_t seen = {0};
|
||||
int kept = 0;
|
||||
for (int i = 0; i < *vcount; i++) {
|
||||
char key[CBM_SZ_1K];
|
||||
with_proj_key(wc, &vbindings[i], key, sizeof(key));
|
||||
bool dup = false;
|
||||
for (int j = 0; j < kept; j++) {
|
||||
int found = key_index_find(&seen, key);
|
||||
for (int j = 0; found == KEY_INDEX_UNKNOWN && j < kept; j++) {
|
||||
char pkey[CBM_SZ_1K];
|
||||
with_proj_key(wc, &vbindings[j], pkey, sizeof(pkey));
|
||||
if (strcmp(key, pkey) == 0) {
|
||||
dup = true;
|
||||
break;
|
||||
found = j;
|
||||
}
|
||||
}
|
||||
if (dup) {
|
||||
if (found >= 0) {
|
||||
binding_free(&vbindings[i]);
|
||||
} else {
|
||||
key_index_add(&seen, key, kept);
|
||||
if (kept != i) {
|
||||
vbindings[kept] = vbindings[i];
|
||||
}
|
||||
@@ -4528,6 +4860,7 @@ static void with_apply_distinct(cbm_return_clause_t *wc, binding_t *vbindings, i
|
||||
}
|
||||
}
|
||||
*vcount = kept;
|
||||
key_index_free(&seen);
|
||||
}
|
||||
|
||||
static void execute_with_clause(cbm_query_t *q, binding_t **bindings_ptr, int *bind_count_ptr) {
|
||||
@@ -4774,6 +5107,7 @@ typedef struct {
|
||||
double *mins, *maxs;
|
||||
char ***collect_lists;
|
||||
int *collect_counts;
|
||||
CBMHashTable **distinct_sets; /* per-item hash over a COUNT(DISTINCT) list (lazy) */
|
||||
} ret_agg_entry_t;
|
||||
|
||||
/* Initialize a new RETURN aggregation group */
|
||||
@@ -4790,6 +5124,7 @@ static void ret_agg_init_group(ret_agg_entry_t *entry, const char *key, int item
|
||||
entry->maxs = calloc(item_count, sizeof(double));
|
||||
entry->collect_lists = calloc(item_count, sizeof(char **));
|
||||
entry->collect_counts = calloc(item_count, sizeof(int));
|
||||
entry->distinct_sets = calloc(item_count, sizeof(CBMHashTable *));
|
||||
for (int ci = 0; ci < item_count; ci++) {
|
||||
entry->mins[ci] = CYP_DBL_MAX;
|
||||
entry->maxs[ci] = -CYP_DBL_MAX;
|
||||
@@ -4836,13 +5171,18 @@ static void ret_agg_accumulate(ret_agg_entry_t *entry, cbm_return_clause_t *ret,
|
||||
entry->maxs[ci] = dv;
|
||||
}
|
||||
if (strcmp(ret->items[ci].func, "COLLECT") == 0) {
|
||||
int idx = entry->collect_counts[ci]++;
|
||||
entry->collect_lists[ci] =
|
||||
safe_realloc(entry->collect_lists[ci], (idx + SKIP_ONE) * sizeof(char *));
|
||||
int idx = entry->collect_counts[ci];
|
||||
entry->collect_lists[ci] = string_list_reserve(entry->collect_lists[ci], idx);
|
||||
if (!entry->collect_lists[ci]) {
|
||||
entry->collect_counts[ci] = 0;
|
||||
continue;
|
||||
}
|
||||
entry->collect_lists[ci][idx] = heap_strdup(raw);
|
||||
entry->collect_counts[ci]++;
|
||||
} else if (ret->items[ci].distinct && strcmp(ret->items[ci].func, "COUNT") == 0) {
|
||||
/* COUNT(DISTINCT x): track unique values; emit the set size (#239). */
|
||||
distinct_list_add(&entry->collect_lists[ci], &entry->collect_counts[ci], raw);
|
||||
distinct_list_add(&entry->collect_lists[ci], &entry->collect_counts[ci],
|
||||
&entry->distinct_sets[ci], raw);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4852,6 +5192,9 @@ static void ret_agg_free(ret_agg_entry_t *aggs, int agg_count, int item_count) {
|
||||
for (int a = 0; a < agg_count; a++) {
|
||||
for (int ci = 0; ci < item_count; ci++) {
|
||||
safe_str_free(&aggs[a].group_vals[ci]);
|
||||
if (aggs[a].distinct_sets && aggs[a].distinct_sets[ci]) {
|
||||
cbm_ht_free(aggs[a].distinct_sets[ci]);
|
||||
}
|
||||
for (int j = 0; j < aggs[a].collect_counts[ci]; j++) {
|
||||
free(aggs[a].collect_lists[ci][j]);
|
||||
}
|
||||
@@ -4864,6 +5207,7 @@ static void ret_agg_free(ret_agg_entry_t *aggs, int agg_count, int item_count) {
|
||||
free(aggs[a].maxs);
|
||||
free(aggs[a].collect_lists);
|
||||
free(aggs[a].collect_counts);
|
||||
free(aggs[a].distinct_sets);
|
||||
}
|
||||
free(aggs);
|
||||
}
|
||||
@@ -4921,10 +5265,11 @@ static void execute_return_agg(cbm_return_clause_t *ret, binding_t *bindings, in
|
||||
int agg_cap = CBM_SZ_256;
|
||||
ret_agg_entry_t *aggs = calloc(agg_cap, sizeof(ret_agg_entry_t));
|
||||
int agg_count = 0;
|
||||
cypher_key_index_t groups = {0};
|
||||
|
||||
for (int bi = 0; bi < bind_count; bi++) {
|
||||
/* #601: grouping is O(bindings x groups) — the dominant cost on a
|
||||
* whole-graph GROUP BY. Abort if we blow the wall-clock budget. */
|
||||
/* #601: a whole-graph GROUP BY visits every binding. Abort if we blow
|
||||
* the wall-clock budget. */
|
||||
if ((bi & CYPHER_DEADLINE_CHECK_MASK) == 0 && cypher_deadline_exceeded()) {
|
||||
break;
|
||||
}
|
||||
@@ -4933,11 +5278,10 @@ static void execute_return_agg(cbm_return_clause_t *ret, binding_t *bindings, in
|
||||
char valbufs[CBM_SZ_32][CBM_SZ_512];
|
||||
ret_agg_build_key(ret, &bindings[bi], key, sizeof(key), vals, valbufs);
|
||||
|
||||
int found = CYP_FOUND_NONE;
|
||||
for (int a = 0; a < agg_count; a++) {
|
||||
int found = key_index_find(&groups, key);
|
||||
for (int a = 0; found == KEY_INDEX_UNKNOWN && a < agg_count; a++) {
|
||||
if (strcmp(aggs[a].group_key, key) == 0) {
|
||||
found = a;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found < 0) {
|
||||
@@ -4947,9 +5291,11 @@ static void execute_return_agg(cbm_return_clause_t *ret, binding_t *bindings, in
|
||||
}
|
||||
found = agg_count++;
|
||||
ret_agg_init_group(&aggs[found], key, ret->count, vals);
|
||||
key_index_add(&groups, aggs[found].group_key, found);
|
||||
}
|
||||
ret_agg_accumulate(&aggs[found], ret, &bindings[bi]);
|
||||
}
|
||||
key_index_free(&groups);
|
||||
|
||||
for (int a = 0; a < agg_count; a++) {
|
||||
ret_agg_emit_row(ret, &aggs[a], rb);
|
||||
|
||||
@@ -313,6 +313,30 @@ int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cbm_git_branch_resolve(const char *path, bool *is_git, char **branch) {
|
||||
if (!is_git || !branch) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
*is_git = false;
|
||||
*branch = NULL;
|
||||
if (!path || !path[0]) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
if (!cbm_is_dir(path)) {
|
||||
return 0;
|
||||
}
|
||||
char *worktree_root = NULL;
|
||||
if (git_capture(path, "rev-parse --show-toplevel", &worktree_root) != 0) {
|
||||
return 0;
|
||||
}
|
||||
free(worktree_root);
|
||||
*is_git = true;
|
||||
if (git_capture(path, "symbolic-ref --quiet --short HEAD", branch) != 0) {
|
||||
*branch = git_strdup("DETACHED");
|
||||
}
|
||||
return *branch ? 0 : CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx) {
|
||||
const char *project = project_name && project_name[0] ? project_name : "project";
|
||||
const char *slug = "working-tree";
|
||||
|
||||
@@ -20,6 +20,10 @@ typedef struct {
|
||||
} cbm_git_context_t;
|
||||
|
||||
int cbm_git_context_resolve(const char *path, cbm_git_context_t *out);
|
||||
/* Only the is_git and branch fields of cbm_git_context_resolve, from the same
|
||||
* two git commands it uses for them (2 git processes instead of 7). *branch
|
||||
* is heap-owned, NULL when path is not in a git work tree. */
|
||||
int cbm_git_branch_resolve(const char *path, bool *is_git, char **branch);
|
||||
void cbm_git_context_free(cbm_git_context_t *ctx);
|
||||
char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx);
|
||||
int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size);
|
||||
|
||||
+674
-59
@@ -82,6 +82,7 @@ enum {
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/limits.h"
|
||||
#include "foundation/hash_table.h"
|
||||
#include "foundation/subprocess.h"
|
||||
#include "foundation/sha256.h"
|
||||
#include "mcp/index_supervisor.h"
|
||||
@@ -1863,6 +1864,7 @@ struct cbm_mcp_server {
|
||||
cbm_mcp_snapshot_read_test_hook_fn snapshot_read_test_hook;
|
||||
void *snapshot_read_test_context;
|
||||
#endif
|
||||
bool read_batch_tool; /* this request's store reads run in a read batch */
|
||||
size_t search_output_limit_override;
|
||||
const char *search_scan_command_override;
|
||||
const char *search_scratch_dir_override; /* NULL = cbm_tmpdir() */
|
||||
@@ -2648,7 +2650,11 @@ static cbm_store_t *resolve_store_internal(cbm_mcp_server_t *srv, const char *pr
|
||||
|
||||
static cbm_store_t *resolve_store(cbm_mcp_server_t *srv, const char *project) {
|
||||
/* Query-only callers (every read tool): strictly non-mutating resolve. */
|
||||
return resolve_store_internal(srv, project, false, false, NULL, false);
|
||||
cbm_store_t *store = resolve_store_internal(srv, project, false, false, NULL, false);
|
||||
if (store && srv->read_batch_tool) {
|
||||
(void)cbm_store_read_batch_begin(store);
|
||||
}
|
||||
return store;
|
||||
}
|
||||
|
||||
/* Forward decl — definition lives below in handle_trace_call_path's helpers. */
|
||||
@@ -3018,16 +3024,17 @@ static bool populate_project_record(const char *dir_path, bool include_stats,
|
||||
* null for non-git roots — cost ~10KB across a full cache and is one
|
||||
* index_status call away for the project you actually care about. */
|
||||
if (record->root_path[0]) {
|
||||
cbm_git_context_t gctx = {0};
|
||||
(void)cbm_git_context_resolve(record->root_path, &gctx);
|
||||
if (gctx.is_git && gctx.branch) {
|
||||
record->branch = heap_strdup(gctx.branch);
|
||||
bool is_git = false;
|
||||
char *branch = NULL;
|
||||
(void)cbm_git_branch_resolve(record->root_path, &is_git, &branch);
|
||||
if (is_git && branch) {
|
||||
record->branch = heap_strdup(branch);
|
||||
if (!record->branch) {
|
||||
cbm_git_context_free(&gctx);
|
||||
free(branch);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
cbm_git_context_free(&gctx);
|
||||
free(branch);
|
||||
}
|
||||
record->populated = true;
|
||||
return true;
|
||||
@@ -10431,20 +10438,33 @@ static char *read_file_lines(const char *path, int start, int end) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Whole line spans per memchr instead of one locked fgetc and one append
|
||||
* per byte: search_code's full mode reads the source of every result, and
|
||||
* the byte loop was half of a 500-result page. Same text-mode stream, so
|
||||
* Windows CRLF translation is unchanged. */
|
||||
enum { READ_FILE_LINES_CHUNK = 16384 };
|
||||
cbm_sb_t selected;
|
||||
cbm_sb_init(&selected);
|
||||
int lineno = 1;
|
||||
int byte;
|
||||
while ((byte = fgetc(fp)) != EOF) {
|
||||
if (lineno >= start && lineno <= end) {
|
||||
char ch = (char)byte;
|
||||
cbm_sb_append_n(&selected, &ch, 1U);
|
||||
}
|
||||
if (byte == '\n') {
|
||||
if (lineno >= end) {
|
||||
break;
|
||||
char chunk[READ_FILE_LINES_CHUNK];
|
||||
bool done = false;
|
||||
size_t got;
|
||||
while (!done && (got = fread(chunk, 1, sizeof(chunk), fp)) > 0) {
|
||||
size_t pos = 0;
|
||||
while (pos < got) {
|
||||
const char *newline = memchr(chunk + pos, '\n', got - pos);
|
||||
size_t span_end = newline ? (size_t)(newline - chunk) + SKIP_ONE : got;
|
||||
if (lineno >= start && lineno <= end) {
|
||||
cbm_sb_append_n(&selected, chunk + pos, span_end - pos);
|
||||
}
|
||||
pos = span_end;
|
||||
if (newline) {
|
||||
if (lineno >= end) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
lineno++;
|
||||
}
|
||||
lineno++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12738,6 +12758,64 @@ bool cbm_path_within_root(const char *root_path, const char *abs_path) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* cbm_path_within_root with the root resolved once per request: search_code
|
||||
* checks every hit file and every rendered result against the same root, and
|
||||
* resolving the root again for each check was a third of a 500-result page.
|
||||
* An unresolvable root fails every check, exactly as cbm_path_within_root. */
|
||||
typedef struct {
|
||||
char canonical[CBM_SZ_4K];
|
||||
bool resolved;
|
||||
/* Verdict per absolute path, for the life of one request: the reply is
|
||||
* rendered up to dozens of times while it is fitted to its byte budget,
|
||||
* and each render checks every result's file again. Keys are owned. */
|
||||
CBMHashTable *verdicts;
|
||||
} root_containment_t;
|
||||
|
||||
enum { ROOT_CONTAINMENT_INSIDE = 1, ROOT_CONTAINMENT_OUTSIDE = 2 };
|
||||
|
||||
static void root_containment_init(root_containment_t *containment, const char *root_path) {
|
||||
containment->verdicts = NULL;
|
||||
containment->resolved = root_path && resolve_canonical_path(root_path, containment->canonical,
|
||||
sizeof(containment->canonical));
|
||||
}
|
||||
|
||||
static void root_containment_free_key(const char *key, void *value, void *userdata) {
|
||||
(void)value;
|
||||
(void)userdata;
|
||||
free((void *)key);
|
||||
}
|
||||
|
||||
static void root_containment_free(root_containment_t *containment) {
|
||||
if (containment->verdicts) {
|
||||
cbm_ht_foreach(containment->verdicts, root_containment_free_key, NULL);
|
||||
cbm_ht_free(containment->verdicts);
|
||||
containment->verdicts = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static bool root_containment_check(root_containment_t *containment, const char *abs_path) {
|
||||
if (!containment->resolved || !abs_path) {
|
||||
return false;
|
||||
}
|
||||
void *cached = containment->verdicts ? cbm_ht_get(containment->verdicts, abs_path) : NULL;
|
||||
if (cached) {
|
||||
return (uintptr_t)cached == ROOT_CONTAINMENT_INSIDE;
|
||||
}
|
||||
char real_file[CBM_SZ_4K];
|
||||
bool inside = resolve_canonical_path(abs_path, real_file, sizeof(real_file)) &&
|
||||
canonical_path_has_root(containment->canonical, real_file);
|
||||
if (!containment->verdicts) {
|
||||
containment->verdicts = cbm_ht_create(0);
|
||||
}
|
||||
char *key = containment->verdicts ? heap_strdup(abs_path) : NULL;
|
||||
if (key) {
|
||||
(void)cbm_ht_set(
|
||||
containment->verdicts, key,
|
||||
(void *)(uintptr_t)(inside ? ROOT_CONTAINMENT_INSIDE : ROOT_CONTAINMENT_OUTSIDE));
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
/* #1690: a path-shaped `project` names a directory; the installed guidance tells
|
||||
* agents to pass their working directory. Turning it into the path-DERIVED name
|
||||
* alone missed a checkout indexed under an explicit name and every directory
|
||||
@@ -14285,7 +14363,7 @@ static yyjson_mut_val *build_dedup_files_array(yyjson_mut_doc *doc, search_resul
|
||||
/* Attach source or context lines to a search result JSON item. */
|
||||
static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, search_result_t *r,
|
||||
int mode, int context_lines, int source_max_lines,
|
||||
const char *root_path) {
|
||||
const char *root_path, root_containment_t *containment) {
|
||||
enum { MODE_FULL = 1 };
|
||||
if (r->start_line <= 0 || r->end_line <= 0) {
|
||||
return;
|
||||
@@ -14307,7 +14385,7 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear
|
||||
* project root (a `..` segment, or a symlink/junction that discovery
|
||||
* followed) must not be read back into the response. Same guard the
|
||||
* snippet path already uses. */
|
||||
if (!cbm_path_within_root(root_path, abs_path)) {
|
||||
if (!root_containment_check(containment, abs_path)) {
|
||||
free(abs_path);
|
||||
return;
|
||||
}
|
||||
@@ -14712,7 +14790,8 @@ static char *assemble_search_output(
|
||||
int gm_count, int result_start, int result_limit, int output_count, int raw_start,
|
||||
int raw_output, int directory_start, int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines, int source_max_lines, const char *root_path,
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit) {
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit,
|
||||
root_containment_t *containment) {
|
||||
enum { MODE_COMPACT = 0, MODE_FULL = 1, MODE_FILES = 2, SEARCH_SLOW_MS = 5000 };
|
||||
|
||||
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
|
||||
@@ -14769,7 +14848,8 @@ static char *assemble_search_output(
|
||||
yyjson_mut_arr_add_int(doc, row, r->out_degree);
|
||||
if (mode == MODE_FULL || attach_context) {
|
||||
yyjson_mut_val *src = yyjson_mut_obj(doc);
|
||||
attach_result_source(doc, src, r, mode, context_lines, source_max_lines, root_path);
|
||||
attach_result_source(doc, src, r, mode, context_lines, source_max_lines, root_path,
|
||||
containment);
|
||||
yyjson_mut_arr_add_val(row, src);
|
||||
}
|
||||
yyjson_mut_arr_add_val(results_arr, row);
|
||||
@@ -14953,15 +15033,13 @@ static char *assemble_search_output(
|
||||
* form. Compact search has a purpose-built table renderer so repeated paths
|
||||
* and qualified-name prefixes can use the response-local directory. Full and
|
||||
* files mode first build the canonical JSON model, then project it to tree. */
|
||||
static char *render_search_payload(search_result_t *sr, int sr_count, grep_match_t *raw,
|
||||
int raw_count, int raw_content_truncated, int gm_count,
|
||||
int result_start, int result_limit, int output_count,
|
||||
int raw_start, int raw_output, int directory_start,
|
||||
int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines,
|
||||
int source_max_lines, const char *root_path, bool scan_saturated,
|
||||
bool warn_literal_pipe, const search_metrics_t *metrics,
|
||||
bool budget_hit, bool json_format) {
|
||||
static char *render_search_payload(
|
||||
search_result_t *sr, int sr_count, grep_match_t *raw, int raw_count, int raw_content_truncated,
|
||||
int gm_count, int result_start, int result_limit, int output_count, int raw_start,
|
||||
int raw_output, int directory_start, int directory_output, int raw_limit, int directory_limit,
|
||||
int match_limit, int mode, int context_lines, int source_max_lines, const char *root_path,
|
||||
bool scan_saturated, bool warn_literal_pipe, const search_metrics_t *metrics, bool budget_hit,
|
||||
bool json_format, root_containment_t *request_containment) {
|
||||
if (mode == 0 && !json_format) {
|
||||
return assemble_search_output_toon(sr, sr_count, raw, raw_count, raw_content_truncated,
|
||||
gm_count, result_start, result_limit, output_count,
|
||||
@@ -14970,11 +15048,22 @@ static char *render_search_payload(search_result_t *sr, int sr_count, grep_match
|
||||
warn_literal_pipe, metrics, budget_hit, -1);
|
||||
}
|
||||
|
||||
/* A caller that renders once passes NULL; search_code passes the
|
||||
* containment it keeps for the whole request (see root_containment_t). */
|
||||
root_containment_t local_containment;
|
||||
root_containment_t *containment = request_containment;
|
||||
if (!containment) {
|
||||
root_containment_init(&local_containment, root_path);
|
||||
containment = &local_containment;
|
||||
}
|
||||
char *json = assemble_search_output(
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start, result_limit,
|
||||
output_count, raw_start, raw_output, directory_start, directory_output, raw_limit,
|
||||
directory_limit, match_limit, mode, context_lines, source_max_lines, root_path,
|
||||
scan_saturated, warn_literal_pipe, metrics, budget_hit);
|
||||
scan_saturated, warn_literal_pipe, metrics, budget_hit, containment);
|
||||
if (!request_containment) {
|
||||
root_containment_free(&local_containment);
|
||||
}
|
||||
if (!json || json_format) {
|
||||
return json;
|
||||
}
|
||||
@@ -15016,7 +15105,7 @@ char *cbm_mcp_render_search_rows_for_testing(const char *const *qualified_names,
|
||||
char *payload =
|
||||
render_search_payload(results, row_count, NULL, 0, 0, row_count, 0, row_count, row_count, 0,
|
||||
0, 0, row_count, 0, row_count > 0 ? 1 : 0, row_count > 0 ? 1 : 0, 0,
|
||||
0, 0, "", false, false, &zero_metrics, false, json_format);
|
||||
0, 0, "", false, false, &zero_metrics, false, json_format, NULL);
|
||||
free_search_results(results, initialized);
|
||||
return payload;
|
||||
}
|
||||
@@ -15250,7 +15339,7 @@ char *cbm_mcp_render_raw_preview_for_testing(const char *content, bool content_o
|
||||
search_raw_preview(&raw, content, content_offset_set, content_offset, false, 0, 0);
|
||||
return render_search_payload(NULL, 0, &raw, 1, raw.content_truncated ? 1 : 0, 1, 0, 0, 0, 0, 1,
|
||||
0, 0, 1, 0, 0, 0, 0, 0, "", false, false, &zero_metrics, false,
|
||||
json_format);
|
||||
json_format, NULL);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -15324,13 +15413,89 @@ static bool append_search_match_line(search_result_t *result, int line) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a grep hit to the search result set (merge into existing or create new). */
|
||||
static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap, cbm_node_t *n,
|
||||
int line) {
|
||||
for (int j = 0; j < *sr_count; j++) {
|
||||
if ((*sr)[j].node_id == n->id) {
|
||||
return append_search_match_line(&(*sr)[j], line);
|
||||
/* Node id -> index into the search result array, so merging a hit into its
|
||||
* node's result is O(1). Finding it by scanning every result so far made a
|
||||
* common pattern quadratic: `def ` on django is 33,672 hits against a result
|
||||
* list that grows to tens of thousands. Open addressing over a power-of-two
|
||||
* table; a slot stores index + 1, so 0 is free and no node id is reserved. */
|
||||
typedef struct {
|
||||
int64_t *ids;
|
||||
int *slots;
|
||||
int capacity;
|
||||
int count;
|
||||
} search_result_index_t;
|
||||
|
||||
enum { SEARCH_RESULT_INDEX_MIN_CAPACITY = 64 };
|
||||
|
||||
static uint32_t search_result_index_hash(int64_t id, int capacity) {
|
||||
uint64_t mixed = (uint64_t)id * UINT64_C(0x9E3779B97F4A7C15);
|
||||
return (uint32_t)(mixed >> 32) & (uint32_t)(capacity - 1);
|
||||
}
|
||||
|
||||
static void search_result_index_free(search_result_index_t *index) {
|
||||
free(index->ids);
|
||||
free(index->slots);
|
||||
memset(index, 0, sizeof(*index));
|
||||
}
|
||||
|
||||
static int search_result_index_find(const search_result_index_t *index, int64_t id) {
|
||||
if (index->capacity == 0) {
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
uint32_t at = search_result_index_hash(id, index->capacity);
|
||||
while (index->slots[at] != 0) {
|
||||
if (index->ids[at] == id) {
|
||||
return index->slots[at] - SKIP_ONE;
|
||||
}
|
||||
at = (at + SKIP_ONE) & (uint32_t)(index->capacity - 1);
|
||||
}
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
|
||||
static bool search_result_index_put(search_result_index_t *index, int64_t id, int value);
|
||||
|
||||
static bool search_result_index_grow(search_result_index_t *index) {
|
||||
int capacity = index->capacity ? index->capacity * PAIR_LEN : SEARCH_RESULT_INDEX_MIN_CAPACITY;
|
||||
search_result_index_t grown = {
|
||||
.ids = calloc((size_t)capacity, sizeof(int64_t)),
|
||||
.slots = calloc((size_t)capacity, sizeof(int)),
|
||||
.capacity = capacity,
|
||||
};
|
||||
if (!grown.ids || !grown.slots) {
|
||||
search_result_index_free(&grown);
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < index->capacity; i++) {
|
||||
if (index->slots[i] != 0) {
|
||||
(void)search_result_index_put(&grown, index->ids[i], index->slots[i] - SKIP_ONE);
|
||||
}
|
||||
}
|
||||
search_result_index_free(index);
|
||||
*index = grown;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool search_result_index_put(search_result_index_t *index, int64_t id, int value) {
|
||||
if ((index->count + SKIP_ONE) * PAIR_LEN > index->capacity &&
|
||||
!search_result_index_grow(index)) {
|
||||
return false;
|
||||
}
|
||||
uint32_t at = search_result_index_hash(id, index->capacity);
|
||||
while (index->slots[at] != 0) {
|
||||
at = (at + SKIP_ONE) & (uint32_t)(index->capacity - 1);
|
||||
}
|
||||
index->ids[at] = id;
|
||||
index->slots[at] = value + SKIP_ONE;
|
||||
index->count++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a grep hit to the search result set (merge into existing or create new). */
|
||||
static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap,
|
||||
search_result_index_t *index, cbm_node_t *n, int line) {
|
||||
int existing = search_result_index_find(index, n->id);
|
||||
if (existing >= 0) {
|
||||
return append_search_match_line(&(*sr)[existing], line);
|
||||
}
|
||||
if (*sr_count >= *sr_cap) {
|
||||
int next_capacity = *sr_cap * PAIR_LEN;
|
||||
@@ -15353,18 +15518,26 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c
|
||||
r->file = NULL;
|
||||
return false;
|
||||
}
|
||||
if (!search_result_index_put(index, n->id, *sr_count)) {
|
||||
free(r->qualified_name);
|
||||
free(r->file);
|
||||
free(r->match_lines);
|
||||
memset(r, 0, sizeof(*r));
|
||||
return false;
|
||||
}
|
||||
(*sr_count)++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Match a single grep hit to the tightest containing node, then add to sr or raw. */
|
||||
static bool classify_grep_hit(grep_match_t *hit, cbm_node_t *file_nodes, int file_node_count,
|
||||
search_result_t **sr, int *sr_count, int *sr_cap, grep_match_t **raw,
|
||||
int raw_offset, int raw_limit, int *raw_count, int *raw_stored_count,
|
||||
int *raw_cap, int *raw_content_truncated) {
|
||||
search_result_t **sr, int *sr_count, int *sr_cap,
|
||||
search_result_index_t *sr_index, grep_match_t **raw, int raw_offset,
|
||||
int raw_limit, int *raw_count, int *raw_stored_count, int *raw_cap,
|
||||
int *raw_content_truncated) {
|
||||
int best = find_tightest_node(file_nodes, file_node_count, hit->line);
|
||||
if (best >= 0) {
|
||||
return add_to_search_results(sr, sr_count, sr_cap, &file_nodes[best], hit->line);
|
||||
return add_to_search_results(sr, sr_count, sr_cap, sr_index, &file_nodes[best], hit->line);
|
||||
} else {
|
||||
if (*raw_count == INT_MAX) {
|
||||
return false;
|
||||
@@ -15421,13 +15594,14 @@ static void free_file_nodes(cbm_node_t *nodes, int count) {
|
||||
* applies before it reads a result's source, applied to the match itself,
|
||||
* once per distinct file (grep groups a file's hits). A path too long for
|
||||
* the canonical-path buffers is one that check cannot resolve either. */
|
||||
static bool search_hit_file_within_root(const char *root_path, const char *file) {
|
||||
static bool search_hit_file_within_root(root_containment_t *containment, const char *root_path,
|
||||
const char *file) {
|
||||
char abs_path[CBM_SZ_4K];
|
||||
int length = snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, file);
|
||||
if (length < 0 || (size_t)length >= sizeof(abs_path)) {
|
||||
return false;
|
||||
}
|
||||
return cbm_path_within_root(root_path, abs_path);
|
||||
return root_containment_check(containment, abs_path);
|
||||
}
|
||||
|
||||
/* Parse and classify the complete grep stream without retaining one object per
|
||||
@@ -15446,6 +15620,16 @@ static bool scan_and_classify_grep_matches(
|
||||
bool current_file_inside = false;
|
||||
cbm_node_t *file_nodes = NULL;
|
||||
int file_node_count = 0;
|
||||
root_containment_t containment;
|
||||
root_containment_init(&containment, root_path);
|
||||
search_result_index_t sr_index = {0};
|
||||
for (int i = 0; i < *sr_count; i++) {
|
||||
if (!search_result_index_put(&sr_index, (*sr)[i].node_id, i)) {
|
||||
search_result_index_free(&sr_index);
|
||||
root_containment_free(&containment);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
cbm_regex_t content_regex;
|
||||
bool content_regex_ready = use_regex && pattern &&
|
||||
cbm_regcomp(&content_regex, pattern, CBM_REG_EXTENDED) == CBM_REG_OK;
|
||||
@@ -15499,7 +15683,7 @@ static bool scan_and_classify_grep_matches(
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
current_file_inside = search_hit_file_within_root(root_path, file);
|
||||
current_file_inside = search_hit_file_within_root(&containment, root_path, file);
|
||||
if (store && current_file_inside) {
|
||||
(void)cbm_store_find_nodes_by_file(store, project, file, &file_nodes,
|
||||
&file_node_count);
|
||||
@@ -15531,8 +15715,8 @@ static bool scan_and_classify_grep_matches(
|
||||
&match_start, &match_end);
|
||||
search_raw_preview(&hit, content, raw_content_offset_set, raw_content_offset, match_known,
|
||||
match_start, match_end);
|
||||
if (!classify_grep_hit(&hit, file_nodes, file_node_count, sr, sr_count, sr_cap, raw,
|
||||
raw_offset, raw_limit, raw_count, raw_stored_count, raw_cap,
|
||||
if (!classify_grep_hit(&hit, file_nodes, file_node_count, sr, sr_count, sr_cap, &sr_index,
|
||||
raw, raw_offset, raw_limit, raw_count, raw_stored_count, raw_cap,
|
||||
raw_content_truncated)) {
|
||||
free(hit.file);
|
||||
ok = false;
|
||||
@@ -15541,6 +15725,8 @@ static bool scan_and_classify_grep_matches(
|
||||
free(hit.file);
|
||||
}
|
||||
|
||||
search_result_index_free(&sr_index);
|
||||
root_containment_free(&containment);
|
||||
if (content_regex_ready) {
|
||||
cbm_regfree(&content_regex);
|
||||
}
|
||||
@@ -15832,6 +16018,372 @@ static void search_scratch_close(search_scratch_t *scratch) {
|
||||
}
|
||||
}
|
||||
|
||||
/* ── In-process content scan ────────────────────────────────────────
|
||||
* search_code used to hand the scoped file list to `xargs sh grep` on POSIX and
|
||||
* to a PowerShell Select-String pipeline on Windows: a process spawn per call,
|
||||
* BSD grep's ~130 MB/s on macOS, and one cmdlet invocation per file on Windows
|
||||
* (23-31 s per call on GitHub's windows-11-arm image, against the 30 s scan
|
||||
* deadline). This scanner reads the same list and writes the records
|
||||
* `grep -Hn` printed, so the classifier and everything after it are unchanged:
|
||||
* - patterns: the pattern split on '\n', as `grep -f` reads its file; an
|
||||
* empty pattern matches every line (grep special-cases it in -E mode too);
|
||||
* - literal mode is a substring match (grep -F); regex mode runs each line
|
||||
* through cbm_regex, the platform's POSIX ERE engine (grep -E);
|
||||
* - a NUL byte in the first 32 KiB marks the file binary and yields no
|
||||
* record, as grep prints no lines for it; after a later NUL nothing more
|
||||
* is reported, as GNU grep stops line output there;
|
||||
* - a record is `<listed path><sep><line number><sep><line bytes>\n` with
|
||||
* ':' (POSIX) or TAB (Windows), the separators the classifier reads; the
|
||||
* line keeps a CR, which the classifier strips as before.
|
||||
* Deadline, cancellation and the output limit end the scan with the causes the
|
||||
* subprocess runner reported; an unreadable listed file fails the scan closed,
|
||||
* as grep's error exit did. */
|
||||
enum {
|
||||
SEARCH_SCAN_BINARY_PROBE = 32768,
|
||||
SEARCH_SCAN_CHECK_EVERY = 4096, /* lines between deadline/cancel checks */
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
char **items;
|
||||
size_t *lengths;
|
||||
int count;
|
||||
cbm_regex_t *regexes; /* regex mode: one per non-empty pattern */
|
||||
int compiled;
|
||||
bool any_empty; /* an empty pattern: every line matches */
|
||||
} search_scan_patterns_t;
|
||||
|
||||
static void search_scan_patterns_free(search_scan_patterns_t *patterns) {
|
||||
for (int i = 0; i < patterns->count; i++) {
|
||||
free(patterns->items[i]);
|
||||
}
|
||||
for (int i = 0; patterns->regexes && i < patterns->count; i++) {
|
||||
if (i < patterns->compiled && patterns->lengths[i] > 0) {
|
||||
cbm_regfree(&patterns->regexes[i]);
|
||||
}
|
||||
}
|
||||
free(patterns->items);
|
||||
free(patterns->lengths);
|
||||
free(patterns->regexes);
|
||||
memset(patterns, 0, sizeof(*patterns));
|
||||
}
|
||||
|
||||
static bool search_scan_patterns_parse(search_scan_patterns_t *patterns, const char *pattern,
|
||||
bool use_regex) {
|
||||
memset(patterns, 0, sizeof(*patterns));
|
||||
int count = 1;
|
||||
for (const char *p = pattern; *p; p++) {
|
||||
count += *p == '\n' ? SKIP_ONE : 0;
|
||||
}
|
||||
patterns->items = calloc((size_t)count, sizeof(char *));
|
||||
patterns->lengths = calloc((size_t)count, sizeof(size_t));
|
||||
patterns->regexes = use_regex ? calloc((size_t)count, sizeof(cbm_regex_t)) : NULL;
|
||||
if (!patterns->items || !patterns->lengths || (use_regex && !patterns->regexes)) {
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
const char *start = pattern;
|
||||
for (int i = 0; i < count; i++) {
|
||||
const char *end = strchr(start, '\n');
|
||||
size_t length = end ? (size_t)(end - start) : strlen(start);
|
||||
patterns->items[i] = malloc(length + SKIP_ONE);
|
||||
if (!patterns->items[i]) {
|
||||
patterns->count = i;
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
memcpy(patterns->items[i], start, length);
|
||||
patterns->items[i][length] = '\0';
|
||||
patterns->lengths[i] = length;
|
||||
patterns->any_empty = patterns->any_empty || length == 0;
|
||||
start = end ? end + SKIP_ONE : start + length;
|
||||
}
|
||||
patterns->count = count;
|
||||
for (int i = 0; use_regex && i < count; i++) {
|
||||
if (patterns->lengths[i] > 0 &&
|
||||
cbm_regcomp(&patterns->regexes[i], patterns->items[i],
|
||||
CBM_REG_EXTENDED | CBM_REG_NOSUB) != CBM_REG_OK) {
|
||||
patterns->compiled = i;
|
||||
search_scan_patterns_free(patterns);
|
||||
return false;
|
||||
}
|
||||
patterns->compiled = i + SKIP_ONE;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* First occurrence of needle in hay; memmem where the C library has it. */
|
||||
static const char *search_scan_find(const char *hay, size_t hay_length, const char *needle,
|
||||
size_t needle_length) {
|
||||
if (needle_length == 0) {
|
||||
return hay;
|
||||
}
|
||||
if (needle_length > hay_length) {
|
||||
return NULL;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
const char *last = hay + (hay_length - needle_length);
|
||||
for (const char *p = hay; p <= last;) {
|
||||
p = memchr(p, (unsigned char)needle[0], (size_t)(last - p) + SKIP_ONE);
|
||||
if (!p) {
|
||||
return NULL;
|
||||
}
|
||||
if (memcmp(p, needle, needle_length) == 0) {
|
||||
return p;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
return NULL;
|
||||
#else
|
||||
return memmem(hay, hay_length, needle, needle_length);
|
||||
#endif
|
||||
}
|
||||
|
||||
static size_t search_scan_count_newlines(const char *from, const char *to) {
|
||||
size_t count = 0;
|
||||
while (from < to) {
|
||||
const char *newline = memchr(from, '\n', (size_t)(to - from));
|
||||
if (!newline) {
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
from = newline + SKIP_ONE;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
FILE *out;
|
||||
size_t written;
|
||||
size_t output_limit;
|
||||
char separator;
|
||||
} search_scan_sink_t;
|
||||
|
||||
/* One `grep -Hn` record. False on a write error or once past the output limit. */
|
||||
static bool search_scan_emit(search_scan_sink_t *sink, const char *path, size_t path_length,
|
||||
size_t line_number, const char *line, size_t line_length,
|
||||
bool *over_limit) {
|
||||
char number[CBM_SZ_32];
|
||||
int number_length =
|
||||
snprintf(number, sizeof(number), "%c%zu%c", sink->separator, line_number, sink->separator);
|
||||
if (number_length <= 0 || (size_t)number_length >= sizeof(number) ||
|
||||
fwrite(path, 1, path_length, sink->out) != path_length ||
|
||||
fwrite(number, 1, (size_t)number_length, sink->out) != (size_t)number_length ||
|
||||
fwrite(line, 1, line_length, sink->out) != line_length || fputc('\n', sink->out) == EOF) {
|
||||
return false;
|
||||
}
|
||||
sink->written += path_length + (size_t)number_length + line_length + SKIP_ONE;
|
||||
if (sink->output_limit > 0 && sink->written > sink->output_limit) {
|
||||
*over_limit = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool search_scan_line_matches(const search_scan_patterns_t *patterns, bool use_regex,
|
||||
const char *line) {
|
||||
if (patterns->any_empty) {
|
||||
return true;
|
||||
}
|
||||
for (int i = 0; i < patterns->count; i++) {
|
||||
if (use_regex ? cbm_regexec(&patterns->regexes[i], line, 0, NULL, 0) == CBM_REG_OK
|
||||
: strstr(line, patterns->items[i]) != NULL) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Scan one file. `buffer` holds its bytes plus one spare byte. */
|
||||
static mcp_scan_cause_t search_scan_buffer(cbm_mcp_server_t *srv, search_scan_sink_t *sink,
|
||||
const search_scan_patterns_t *patterns, bool use_regex,
|
||||
const char *path, size_t path_length, char *buffer,
|
||||
size_t length, uint64_t deadline_ms) {
|
||||
size_t probe = length < SEARCH_SCAN_BINARY_PROBE ? length : SEARCH_SCAN_BINARY_PROBE;
|
||||
if (memchr(buffer, '\0', probe)) {
|
||||
return MCP_SCAN_SUCCESS;
|
||||
}
|
||||
const char *nul = memchr(buffer, '\0', length);
|
||||
if (nul) {
|
||||
/* Lines up to the last complete line before the NUL. */
|
||||
size_t keep = 0;
|
||||
for (size_t i = (size_t)(nul - buffer); i > 0; i--) {
|
||||
if (buffer[i - SKIP_ONE] == '\n') {
|
||||
keep = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
length = keep;
|
||||
}
|
||||
buffer[length] = '\0';
|
||||
bool over_limit = false;
|
||||
size_t line_number = 1;
|
||||
size_t since_check = 0;
|
||||
bool single_literal = !use_regex && !patterns->any_empty && patterns->count == 1;
|
||||
size_t pos = 0;
|
||||
while (pos < length) {
|
||||
if (++since_check >= SEARCH_SCAN_CHECK_EVERY) {
|
||||
since_check = 0;
|
||||
if (mcp_request_cancelled(srv)) {
|
||||
return MCP_SCAN_CANCELLED;
|
||||
}
|
||||
if (cbm_now_ms() >= deadline_ms) {
|
||||
return MCP_SCAN_DEADLINE;
|
||||
}
|
||||
}
|
||||
size_t line_start = pos;
|
||||
if (single_literal) {
|
||||
/* Jump to the next occurrence instead of visiting every line. */
|
||||
const char *hit = search_scan_find(buffer + pos, length - pos, patterns->items[0],
|
||||
patterns->lengths[0]);
|
||||
if (!hit) {
|
||||
break;
|
||||
}
|
||||
line_start = (size_t)(hit - buffer);
|
||||
while (line_start > pos && buffer[line_start - SKIP_ONE] != '\n') {
|
||||
line_start--;
|
||||
}
|
||||
line_number += search_scan_count_newlines(buffer + pos, buffer + line_start);
|
||||
}
|
||||
char *newline = memchr(buffer + line_start, '\n', length - line_start);
|
||||
size_t line_end = newline ? (size_t)(newline - buffer) : length;
|
||||
/* Windows matched Select-String's Line, which has no trailing CR, so
|
||||
* `foo$` matched a CRLF line there; POSIX grep matches with the CR. */
|
||||
size_t match_end = line_end;
|
||||
#ifdef _WIN32
|
||||
if (match_end > line_start && buffer[match_end - SKIP_ONE] == '\r') {
|
||||
match_end--;
|
||||
}
|
||||
#endif
|
||||
char saved = buffer[match_end];
|
||||
buffer[match_end] = '\0';
|
||||
bool matched =
|
||||
single_literal || search_scan_line_matches(patterns, use_regex, buffer + line_start);
|
||||
buffer[match_end] = saved;
|
||||
if (newline) {
|
||||
*newline = '\n';
|
||||
}
|
||||
if (matched && !search_scan_emit(sink, path, path_length, line_number, buffer + line_start,
|
||||
line_end - line_start, &over_limit)) {
|
||||
return over_limit ? MCP_SCAN_OUTPUT_LIMIT : MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
pos = line_end + SKIP_ONE;
|
||||
line_number++;
|
||||
}
|
||||
return MCP_SCAN_SUCCESS;
|
||||
}
|
||||
|
||||
/* Read a whole file into *buffer (grown as needed), leaving one spare byte. */
|
||||
static bool search_scan_read_file(const char *path, char **buffer, size_t *capacity,
|
||||
size_t *length) {
|
||||
FILE *fp = cbm_fopen(path, "rb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
size_t used = 0;
|
||||
bool ok = true;
|
||||
for (;;) {
|
||||
if (*capacity - used < CBM_SZ_64K + SKIP_ONE) {
|
||||
size_t grown = *capacity ? *capacity * PAIR_LEN : (size_t)CBM_SZ_64K * PAIR_LEN;
|
||||
if (grown <= *capacity || grown > (size_t)INT_MAX) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
char *next = realloc(*buffer, grown);
|
||||
if (!next) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
*buffer = next;
|
||||
*capacity = grown;
|
||||
}
|
||||
size_t want = *capacity - used - SKIP_ONE;
|
||||
size_t got = fread(*buffer + used, 1, want, fp);
|
||||
used += got;
|
||||
if (got < want) {
|
||||
ok = !ferror(fp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
(void)fclose(fp);
|
||||
*length = used;
|
||||
return ok;
|
||||
}
|
||||
|
||||
static mcp_scan_cause_t search_scan_in_process(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, FILE *out,
|
||||
size_t output_limit, uint64_t deadline_ms) {
|
||||
#ifdef _WIN32
|
||||
const char record_separator = '\n';
|
||||
const char field_separator = '\t';
|
||||
#else
|
||||
const char record_separator = '\0';
|
||||
const char field_separator = ':';
|
||||
#endif
|
||||
search_scan_patterns_t patterns;
|
||||
if (!search_scan_patterns_parse(&patterns, pattern, use_regex)) {
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
char *list = NULL;
|
||||
size_t list_capacity = 0;
|
||||
size_t list_length = 0;
|
||||
if (!search_scan_read_file(filelist_path, &list, &list_capacity, &list_length)) {
|
||||
free(list);
|
||||
search_scan_patterns_free(&patterns);
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
search_scan_sink_t sink = {
|
||||
.out = out, .output_limit = output_limit, .separator = field_separator};
|
||||
char *buffer = NULL;
|
||||
size_t capacity = 0;
|
||||
mcp_scan_cause_t cause = MCP_SCAN_SUCCESS;
|
||||
size_t pos = 0;
|
||||
while (cause == MCP_SCAN_SUCCESS && pos < list_length) {
|
||||
const char *end = memchr(list + pos, record_separator, list_length - pos);
|
||||
size_t record_end = end ? (size_t)(end - list) : list_length;
|
||||
list[record_end] = '\0';
|
||||
const char *path = list + pos;
|
||||
size_t path_length = record_end - pos;
|
||||
pos = record_end + SKIP_ONE;
|
||||
if (path_length == 0) {
|
||||
continue;
|
||||
}
|
||||
if (mcp_request_cancelled(srv)) {
|
||||
cause = MCP_SCAN_CANCELLED;
|
||||
break;
|
||||
}
|
||||
if (cbm_now_ms() >= deadline_ms) {
|
||||
cause = MCP_SCAN_DEADLINE;
|
||||
break;
|
||||
}
|
||||
size_t length = 0;
|
||||
if (!search_scan_read_file(path, &buffer, &capacity, &length)) {
|
||||
cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
break;
|
||||
}
|
||||
cause = search_scan_buffer(srv, &sink, &patterns, use_regex, path, path_length, buffer,
|
||||
length, deadline_ms);
|
||||
}
|
||||
free(buffer);
|
||||
free(list);
|
||||
search_scan_patterns_free(&patterns);
|
||||
return cause;
|
||||
}
|
||||
|
||||
int cbm_mcp_search_scan_for_test(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, const char *out_path,
|
||||
size_t output_limit, uint64_t deadline_ms) {
|
||||
FILE *out = cbm_fopen(out_path, "wb");
|
||||
if (!out) {
|
||||
return MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
mcp_scan_cause_t cause = search_scan_in_process(srv, filelist_path, pattern, use_regex, out,
|
||||
output_limit, deadline_ms);
|
||||
if (fclose(out) != 0 && cause == MCP_SCAN_SUCCESS) {
|
||||
cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
return (int)cause;
|
||||
}
|
||||
|
||||
/* Open the scratch directory, write `pattern` to the grep -f file, and leave the
|
||||
* file list open for write_scoped_filelist. Returns true on success; on failure
|
||||
* everything already created is removed before returning. */
|
||||
@@ -16258,10 +16810,39 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
: MCP_SEARCH_OUTPUT_MAX;
|
||||
const char *scan_command =
|
||||
srv->search_scan_command_override ? srv->search_scan_command_override : cmd;
|
||||
mcp_scan_cause_t scan_cause = mcp_run_shell_command_cancellable_bounded(
|
||||
srv, scan_command, output_path, scan_output_limit, scan_deadline_ms, true,
|
||||
scan_deadline_latched, /*exit_one_is_no_match=*/false, /*git_child=*/false,
|
||||
&scan_result);
|
||||
/* An indexed project is scanned in process (see search_scan_in_process).
|
||||
* The recursive fallback for an unindexed root, a Windows file_pattern
|
||||
* whose -like post-filter cannot be applied ahead of the scan, and the
|
||||
* test command override keep the subprocess runner. */
|
||||
bool scan_in_process = scoped && !srv->search_scan_command_override;
|
||||
#ifdef _WIN32
|
||||
scan_in_process =
|
||||
scan_in_process &&
|
||||
(!file_pattern || cbm_search_code_file_pattern_can_prefilter(file_pattern));
|
||||
#endif
|
||||
mcp_scan_cause_t scan_cause;
|
||||
if (scan_in_process) {
|
||||
FILE *scan_out = NULL;
|
||||
if (scan_cancellation_latched || mcp_request_cancelled(srv)) {
|
||||
scan_cause = MCP_SCAN_CANCELLED;
|
||||
} else if (scan_deadline_latched || cbm_now_ms() >= scan_deadline_ms) {
|
||||
scan_cause = MCP_SCAN_DEADLINE;
|
||||
} else if (!(scan_out = search_scratch_file(scratch.dir, "scan", output_path,
|
||||
sizeof(output_path)))) {
|
||||
scan_cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
} else {
|
||||
scan_cause = search_scan_in_process(srv, filelist, pattern, use_regex, scan_out,
|
||||
scan_output_limit, scan_deadline_ms);
|
||||
}
|
||||
if (scan_out && fclose(scan_out) != 0 && scan_cause == MCP_SCAN_SUCCESS) {
|
||||
scan_cause = MCP_SCAN_COMMAND_FAILURE;
|
||||
}
|
||||
} else {
|
||||
scan_cause = mcp_run_shell_command_cancellable_bounded(
|
||||
srv, scan_command, output_path, scan_output_limit, scan_deadline_ms, true,
|
||||
scan_deadline_latched, /*exit_one_is_no_match=*/false, /*git_child=*/false,
|
||||
&scan_result);
|
||||
}
|
||||
/* Both POSIX commands wrap grep and map its no-match status to 0, so any
|
||||
* non-zero exit (a failed find/sort, an unreadable operand, a broken
|
||||
* grep) is an incomplete scan and fails closed. */
|
||||
@@ -16409,11 +16990,15 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
directory_output = directory_limit;
|
||||
}
|
||||
bool warn_literal_pipe = pat_has_pipe && !use_regex;
|
||||
/* Every render below re-checks each result's file against the root; one
|
||||
* containment for the request keeps that to one resolution per file. */
|
||||
root_containment_t request_containment;
|
||||
root_containment_init(&request_containment, root_path);
|
||||
char *payload = render_search_payload(
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start, result_limit,
|
||||
output_count, raw_start, raw_output, directory_start, directory_output, raw_limit,
|
||||
directory_limit, match_limit, mode, context_lines, source_max_lines, root_path,
|
||||
scan_saturated, warn_literal_pipe, &metrics, false, sc_legacy_json);
|
||||
scan_saturated, warn_literal_pipe, &metrics, false, sc_legacy_json, &request_containment);
|
||||
|
||||
if (payload && strlen(payload) > byte_budget) {
|
||||
free(payload);
|
||||
@@ -16421,7 +17006,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
|
||||
/* Preserve ranked graph answers. Unclassified raw grep rows and directory
|
||||
@@ -16433,7 +17019,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
while (payload && strlen(payload) > byte_budget && directory_output > 0) {
|
||||
directory_output--;
|
||||
@@ -16442,7 +17029,8 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
sr, sr_count, raw, raw_count, raw_content_truncated, gm_count, result_start,
|
||||
result_limit, output_count, raw_start, raw_output, directory_start, directory_output,
|
||||
raw_limit, directory_limit, match_limit, mode, context_lines, source_max_lines,
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json);
|
||||
root_path, scan_saturated, warn_literal_pipe, &metrics, true, sc_legacy_json,
|
||||
&request_containment);
|
||||
}
|
||||
|
||||
/* Full source is the next detail tier. Find the largest whole-line window
|
||||
@@ -16459,7 +17047,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, output_count, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
middle, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
if (candidate && strlen(candidate) <= byte_budget) {
|
||||
free(best_payload);
|
||||
best_payload = candidate;
|
||||
@@ -16481,7 +17069,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, output_count, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
source_max_lines, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16497,7 +17085,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
result_limit, candidate_rows, raw_start, raw_output, directory_start,
|
||||
directory_output, raw_limit, directory_limit, match_limit, mode, context_lines,
|
||||
source_max_lines, root_path, scan_saturated, warn_literal_pipe, &metrics, true,
|
||||
sc_legacy_json);
|
||||
sc_legacy_json, &request_containment);
|
||||
if (candidate && strlen(candidate) <= byte_budget) {
|
||||
best_payload = candidate;
|
||||
break;
|
||||
@@ -16514,6 +17102,7 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) {
|
||||
raw_start, raw_limit, directory_start, directory_limit,
|
||||
dir_total, scan_saturated, sc_legacy_json, max_output_tokens);
|
||||
}
|
||||
root_containment_free(&request_containment);
|
||||
char *result = cbm_mcp_text_result(payload ? payload : "out of memory", payload == NULL);
|
||||
free(payload);
|
||||
free_search_results(sr, sr_count);
|
||||
@@ -19963,6 +20552,24 @@ static char *dispatch_tool(cbm_mcp_server_t *srv, const char *tool_name, const c
|
||||
return cbm_mcp_text_result(msg, true);
|
||||
}
|
||||
|
||||
/* Read tools whose work is store reads from start to end run them in one store
|
||||
* read batch (cbm_store_read_batch_begin). Left out: tools that write, open
|
||||
* their own transaction or read scope (detect_changes), or hold the store
|
||||
* across a long scan or a git subprocess (search_code, index_status). */
|
||||
static bool mcp_tool_reads_in_batch(const char *tool_name) {
|
||||
static const char *const batched[] = {
|
||||
"query_graph", "trace_path", "trace_call_path", "search_graph",
|
||||
"get_architecture", "get_graph_schema", "get_code_snippet", "get_file_outline",
|
||||
"compare_graphs", "check_index_coverage",
|
||||
};
|
||||
for (size_t i = 0; tool_name && i < sizeof(batched) / sizeof(batched[0]); i++) {
|
||||
if (strcmp(tool_name, batched[i]) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* File-backed query stores are request-scoped. Keeping one open between MCP
|
||||
* calls pins an old database generation after another process atomically
|
||||
* replaces the project DB. On Windows it can also prevent that replacement
|
||||
@@ -19998,7 +20605,15 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
|
||||
return cbm_mcp_text_result("request cancellation scope unavailable", true);
|
||||
}
|
||||
cbm_mem_phase_mark("request.dispatch_tool");
|
||||
if (srv) {
|
||||
srv->read_batch_tool = mcp_tool_reads_in_batch(tool_name);
|
||||
}
|
||||
char *result = dispatch_tool(srv, tool_name, args_json);
|
||||
if (srv) {
|
||||
/* An embedded store outlives the request: end its batch here. */
|
||||
srv->read_batch_tool = false;
|
||||
cbm_store_read_batch_end(srv->store);
|
||||
}
|
||||
cbm_mem_phase_mark("request.scope_end");
|
||||
if (srv) {
|
||||
cbm_mcp_server_request_scope_end(srv);
|
||||
|
||||
@@ -39,6 +39,13 @@ void cbm_mcp_server_set_search_scan_timeout_for_test(cbm_mcp_server_t *srv, uint
|
||||
* scratch count cannot see other processes' entries in the shared temp dir.
|
||||
* The server borrows dir until it is reset or the server is freed. */
|
||||
void cbm_mcp_server_set_search_scratch_dir_for_test(cbm_mcp_server_t *srv, const char *dir);
|
||||
/* search_code's in-process scan of `filelist_path` (records separated as the
|
||||
* scoped list is written: NUL on POSIX, newline on Windows) into `out_path`.
|
||||
* Returns the scan cause in mcp_scan_cause_t order: 0 success, 1 command
|
||||
* failure, 3 output limit, 4 deadline, 5 cancelled. */
|
||||
int cbm_mcp_search_scan_for_test(cbm_mcp_server_t *srv, const char *filelist_path,
|
||||
const char *pattern, bool use_regex, const char *out_path,
|
||||
size_t output_limit, uint64_t deadline_ms);
|
||||
|
||||
/* Release only the constructor-created pristine in-memory store. Public
|
||||
* cbm_mcp_server_new(NULL) semantics remain unchanged; daemon sessions use
|
||||
|
||||
@@ -121,6 +121,8 @@ struct cbm_store {
|
||||
const void *progress_owner; /* exclusive native progress guard token */
|
||||
const char *db_path; /* heap-allocated, or NULL for :memory: */
|
||||
char errbuf[CBM_SZ_512];
|
||||
bool read_batch; /* a read batch's transaction is open */
|
||||
uint64_t read_batch_started_ms; /* when it was last (re)begun */
|
||||
|
||||
/* Prepared statements (lazily initialized, cached for lifetime) */
|
||||
sqlite3_stmt *stmt_upsert_node;
|
||||
@@ -207,10 +209,60 @@ static char *heap_strdup(const char *s) {
|
||||
}
|
||||
|
||||
/* Prepare a statement (cached). If already prepared, reset+clear. */
|
||||
/* ── Read batch ────────────────────────────────────────────────────
|
||||
* A read tool issues thousands of small statements. In autocommit mode each
|
||||
* one takes and drops the database file lock and stats the journal and WAL
|
||||
* files — most of a whole-graph count(*)'s time. A read batch runs them in one
|
||||
* deferred read transaction, renewed every ST_READ_BATCH_MS at the next
|
||||
* statement: a writer never waits behind the batch much longer than that, and
|
||||
* every statement still reads data committed at most that long ago (while the
|
||||
* batch holds a rollback-journal database's shared lock, no commit happens).
|
||||
* Committing a read transaction is allowed with read statements still
|
||||
* stepping; their lock is then held until they finish, as before. */
|
||||
enum { ST_READ_BATCH_MS = 100 };
|
||||
|
||||
bool cbm_store_read_batch_begin(cbm_store_t *s) {
|
||||
if (!s || !s->db || s->read_batch || !sqlite3_get_autocommit(s->db)) {
|
||||
return false;
|
||||
}
|
||||
if (sqlite3_exec(s->db, "BEGIN DEFERRED;", NULL, NULL, NULL) != SQLITE_OK) {
|
||||
return false;
|
||||
}
|
||||
s->read_batch = true;
|
||||
s->read_batch_started_ms = cbm_now_ms();
|
||||
return true;
|
||||
}
|
||||
|
||||
void cbm_store_read_batch_end(cbm_store_t *s) {
|
||||
if (!s || !s->read_batch) {
|
||||
return;
|
||||
}
|
||||
s->read_batch = false;
|
||||
if (s->db && !sqlite3_get_autocommit(s->db)) {
|
||||
(void)sqlite3_exec(s->db, "COMMIT;", NULL, NULL, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
void cbm_store_read_batch_expire_for_test(cbm_store_t *s) {
|
||||
if (s && s->read_batch) {
|
||||
uint64_t now = cbm_now_ms();
|
||||
s->read_batch_started_ms = now > ST_READ_BATCH_MS ? now - ST_READ_BATCH_MS : 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void read_batch_renew(cbm_store_t *s) {
|
||||
if (!s->read_batch || cbm_now_ms() - s->read_batch_started_ms < ST_READ_BATCH_MS) {
|
||||
return;
|
||||
}
|
||||
cbm_store_read_batch_end(s);
|
||||
(void)cbm_store_read_batch_begin(s);
|
||||
}
|
||||
|
||||
static sqlite3_stmt *prepare_cached(cbm_store_t *s, sqlite3_stmt **slot, const char *sql) {
|
||||
if (!s || !s->db) {
|
||||
return NULL;
|
||||
}
|
||||
read_batch_renew(s);
|
||||
if (*slot) {
|
||||
sqlite3_reset(*slot);
|
||||
sqlite3_clear_bindings(*slot);
|
||||
@@ -1363,6 +1415,7 @@ void cbm_store_close(cbm_store_t *s) {
|
||||
if (!s) {
|
||||
return;
|
||||
}
|
||||
cbm_store_read_batch_end(s);
|
||||
|
||||
/* Checkpoint WAL before close to prevent orphan WAL accumulation.
|
||||
* Best-effort — silently skips if concurrent reader holds a lock. */
|
||||
|
||||
@@ -478,6 +478,16 @@ int cbm_store_commit(cbm_store_t *s);
|
||||
/* Rollback the current transaction. */
|
||||
int cbm_store_rollback(cbm_store_t *s);
|
||||
|
||||
/* Read batch: run the reads that follow in one deferred read transaction,
|
||||
* renewed every 100 ms so a writer is never held back longer, instead of one
|
||||
* lock round trip per statement. Begin returns false (and does nothing) when
|
||||
* a transaction is already open. For read-only work only; end it before any
|
||||
* write, transaction or read scope on the same store. Close ends it. */
|
||||
bool cbm_store_read_batch_begin(cbm_store_t *s);
|
||||
void cbm_store_read_batch_end(cbm_store_t *s);
|
||||
/* Test seam: make an open read batch due for renewal at its next statement. */
|
||||
void cbm_store_read_batch_expire_for_test(cbm_store_t *s);
|
||||
|
||||
/* Stable request read scope. Borrows an exclusively owned, single-thread
|
||||
* store and callback context. Refuses a transaction, busy statement or owned
|
||||
* progress handler; idle cached statements are allowed. Saves query_only and
|
||||
|
||||
@@ -5580,6 +5580,257 @@ TEST(cypher_fifty_union_branches) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ── ORDER BY, DISTINCT and grouping against a reference ───────────
|
||||
* These were bubble sorts and linear duplicate/group scans — seconds to
|
||||
* minutes on a 30k-row result. The results must stay exactly the old ones: a
|
||||
* stable sort of the unsorted rows, first-occurrence DISTINCT, first-seen
|
||||
* group order. Each case derives that reference from the engine's own
|
||||
* unsorted rows, on a graph built for ties, duplicates and large groups. */
|
||||
|
||||
enum { REF_NODES = 400, REF_NAMES = 97 };
|
||||
|
||||
static cbm_store_t *setup_reference_store(void) {
|
||||
cbm_store_t *s = cbm_store_open_memory();
|
||||
cbm_store_upsert_project(s, "test", "/tmp/test");
|
||||
int64_t ids[REF_NODES];
|
||||
for (int i = 0; i < REF_NODES; i++) {
|
||||
char name[32];
|
||||
char qn[32];
|
||||
char file[32];
|
||||
snprintf(name, sizeof(name), "fn_%02d", (i * 7) % REF_NAMES); /* repeated names */
|
||||
snprintf(qn, sizeof(qn), "test.n%d", i);
|
||||
snprintf(file, sizeof(file), "f%02d.c", i % 13);
|
||||
cbm_node_t n = {.project = "test",
|
||||
.label = "Function",
|
||||
.name = name,
|
||||
.qualified_name = qn,
|
||||
.file_path = file,
|
||||
.start_line = (i * 31) % 50, /* tied lines */
|
||||
.end_line = 60};
|
||||
ids[i] = cbm_store_upsert_node(s, &n);
|
||||
}
|
||||
/* Every node calls both hubs (n0, n1: ~400 callers, 97 distinct names,
|
||||
* past the hashed COUNT(DISTINCT) threshold) and one more node. */
|
||||
for (int i = 0; i < REF_NODES; i++) {
|
||||
int targets[3] = {0, 1, (i * i + 3) % REF_NODES};
|
||||
for (int t = 0; t < 3; t++) {
|
||||
if (targets[t] != i) {
|
||||
cbm_edge_t e = {.project = "test",
|
||||
.source_id = ids[i],
|
||||
.target_id = ids[targets[t]],
|
||||
.type = "CALLS"};
|
||||
cbm_store_insert_edge(s, &e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
typedef int (*ref_cmp_fn)(const cbm_cypher_result_t *r, int a, int b);
|
||||
|
||||
/* The reference order: an insertion sort that moves a row before an earlier
|
||||
* one only when the earlier one compares greater — stable, like the old
|
||||
* bubble sorts. */
|
||||
static void ref_stable_order(const cbm_cypher_result_t *r, int *order, ref_cmp_fn cmp) {
|
||||
for (int i = 0; i < r->row_count; i++) {
|
||||
order[i] = i;
|
||||
}
|
||||
for (int i = 1; i < r->row_count; i++) {
|
||||
for (int j = i; j > 0 && cmp(r, order[j - 1], order[j]) > 0; j--) {
|
||||
int swap = order[j];
|
||||
order[j] = order[j - 1];
|
||||
order[j - 1] = swap;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Columns 0 (name) and 1 (line): line descending, then name ascending. */
|
||||
static int ref_line_desc_name(const cbm_cypher_result_t *r, int a, int b) {
|
||||
long la = strtol(r->rows[a][1], NULL, 10);
|
||||
long lb = strtol(r->rows[b][1], NULL, 10);
|
||||
int cmp = (lb > la) - (lb < la);
|
||||
return cmp != 0 ? cmp : strcmp(r->rows[a][0], r->rows[b][0]);
|
||||
}
|
||||
|
||||
static int ref_all_equal(const cbm_cypher_result_t *r, int a, int b) {
|
||||
(void)r;
|
||||
(void)a;
|
||||
(void)b;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* `sorted` must hold the rows of `unsorted` in the reference order. */
|
||||
static bool ref_rows_in_order(const cbm_cypher_result_t *sorted,
|
||||
const cbm_cypher_result_t *unsorted, ref_cmp_fn cmp) {
|
||||
if (sorted->row_count != unsorted->row_count || sorted->col_count != unsorted->col_count ||
|
||||
unsorted->row_count < REF_NODES) {
|
||||
return false;
|
||||
}
|
||||
int order[REF_NODES * 4];
|
||||
ref_stable_order(unsorted, order, cmp);
|
||||
for (int i = 0; i < sorted->row_count; i++) {
|
||||
for (int c = 0; c < sorted->col_count; c++) {
|
||||
if (strcmp(sorted->rows[i][c], unsorted->rows[order[i]][c]) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ref_query(cbm_store_t *s, const char *query, cbm_cypher_result_t *out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
return cbm_cypher_execute(s, query, "test", 0, out) == 0 && !out->truncated;
|
||||
}
|
||||
|
||||
TEST(cypher_order_by_is_the_stable_reference_order) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t plain;
|
||||
cbm_cypher_result_t sorted;
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.start_line", &plain));
|
||||
|
||||
/* RETURN: numeric key descending, ties broken by the next key. */
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (f:Function) RETURN f.name, f.start_line ORDER BY f.start_line DESC, f.name",
|
||||
&sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
|
||||
/* WITH: the same keys through a projection. */
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (f:Function) WITH f.name AS n, f.start_line AS l "
|
||||
"ORDER BY l DESC, n RETURN n, l",
|
||||
&sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
cbm_cypher_result_free(&plain);
|
||||
|
||||
/* All keys tied: both sorts keep the unsorted order. */
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.label", &plain));
|
||||
ASSERT_TRUE(
|
||||
ref_query(s, "MATCH (f:Function) RETURN f.name, f.label ORDER BY f.label", &sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (f:Function) WITH f.name AS n, f.label AS l ORDER BY l RETURN n, l", &sorted));
|
||||
ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal));
|
||||
cbm_cypher_result_free(&sorted);
|
||||
cbm_cypher_result_free(&plain);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* `got` must be the first occurrence of every value in column 0 of `all`. */
|
||||
static bool ref_first_occurrences(const cbm_cypher_result_t *got, const cbm_cypher_result_t *all) {
|
||||
int kept = 0;
|
||||
for (int i = 0; i < all->row_count; i++) {
|
||||
bool seen = false;
|
||||
for (int j = 0; j < i && !seen; j++) {
|
||||
seen = strcmp(all->rows[i][0], all->rows[j][0]) == 0;
|
||||
}
|
||||
if (seen) {
|
||||
continue;
|
||||
}
|
||||
if (kept >= got->row_count || strcmp(got->rows[kept][0], all->rows[i][0]) != 0) {
|
||||
return false;
|
||||
}
|
||||
kept++;
|
||||
}
|
||||
return kept == got->row_count;
|
||||
}
|
||||
|
||||
TEST(cypher_distinct_keeps_first_occurrences) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t all;
|
||||
cbm_cypher_result_t got;
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name", &all));
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN DISTINCT f.name", &got));
|
||||
ASSERT_EQ(got.row_count, REF_NAMES);
|
||||
ASSERT_TRUE(ref_first_occurrences(&got, &all));
|
||||
cbm_cypher_result_free(&got);
|
||||
ASSERT_TRUE(ref_query(s, "MATCH (f:Function) WITH DISTINCT f.name AS n RETURN n", &got));
|
||||
ASSERT_EQ(got.row_count, REF_NAMES);
|
||||
ASSERT_TRUE(ref_first_occurrences(&got, &all));
|
||||
cbm_cypher_result_free(&got);
|
||||
cbm_cypher_result_free(&all);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* `got` (column 0 = callee, column 1 = count) must list the callees of
|
||||
* `pairs` (column 0 = callee, column 1 = caller name) in first-seen order,
|
||||
* each with its caller count — of distinct names when `distinct`. */
|
||||
static bool ref_groups_match(const cbm_cypher_result_t *got, const cbm_cypher_result_t *pairs,
|
||||
bool distinct) {
|
||||
enum { REF_PAIRS_MAX = REF_NODES * 4 };
|
||||
if (pairs->row_count > REF_PAIRS_MAX) {
|
||||
return false;
|
||||
}
|
||||
bool first_callee[REF_PAIRS_MAX];
|
||||
bool first_pair[REF_PAIRS_MAX];
|
||||
for (int k = 0; k < pairs->row_count; k++) {
|
||||
first_callee[k] = true;
|
||||
first_pair[k] = true;
|
||||
for (int m = 0; m < k; m++) {
|
||||
if (strcmp(pairs->rows[m][0], pairs->rows[k][0]) == 0) {
|
||||
first_callee[k] = false;
|
||||
first_pair[k] = first_pair[k] && strcmp(pairs->rows[m][1], pairs->rows[k][1]) != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
int groups = 0;
|
||||
for (int i = 0; i < pairs->row_count; i++) {
|
||||
if (!first_callee[i]) {
|
||||
continue;
|
||||
}
|
||||
int count = 0;
|
||||
for (int k = 0; k < pairs->row_count; k++) {
|
||||
if (strcmp(pairs->rows[k][0], pairs->rows[i][0]) == 0) {
|
||||
count += !distinct || first_pair[k] ? 1 : 0;
|
||||
}
|
||||
}
|
||||
char want[16];
|
||||
snprintf(want, sizeof(want), "%d", count);
|
||||
if (groups >= got->row_count || strcmp(got->rows[groups][0], pairs->rows[i][0]) != 0 ||
|
||||
strcmp(got->rows[groups][1], want) != 0) {
|
||||
return false;
|
||||
}
|
||||
groups++;
|
||||
}
|
||||
return groups == got->row_count;
|
||||
}
|
||||
|
||||
TEST(cypher_grouping_keeps_first_seen_groups) {
|
||||
cbm_store_t *s = setup_reference_store();
|
||||
cbm_cypher_result_t pairs;
|
||||
cbm_cypher_result_t got;
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, a.name", &pairs));
|
||||
ASSERT_GT(pairs.row_count, REF_NODES * 2);
|
||||
|
||||
ASSERT_TRUE(ref_query(
|
||||
s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, count(a) AS c", &got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, false));
|
||||
cbm_cypher_result_free(&got);
|
||||
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (a)-[:CALLS]->(b:Function) "
|
||||
"RETURN b.qualified_name, count(DISTINCT a.name) AS c",
|
||||
&got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, true));
|
||||
cbm_cypher_result_free(&got);
|
||||
|
||||
ASSERT_TRUE(ref_query(s,
|
||||
"MATCH (a)-[:CALLS]->(b:Function) WITH b.qualified_name AS q, "
|
||||
"count(DISTINCT a.name) AS c RETURN q, c",
|
||||
&got));
|
||||
ASSERT_TRUE(ref_groups_match(&got, &pairs, true));
|
||||
cbm_cypher_result_free(&got);
|
||||
cbm_cypher_result_free(&pairs);
|
||||
cbm_store_close(s);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
SUITE(cypher) {
|
||||
@@ -5823,4 +6074,7 @@ SUITE(cypher) {
|
||||
RUN_TEST(cypher_chain_with_unbound_alias_keeps_seed);
|
||||
RUN_TEST(cypher_many_union_branches);
|
||||
RUN_TEST(cypher_fifty_union_branches);
|
||||
RUN_TEST(cypher_order_by_is_the_stable_reference_order);
|
||||
RUN_TEST(cypher_distinct_keeps_first_occurrences);
|
||||
RUN_TEST(cypher_grouping_keeps_first_seen_groups);
|
||||
}
|
||||
|
||||
@@ -440,9 +440,66 @@ TEST(githistory_ignores_inherited_repo_env) {
|
||||
#endif /* _WIN32 */
|
||||
}
|
||||
|
||||
/* ── Branch-only resolve (list_projects) ───────────────────────────
|
||||
* The listing prints only the branch; cbm_git_branch_resolve gets it from the
|
||||
* two git calls cbm_git_context_resolve uses for it, so both must agree. */
|
||||
|
||||
#ifndef _WIN32
|
||||
/* The branch resolver against the full context at `path`; NULL `want` = not git. */
|
||||
static bool branch_resolve_agrees(const char *path, const char *want) {
|
||||
cbm_git_context_t ctx = {0};
|
||||
(void)cbm_git_context_resolve(path, &ctx);
|
||||
bool is_git = false;
|
||||
char *branch = NULL;
|
||||
(void)cbm_git_branch_resolve(path, &is_git, &branch);
|
||||
bool agrees = is_git == ctx.is_git && is_git == (want != NULL) &&
|
||||
(!want || (branch && ctx.branch && strcmp(branch, want) == 0 &&
|
||||
strcmp(ctx.branch, want) == 0));
|
||||
free(branch);
|
||||
cbm_git_context_free(&ctx);
|
||||
return agrees;
|
||||
}
|
||||
#endif /* _WIN32 */
|
||||
|
||||
TEST(git_branch_resolve_matches_full_context) {
|
||||
#ifdef _WIN32
|
||||
SKIP_PLATFORM("git-based branch test not supported on Windows CI");
|
||||
#else
|
||||
char plain[512];
|
||||
char *tmp = th_mktempdir("cbm_gitbranch_plain");
|
||||
if (!tmp) FAIL("th_mktempdir returned NULL");
|
||||
snprintf(plain, sizeof(plain), "%s", tmp);
|
||||
tmp = th_mktempdir("cbm_gitbranch");
|
||||
if (!tmp) FAIL("th_mktempdir returned NULL");
|
||||
if (make_git_repo(tmp) != 0) {
|
||||
th_rmtree(tmp);
|
||||
th_rmtree(plain);
|
||||
SKIP_PLATFORM("git not available to init a repo");
|
||||
}
|
||||
char sub[600];
|
||||
snprintf(sub, sizeof(sub), "%s/sub", tmp);
|
||||
ASSERT_EQ(th_mkdir_p(sub), 0);
|
||||
char missing[600];
|
||||
snprintf(missing, sizeof(missing), "%s/missing", plain);
|
||||
|
||||
ASSERT_TRUE(branch_resolve_agrees(plain, NULL));
|
||||
ASSERT_TRUE(branch_resolve_agrees(missing, NULL));
|
||||
ASSERT_EQ(git_run(tmp, "checkout -q -b feat/x"), 0);
|
||||
ASSERT_TRUE(branch_resolve_agrees(tmp, "feat/x"));
|
||||
ASSERT_TRUE(branch_resolve_agrees(sub, "feat/x"));
|
||||
ASSERT_EQ(git_run(tmp, "checkout -q --detach"), 0);
|
||||
ASSERT_TRUE(branch_resolve_agrees(tmp, "DETACHED"));
|
||||
|
||||
th_rmtree(tmp);
|
||||
th_rmtree(plain);
|
||||
PASS();
|
||||
#endif /* _WIN32 */
|
||||
}
|
||||
|
||||
/* ── Suite ──────────────────────────────────────────────────────── */
|
||||
|
||||
SUITE(git_context) {
|
||||
RUN_TEST(git_branch_resolve_matches_full_context);
|
||||
RUN_TEST(root_exists_non_ascii_directory);
|
||||
RUN_TEST(canonical_root_repo_root);
|
||||
RUN_TEST(canonical_root_subdir);
|
||||
|
||||
@@ -12205,6 +12205,16 @@ TEST(search_code_windows_scope_prefilter_removes_pipeline_filter) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/* An indexed project's scan runs in process; the supervised runner now serves
|
||||
* the unindexed recursive fallback and a Windows file_pattern it cannot
|
||||
* prefilter. The supervision tests below pin that runner through the command
|
||||
* override, with a command that does nothing: their probes act at spawn. */
|
||||
#ifdef _WIN32
|
||||
static const char MCP_SUPERVISED_SCAN_NOOP[] = "cd .";
|
||||
#else
|
||||
static const char MCP_SUPERVISED_SCAN_NOOP[] = ":";
|
||||
#endif
|
||||
|
||||
TEST(search_code_cancel_cleans_supervised_scan) {
|
||||
mcp_search_cache_t cache;
|
||||
ASSERT_TRUE(mcp_search_cache_open(&cache, "cbm-search-cancel"));
|
||||
@@ -12213,6 +12223,7 @@ TEST(search_code_cancel_cleans_supervised_scan) {
|
||||
cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path),
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {
|
||||
.server = srv,
|
||||
.cancel_on_call = true,
|
||||
@@ -12490,6 +12501,7 @@ TEST(search_code_scan_deadline_precedes_output_limit) {
|
||||
cbm_mcp_server_t *srv = setup_prefilter_server(tmp, sizeof(tmp), src_path, sizeof(src_path),
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {.fill_output_directory = logs, .delay_ms = 1100};
|
||||
cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe);
|
||||
cbm_mcp_server_set_search_scan_timeout_for_test(srv, 1000, true);
|
||||
@@ -12521,6 +12533,7 @@ TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up) {
|
||||
vendor_path, sizeof(vendor_path));
|
||||
ASSERT_NOT_NULL(srv);
|
||||
cbm_mcp_server_set_search_scratch_dir_for_test(srv, cache.path);
|
||||
cbm_mcp_server_set_search_scan_command_for_test(srv, MCP_SUPERVISED_SCAN_NOOP);
|
||||
mcp_search_command_probe_t probe = {.reject = true, .scratch_directory = cache.path};
|
||||
cbm_mcp_server_set_command_test_hook(srv, mcp_search_command_hook_probe, &probe);
|
||||
|
||||
@@ -24546,6 +24559,272 @@ TEST(search_code_drops_walker_hits_that_resolve_outside_root) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ── In-process search_code scan ───────────────────────────────────
|
||||
* The scoped scan used to shell out to grep (POSIX) or Select-String
|
||||
* (Windows) and now runs in process. These cases pin its records to the
|
||||
* `grep -Hn` shape the subprocess wrote: path:line:content on POSIX, the
|
||||
* same fields TAB-separated on Windows. */
|
||||
|
||||
#ifdef _WIN32
|
||||
#define SCAN_FS "\t"
|
||||
#define SCAN_RS '\n'
|
||||
#else
|
||||
#define SCAN_FS ":"
|
||||
#define SCAN_RS '\0'
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
char dir[256];
|
||||
char list[512];
|
||||
char out[512];
|
||||
char files[4][512];
|
||||
int count;
|
||||
} scan_fixture_t;
|
||||
|
||||
static bool scan_fixture_open(scan_fixture_t *fx) {
|
||||
memset(fx, 0, sizeof(*fx));
|
||||
const char *tmp = th_mktempdir("cbm_scan");
|
||||
if (!tmp) {
|
||||
return false;
|
||||
}
|
||||
snprintf(fx->dir, sizeof(fx->dir), "%s", tmp);
|
||||
snprintf(fx->list, sizeof(fx->list), "%s/list", tmp);
|
||||
snprintf(fx->out, sizeof(fx->out), "%s/out", tmp);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Add a file holding exactly `length` bytes (NUL bytes allowed). */
|
||||
static bool scan_fixture_add(scan_fixture_t *fx, const char *name, const char *data,
|
||||
size_t length) {
|
||||
if (fx->count >= 4) {
|
||||
return false;
|
||||
}
|
||||
/* Format from a copy: GCC's -Werror=restrict rejects a source and a
|
||||
* destination inside the same struct. */
|
||||
char dir[sizeof(fx->dir)];
|
||||
memcpy(dir, fx->dir, sizeof(dir));
|
||||
char *path = fx->files[fx->count];
|
||||
snprintf(path, sizeof(fx->files[0]), "%s/%s", dir, name);
|
||||
FILE *fp = cbm_fopen(path, "wb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
bool ok = fwrite(data, 1, length, fp) == length;
|
||||
ok = fclose(fp) == 0 && ok;
|
||||
fx->count += ok ? 1 : 0;
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Write the scoped file list as search_code does (NUL records on POSIX,
|
||||
* lines on Windows); `extra` names one more path that need not exist. */
|
||||
static bool scan_fixture_list(const scan_fixture_t *fx, const char *extra) {
|
||||
FILE *fp = cbm_fopen(fx->list, "wb");
|
||||
if (!fp) {
|
||||
return false;
|
||||
}
|
||||
bool ok = true;
|
||||
for (int i = 0; i < fx->count; i++) {
|
||||
ok = ok && fputs(fx->files[i], fp) != EOF && fputc(SCAN_RS, fp) != EOF;
|
||||
}
|
||||
if (extra) {
|
||||
ok = ok && fputs(extra, fp) != EOF && fputc(SCAN_RS, fp) != EOF;
|
||||
}
|
||||
return fclose(fp) == 0 && ok;
|
||||
}
|
||||
|
||||
/* Run the scan; returns the records it wrote (caller frees), CRs kept. */
|
||||
static char *scan_fixture_run(const scan_fixture_t *fx, const char *pattern, bool use_regex,
|
||||
size_t output_limit, int *cause) {
|
||||
*cause = cbm_mcp_search_scan_for_test(NULL, fx->list, pattern, use_regex, fx->out,
|
||||
output_limit, UINT64_MAX);
|
||||
FILE *fp = cbm_fopen(fx->out, "rb");
|
||||
if (!fp) {
|
||||
return NULL;
|
||||
}
|
||||
char *text = calloc(CBM_SZ_64K, 1);
|
||||
size_t length = text ? fread(text, 1, CBM_SZ_64K - 1, fp) : 0;
|
||||
(void)fclose(fp);
|
||||
if (text) {
|
||||
text[length] = '\0';
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/* Append one expected record to `buf`. */
|
||||
static void scan_expect(char *buf, size_t cap, const char *path, int line, const char *content) {
|
||||
size_t used = strlen(buf);
|
||||
snprintf(buf + used, cap - used, "%s" SCAN_FS "%d" SCAN_FS "%s\n", path, line, content);
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_writes_grep_records) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
/* The last line has no newline and line 3 is empty. */
|
||||
static const char a[] = "alpha\nbeta gamma\n\nalpha beta\nlast alpha";
|
||||
static const char b[] = "nothing\nalpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "a.txt", a, sizeof(a) - 1));
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "b.txt", b, sizeof(b) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
const char *pa = fx.files[0];
|
||||
const char *pb = fx.files[1];
|
||||
char want[4096];
|
||||
int cause = -1;
|
||||
|
||||
/* Literal: every matching line, file order, then line order. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* A multi-line pattern is one pattern per line, as with grep -f. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 2, "beta gamma");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
got = scan_fixture_run(&fx, "gamma\nbeta", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* An empty pattern matches every line, the empty one included. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 2, "beta gamma");
|
||||
scan_expect(want, sizeof(want), pa, 3, "");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 1, "nothing");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
got = scan_fixture_run(&fx, "", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* Extended regex, anchored per line. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), pa, 1, "alpha");
|
||||
scan_expect(want, sizeof(want), pa, 4, "alpha beta");
|
||||
scan_expect(want, sizeof(want), pa, 5, "last alpha");
|
||||
scan_expect(want, sizeof(want), pb, 2, "alpha");
|
||||
got = scan_fixture_run(&fx, "^(alpha|last)", true, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* No match: success with no records. */
|
||||
got = scan_fixture_run(&fx, "zzqqxx", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, "");
|
||||
free(got);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_binary_and_crlf) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
/* A NUL in the first 32 KiB marks the whole file binary: no records. */
|
||||
static const char bin[] = "alpha\n\0alpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "bin.dat", bin, sizeof(bin) - 1));
|
||||
/* A later NUL ends the text at the last complete line before it. */
|
||||
enum { FILLER_LINES = 4000 }; /* 10 bytes each: past the 32 KiB probe */
|
||||
size_t late_cap = (size_t)FILLER_LINES * 10 + 64;
|
||||
char *late = malloc(late_cap);
|
||||
ASSERT_NOT_NULL(late);
|
||||
size_t used = 0;
|
||||
for (int i = 0; i < FILLER_LINES; i++) {
|
||||
memcpy(late + used, "xxxxxxxxx\n", 10);
|
||||
used += 10;
|
||||
}
|
||||
static const char tail[] = "alpha one\nalpha two\0\nalpha three\n";
|
||||
memcpy(late + used, tail, sizeof(tail) - 1);
|
||||
used += sizeof(tail) - 1;
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "late.txt", late, used));
|
||||
free(late);
|
||||
static const char crlf[] = "alpha\r\nbeta\r\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "crlf.txt", crlf, sizeof(crlf) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
char want[4096];
|
||||
int cause = -1;
|
||||
|
||||
/* The record keeps the line's CR; the result parser trims it. */
|
||||
want[0] = '\0';
|
||||
scan_expect(want, sizeof(want), fx.files[1], FILLER_LINES + 1, "alpha one");
|
||||
scan_expect(want, sizeof(want), fx.files[2], 1, "alpha\r");
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
|
||||
/* `$` before a CR: grep on POSIX sees the CR and does not match;
|
||||
* Select-String's Line on Windows had no CR and did. */
|
||||
want[0] = '\0';
|
||||
#ifdef _WIN32
|
||||
scan_expect(want, sizeof(want), fx.files[2], 2, "beta\r");
|
||||
#endif
|
||||
got = scan_fixture_run(&fx, "^beta$", true, 0, &cause);
|
||||
ASSERT_NOT_NULL(got);
|
||||
ASSERT_EQ(cause, 0);
|
||||
ASSERT_STR_EQ(got, want);
|
||||
free(got);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(search_code_scan_in_process_limits_and_failures) {
|
||||
scan_fixture_t fx;
|
||||
ASSERT_TRUE(scan_fixture_open(&fx));
|
||||
static const char a[] = "alpha\nalpha\nalpha\n";
|
||||
ASSERT_TRUE(scan_fixture_add(&fx, "a.txt", a, sizeof(a) - 1));
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, NULL));
|
||||
int cause = -1;
|
||||
|
||||
/* Past the output limit: the output-limit cause (3), not a failure. */
|
||||
char *got = scan_fixture_run(&fx, "alpha", false, 8, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 3);
|
||||
|
||||
/* An invalid regex is a command failure (1), as grep's exit 2 was. */
|
||||
got = scan_fixture_run(&fx, "(", true, 0, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 1);
|
||||
|
||||
/* A passed deadline stops the scan before the next file (4); a request
|
||||
* cancelled under the server stops it too, and takes precedence (5). */
|
||||
ASSERT_EQ(cbm_mcp_search_scan_for_test(NULL, fx.list, "alpha", false, fx.out, 0, 0), 4);
|
||||
cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL);
|
||||
ASSERT_NOT_NULL(srv);
|
||||
ASSERT_TRUE(cbm_mcp_server_request_scope_begin(srv));
|
||||
ASSERT_TRUE(cbm_mcp_server_cancel_active(srv));
|
||||
cause = cbm_mcp_search_scan_for_test(srv, fx.list, "alpha", false, fx.out, 0, 0);
|
||||
cbm_mcp_server_request_scope_end(srv);
|
||||
cbm_mcp_server_free(srv);
|
||||
ASSERT_EQ(cause, 5);
|
||||
|
||||
/* A listed file that cannot be read is a command failure (1) as well. */
|
||||
char missing[600];
|
||||
snprintf(missing, sizeof(missing), "%s/missing.txt", fx.dir);
|
||||
ASSERT_TRUE(scan_fixture_list(&fx, missing));
|
||||
got = scan_fixture_run(&fx, "alpha", false, 0, &cause);
|
||||
free(got);
|
||||
ASSERT_EQ(cause, 1);
|
||||
th_rmtree(fx.dir);
|
||||
PASS();
|
||||
}
|
||||
|
||||
#undef SCAN_FS
|
||||
#undef SCAN_RS
|
||||
|
||||
SUITE(mcp) {
|
||||
RUN_TEST(index_repository_async_and_status_refused_without_daemon_issue2144);
|
||||
RUN_TEST(index_repository_schema_documents_async_polling_issue2144);
|
||||
@@ -24813,6 +25092,9 @@ SUITE(mcp) {
|
||||
RUN_TEST(search_code_scan_deadline_override_is_per_server);
|
||||
RUN_TEST(search_code_scan_setup_failures_respect_cause_precedence);
|
||||
RUN_TEST(search_code_scan_live_child_deadline_is_bounded_and_fails_closed);
|
||||
RUN_TEST(search_code_scan_in_process_writes_grep_records);
|
||||
RUN_TEST(search_code_scan_in_process_binary_and_crlf);
|
||||
RUN_TEST(search_code_scan_in_process_limits_and_failures);
|
||||
RUN_TEST(search_code_scan_cancellation_precedes_zero_deadline);
|
||||
RUN_TEST(search_code_scan_deadline_precedes_output_limit);
|
||||
RUN_TEST(search_code_scan_hook_rejection_is_contained_and_cleans_up);
|
||||
|
||||
@@ -345,7 +345,75 @@ TEST(corrupt_page_scan_returns_error_not_truncation) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* A read tool's statements share one read batch instead of taking the file
|
||||
* lock once per statement. The batch reads one snapshot until it renews
|
||||
* (every 100 ms, forced here through the seam), then sees later commits. */
|
||||
static char *read_batch_name(cbm_store_t *s) {
|
||||
cbm_node_t found = {0};
|
||||
if (cbm_store_find_node_by_qn(s, "p", "p.a", &found) != CBM_STORE_OK) {
|
||||
return NULL;
|
||||
}
|
||||
char *name = found.name ? strdup(found.name) : NULL;
|
||||
cbm_node_free_fields(&found);
|
||||
return name;
|
||||
}
|
||||
|
||||
static bool read_batch_rename(cbm_store_t *writer, const char *name) {
|
||||
cbm_node_t node = {
|
||||
.project = "p", .label = "Function", .name = name, .qualified_name = "p.a", .file_path = "a.c"};
|
||||
return cbm_store_upsert_node(writer, &node) > 0;
|
||||
}
|
||||
|
||||
TEST(store_read_batch_snapshot_renews) {
|
||||
char *td = th_mktempdir("cbm_read_batch");
|
||||
ASSERT_NOT_NULL(td);
|
||||
char db_path[512];
|
||||
snprintf(db_path, sizeof(db_path), "%s/batch.db", td);
|
||||
cbm_store_t *writer = cbm_store_open_path(db_path); /* WAL: commits beside a reader */
|
||||
ASSERT_NOT_NULL(writer);
|
||||
ASSERT_EQ(cbm_store_upsert_project(writer, "p", td), CBM_STORE_OK);
|
||||
ASSERT_TRUE(read_batch_rename(writer, "before"));
|
||||
cbm_store_t *reader = cbm_store_open_path_query(db_path);
|
||||
ASSERT_NOT_NULL(reader);
|
||||
|
||||
ASSERT_TRUE(cbm_store_read_batch_begin(reader));
|
||||
ASSERT_FALSE(cbm_store_read_batch_begin(reader)); /* one batch at a time */
|
||||
char *name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "before");
|
||||
free(name);
|
||||
|
||||
/* Inside the batch the reader keeps its snapshot... */
|
||||
ASSERT_TRUE(read_batch_rename(writer, "after"));
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "before");
|
||||
free(name);
|
||||
|
||||
/* ...until the batch renews at its next statement. */
|
||||
cbm_store_read_batch_expire_for_test(reader);
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "after");
|
||||
free(name);
|
||||
|
||||
/* Ended, every statement reads the latest commit again. */
|
||||
cbm_store_read_batch_end(reader);
|
||||
ASSERT_TRUE(read_batch_rename(writer, "latest"));
|
||||
name = read_batch_name(reader);
|
||||
ASSERT_STR_EQ(name, "latest");
|
||||
free(name);
|
||||
|
||||
/* Begin refuses while another transaction is open; close ends a batch. */
|
||||
ASSERT_EQ(cbm_store_begin(writer), CBM_STORE_OK);
|
||||
ASSERT_FALSE(cbm_store_read_batch_begin(writer));
|
||||
ASSERT_EQ(cbm_store_rollback(writer), CBM_STORE_OK);
|
||||
ASSERT_TRUE(cbm_store_read_batch_begin(reader));
|
||||
cbm_store_close(reader);
|
||||
cbm_store_close(writer);
|
||||
th_rmtree(td);
|
||||
PASS();
|
||||
}
|
||||
|
||||
SUITE(store_pragmas) {
|
||||
RUN_TEST(store_read_batch_snapshot_renews);
|
||||
RUN_TEST(journal_size_limit_bounds_wal_issue1083);
|
||||
RUN_TEST(store_role_pragmas_issue1419);
|
||||
RUN_TEST(store_generation_tracks_mutations);
|
||||
|
||||
Reference in New Issue
Block a user