mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-05 08:23:53 +08:00
fix(pipeline): preserve import frontier parity
Consolidate URL-argument route detection so sequential and parallel call resolution emit the same route nodes and HTTP_CALLS edges for arg-url callsites. Follow package/module re-export IMPORTS edges when building per-file import maps, preventing same-name suffix collisions from selecting the wrong target. Expand exact and FAST regular incremental frontiers through persisted graph IMPORTS edges, while keeping full/moderate modes on full rebuild when global derived edges make regular expansion unsafe. Validation: git diff --check; product build /private/tmp/cbm-pan1-20260703T-prod-build-after-import-frontier-constants.log rc=0; pipeline suite /private/tmp/cbm-pan1-20260703T-pipeline-suite-after-import-frontier-constants.log rc=0 with 309 passed. Signed-off-by: Andrew Hundt <ATHundt@gmail.com>
This commit is contained in:
@@ -193,18 +193,18 @@ static void emit_http_async_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call,
|
||||
}
|
||||
|
||||
/* Classify a resolved call and emit the appropriate edge. */
|
||||
static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call,
|
||||
static bool emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call,
|
||||
const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target,
|
||||
const cbm_resolution_t *res, const char *module_qn,
|
||||
const char **imp_keys, const char **imp_vals, int imp_count) {
|
||||
cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name);
|
||||
if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') {
|
||||
handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) {
|
||||
emit_http_async_edge(ctx, call, source, target, res, svc);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
if (svc == CBM_SVC_CONFIG) {
|
||||
char esc_c[CBM_SZ_256];
|
||||
@@ -216,7 +216,7 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call,
|
||||
esc_c, esc_k, res->confidence);
|
||||
calls_emit_edge(ctx->gbuf, source->id, target->id, "CONFIGURES", props, sizeof(props),
|
||||
call);
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
char esc_c2[CBM_SZ_256];
|
||||
cbm_json_escape(esc_c2, sizeof(esc_c2), call->callee_name);
|
||||
@@ -226,6 +226,7 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call,
|
||||
esc_c2, res->confidence, res->strategy ? res->strategy : "unknown",
|
||||
res->candidate_count);
|
||||
calls_emit_edge(ctx->gbuf, source->id, target->id, "CALLS", props, sizeof(props), call);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Find source node for a call: enclosing function or file node. */
|
||||
@@ -285,8 +286,10 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call,
|
||||
res.confidence = lsp->confidence;
|
||||
res.strategy = lsp->strategy;
|
||||
res.candidate_count = 1;
|
||||
emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys,
|
||||
imp_vals, imp_count);
|
||||
if (emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn,
|
||||
imp_keys, imp_vals, imp_count)) {
|
||||
cbm_pipeline_detect_url_arg_routes(ctx->gbuf, source_node, call, rel, lang);
|
||||
}
|
||||
return SKIP_ONE;
|
||||
}
|
||||
}
|
||||
@@ -313,8 +316,10 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call,
|
||||
if (!target_node || source_node->id == target_node->id) {
|
||||
return 0;
|
||||
}
|
||||
emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals,
|
||||
imp_count);
|
||||
if (emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys,
|
||||
imp_vals, imp_count)) {
|
||||
cbm_pipeline_detect_url_arg_routes(ctx->gbuf, source_node, call, rel, lang);
|
||||
}
|
||||
return SKIP_ONE;
|
||||
}
|
||||
|
||||
|
||||
@@ -1149,93 +1149,6 @@ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sou
|
||||
}
|
||||
}
|
||||
|
||||
/* Reject regex metacharacters, spaces, double-slashes in URL candidates. */
|
||||
static bool is_junk_url(const char *s) {
|
||||
for (int i = 0; s[i]; i++) {
|
||||
char ch = s[i];
|
||||
if (ch == '\\' || ch == '^' || ch == '$' || ch == '*' || ch == '+' || ch == '(' ||
|
||||
ch == ')' || ch == '[' || ch == ']' || ch == '|' || ch == ' ') {
|
||||
return true;
|
||||
}
|
||||
if (ch == '/' && i > 0 && s[i - SKIP_ONE] == '/') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Normalize a template literal URL and reject junk patterns.
|
||||
* Returns true if norm contains a valid API path. */
|
||||
static bool normalize_url_arg(const char *url, char *norm, int norm_sz) {
|
||||
int ni = 0;
|
||||
const char *p = url;
|
||||
if (*p == '`' || *p == '"' || *p == '\'') {
|
||||
p++;
|
||||
}
|
||||
if (*p != '/') {
|
||||
return false;
|
||||
}
|
||||
while (*p && ni < norm_sz - PAIR_LEN) {
|
||||
if (*p == '$' && *(p + SKIP_ONE) == '{') {
|
||||
norm[ni++] = ':';
|
||||
p += PAIR_LEN;
|
||||
while (*p && *p != '}' && ni < norm_sz - PAIR_LEN) {
|
||||
norm[ni++] = *p++;
|
||||
}
|
||||
if (*p == '}') {
|
||||
p++;
|
||||
}
|
||||
} else if (*p == '`' || *p == '"' || *p == '\'' || *p == '?') {
|
||||
break;
|
||||
} else {
|
||||
norm[ni++] = *p++;
|
||||
}
|
||||
}
|
||||
norm[ni] = '\0';
|
||||
enum { MIN_URL_LEN = 4 };
|
||||
if (ni < MIN_URL_LEN || !strchr(norm + SKIP_ONE, '/')) {
|
||||
return false;
|
||||
}
|
||||
return !is_junk_url(norm);
|
||||
}
|
||||
|
||||
/* Detect API paths in call arguments and create HTTP_CALLS edges. */
|
||||
static void detect_url_in_args(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call,
|
||||
const char *rel, CBMLanguage lang) {
|
||||
const char *source_path = source && source->file_path && source->file_path[0] ? source->file_path : rel;
|
||||
if (cbm_is_test_file(source_path, lang)) {
|
||||
return;
|
||||
}
|
||||
for (int ai = 0; ai < call->arg_count; ai++) {
|
||||
const CBMCallArg *ca = &call->args[ai];
|
||||
const char *url = ca->value ? ca->value : ca->expr;
|
||||
if (!url || (url[0] != '/' && url[0] != '`')) {
|
||||
continue;
|
||||
}
|
||||
char norm[CBM_SZ_256];
|
||||
if (!normalize_url_arg(url, norm, (int)sizeof(norm))) {
|
||||
continue;
|
||||
}
|
||||
if (!cbm_service_pattern_is_http_route_literal(norm, call->callee_name)) {
|
||||
continue;
|
||||
}
|
||||
int64_t route_id = cbm_pipeline_upsert_service_route(
|
||||
gbuf, norm, CBM_SVC_HTTP, NULL, NULL, "arg_url", source_path ? source_path : "");
|
||||
if (route_id == 0) {
|
||||
continue;
|
||||
}
|
||||
char esc_c[CBM_SZ_256];
|
||||
char esc_n[CBM_SZ_256];
|
||||
cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name);
|
||||
cbm_json_escape(esc_n, sizeof(esc_n), norm);
|
||||
char eprops[CBM_SZ_512];
|
||||
snprintf(eprops, sizeof(eprops),
|
||||
"{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"arg_url\"}", esc_c, esc_n);
|
||||
cbm_gbuf_insert_edge(gbuf, source->id, route_id, "HTTP_CALLS", eprops);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Extract gRPC service and method from a callee name.
|
||||
* Handles patterns like: pb.NewFooServiceClient(conn).GetBar → Foo/GetBar
|
||||
* Also: FooServiceGrpc.newBlockingStub(ch).getBar → FooService/getBar */
|
||||
@@ -1465,7 +1378,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source,
|
||||
emit_normal_calls_edge(gbuf, source, target, call, res);
|
||||
}
|
||||
|
||||
detect_url_in_args(gbuf, source, call, rel, lang);
|
||||
cbm_pipeline_detect_url_arg_routes(gbuf, source, call, rel, lang);
|
||||
}
|
||||
|
||||
/* Find the source node for an edge: enclosing function or file node. */
|
||||
|
||||
@@ -1443,6 +1443,54 @@ char *cbm_pipeline_import_edge_local_name_dup(const cbm_gbuf_edge_t *edge) {
|
||||
return cbm_strndup(start, len);
|
||||
}
|
||||
|
||||
static const cbm_gbuf_node_t *find_file_node_for_module_qn(const cbm_gbuf_t *gbuf,
|
||||
const char *module_qn);
|
||||
|
||||
static bool import_map_target_can_own_reexports(const cbm_gbuf_node_t *target) {
|
||||
return target && target->label &&
|
||||
(strcmp(target->label, "Folder") == 0 || strcmp(target->label, "Module") == 0);
|
||||
}
|
||||
|
||||
static const cbm_gbuf_node_t *
|
||||
resolve_import_map_reexport_target(const cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source_file,
|
||||
const cbm_gbuf_node_t *target, const char *local_name) {
|
||||
if (!gbuf || !source_file || !target || !target->qualified_name || !local_name ||
|
||||
!local_name[0] || strcmp(local_name, "*") == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const cbm_gbuf_node_t *owner_file = NULL;
|
||||
if (target->label && strcmp(target->label, "File") == 0) {
|
||||
owner_file = target;
|
||||
} else if (import_map_target_can_own_reexports(target)) {
|
||||
owner_file = find_file_node_for_module_qn(gbuf, target->qualified_name);
|
||||
}
|
||||
if (!owner_file || owner_file->id == source_file->id) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const cbm_gbuf_edge_t **edges = NULL;
|
||||
int edge_count = 0;
|
||||
int rc = cbm_gbuf_find_edges_by_source_type(gbuf, owner_file->id, "IMPORTS", &edges,
|
||||
&edge_count);
|
||||
if (rc != 0 || edge_count <= 0 || !edges) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const cbm_gbuf_node_t *best = NULL;
|
||||
for (int i = 0; i < edge_count; i++) {
|
||||
if (!import_edge_local_name_equals(edges[i], local_name)) {
|
||||
continue;
|
||||
}
|
||||
const cbm_gbuf_node_t *candidate = cbm_gbuf_find_by_id(gbuf, edges[i]->target_id);
|
||||
if (candidate && cbm_pipeline_label_is_import_target(candidate->label) &&
|
||||
import_target_better(candidate, best, target->qualified_name)) {
|
||||
best = candidate;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
int cbm_pipeline_build_import_map_from_edges(const cbm_gbuf_t *gbuf, const char *project_name,
|
||||
const char *rel_path, const char ***out_keys,
|
||||
const char ***out_vals, int *out_count) {
|
||||
@@ -1491,8 +1539,10 @@ int cbm_pipeline_build_import_map_from_edges(const cbm_gbuf_t *gbuf, const char
|
||||
free(key);
|
||||
continue;
|
||||
}
|
||||
const cbm_gbuf_node_t *resolved =
|
||||
resolve_import_map_reexport_target(gbuf, file_node, target, key);
|
||||
keys[count] = key;
|
||||
vals[count] = target->qualified_name;
|
||||
vals[count] = resolved ? resolved->qualified_name : target->qualified_name;
|
||||
count++;
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ enum {
|
||||
#include <stdint.h>
|
||||
#include "graph_buffer/graph_buffer.h"
|
||||
#include "foundation/log.h"
|
||||
#include "helpers.h"
|
||||
#include "service_patterns.h"
|
||||
|
||||
#include <stdio.h>
|
||||
@@ -198,6 +199,100 @@ int64_t cbm_pipeline_upsert_service_route(cbm_gbuf_t *gb, const char *path, cbm_
|
||||
route_props);
|
||||
}
|
||||
|
||||
/* Reject regex metacharacters, spaces, and double-slashes in URL candidates. */
|
||||
static bool route_arg_has_junk_chars(const char *s) {
|
||||
if (!s) {
|
||||
return true;
|
||||
}
|
||||
for (size_t i = 0; s[i] != '\0'; i++) {
|
||||
char ch = s[i];
|
||||
if (ch == '\\' || ch == '^' || ch == '$' || ch == '*' || ch == '+' || ch == '(' ||
|
||||
ch == ')' || ch == '[' || ch == ']' || ch == '|' || ch == ' ') {
|
||||
return true;
|
||||
}
|
||||
if (ch == '/' && i > 0 && s[i - SKIP_ONE] == '/') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Normalize a slash-path or template-literal URL argument. */
|
||||
static bool route_arg_normalize_url(const char *url, char *norm, size_t norm_sz) {
|
||||
enum { RN_URL_ARG_MIN_PATH_LEN = 4 };
|
||||
if (!url || !norm || norm_sz == 0) {
|
||||
return false;
|
||||
}
|
||||
size_t ni = 0;
|
||||
const char *p = url;
|
||||
if (*p == '`' || *p == '"' || *p == '\'') {
|
||||
p++;
|
||||
}
|
||||
if (*p != '/') {
|
||||
return false;
|
||||
}
|
||||
while (*p && ni + PAIR_LEN < norm_sz) {
|
||||
if (*p == '$' && *(p + SKIP_ONE) == '{') {
|
||||
norm[ni++] = ':';
|
||||
p += PAIR_LEN;
|
||||
while (*p && *p != '}' && ni + PAIR_LEN < norm_sz) {
|
||||
norm[ni++] = *p++;
|
||||
}
|
||||
if (*p == '}') {
|
||||
p++;
|
||||
}
|
||||
} else if (*p == '`' || *p == '"' || *p == '\'' || *p == '?') {
|
||||
break;
|
||||
} else {
|
||||
norm[ni++] = *p++;
|
||||
}
|
||||
}
|
||||
norm[ni] = '\0';
|
||||
if (ni < RN_URL_ARG_MIN_PATH_LEN || !strchr(norm + SKIP_ONE, '/')) {
|
||||
return false;
|
||||
}
|
||||
return !route_arg_has_junk_chars(norm);
|
||||
}
|
||||
|
||||
void cbm_pipeline_detect_url_arg_routes(cbm_gbuf_t *gb, const cbm_gbuf_node_t *source,
|
||||
const CBMCall *call, const char *rel_path,
|
||||
CBMLanguage lang) {
|
||||
if (!gb || !source || !call) {
|
||||
return;
|
||||
}
|
||||
const char *source_path =
|
||||
source->file_path && source->file_path[0] ? source->file_path : rel_path;
|
||||
if (cbm_is_test_file(source_path, lang)) {
|
||||
return;
|
||||
}
|
||||
for (int ai = 0; ai < call->arg_count; ai++) {
|
||||
const CBMCallArg *ca = &call->args[ai];
|
||||
const char *url = ca->value ? ca->value : ca->expr;
|
||||
if (!url || (url[0] != '/' && url[0] != '`')) {
|
||||
continue;
|
||||
}
|
||||
char norm[CBM_SZ_256];
|
||||
if (!route_arg_normalize_url(url, norm, sizeof(norm)) ||
|
||||
!cbm_service_pattern_is_http_route_literal(norm, call->callee_name)) {
|
||||
continue;
|
||||
}
|
||||
int64_t route_id = cbm_pipeline_upsert_service_route(
|
||||
gb, norm, CBM_SVC_HTTP, NULL, NULL, "arg_url", source_path ? source_path : "");
|
||||
if (route_id == 0) {
|
||||
continue;
|
||||
}
|
||||
char esc_c[CBM_SZ_256];
|
||||
char esc_n[CBM_SZ_256];
|
||||
cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name);
|
||||
cbm_json_escape(esc_n, sizeof(esc_n), norm);
|
||||
char props[CBM_SZ_512];
|
||||
snprintf(props, sizeof(props),
|
||||
"{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"arg_url\"}", esc_c, esc_n);
|
||||
cbm_gbuf_insert_edge(gb, source->id, route_id, "HTTP_CALLS", props);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Extract a simple JSON string property emitted by the pipeline. Returns false
|
||||
* for missing, empty, overlong, or non-string-looking values; callers must not
|
||||
* route on truncated data. */
|
||||
|
||||
@@ -1031,6 +1031,84 @@ static bool incr_inbound_edge_owner_is_exact_file(const cbm_store_inbound_edge_t
|
||||
incr_file_info_has_rel_path(exact_files, exact_count, edge->edge_rel_path);
|
||||
}
|
||||
|
||||
static void incr_free_text_array(char **items, int count) {
|
||||
if (!items) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < count; i++) {
|
||||
free(items[i]);
|
||||
}
|
||||
free(items);
|
||||
}
|
||||
|
||||
static int incr_append_exact_frontier_path(const char *source_rel_path,
|
||||
const cbm_file_info_t *all_files, int all_file_count,
|
||||
cbm_file_info_t *exact_files, int *exact_count,
|
||||
int max_exact_files, const char **out_reason) {
|
||||
if (!source_rel_path || !exact_files || !exact_count) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR;
|
||||
}
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
if (incr_file_info_has_rel_path(exact_files, *exact_count, source_rel_path)) {
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
if (*exact_count >= max_exact_files) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_FRONTIER_TOO_LARGE;
|
||||
}
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
const cbm_file_info_t *source_file =
|
||||
incr_find_file_info_by_rel_path(all_files, all_file_count, source_rel_path);
|
||||
if (!source_file) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_FRONTIER_REQUIRES_BATCH;
|
||||
}
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
exact_files[(*exact_count)++] = *source_file;
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
|
||||
static const char *incr_path_basename(const char *rel_path) {
|
||||
const char *slash = rel_path ? strrchr(rel_path, '/') : NULL;
|
||||
return slash ? slash + SKIP_ONE : rel_path;
|
||||
}
|
||||
|
||||
static bool incr_basename_is_package_entry(const char *basename) {
|
||||
static const char init_stem[] = "__init__";
|
||||
static const char index_stem[] = "index";
|
||||
if (!basename || !basename[0]) {
|
||||
return false;
|
||||
}
|
||||
const char *dot = strrchr(basename, '.');
|
||||
size_t stem_len = dot ? (size_t)(dot - basename) : strlen(basename);
|
||||
return (stem_len == sizeof(init_stem) - SKIP_ONE &&
|
||||
memcmp(basename, init_stem, stem_len) == 0) ||
|
||||
(stem_len == sizeof(index_stem) - SKIP_ONE &&
|
||||
memcmp(basename, index_stem, stem_len) == 0);
|
||||
}
|
||||
|
||||
static char *incr_frontier_import_target_qn(const char *project, const char *rel_path) {
|
||||
const char *basename = incr_path_basename(rel_path);
|
||||
if (basename && basename != rel_path && incr_basename_is_package_entry(basename)) {
|
||||
size_t dir_len = (size_t)(basename - rel_path);
|
||||
while (dir_len > 0 && rel_path[dir_len - SKIP_ONE] == '/') {
|
||||
dir_len--;
|
||||
}
|
||||
if (dir_len < CBM_PATH_MAX) {
|
||||
char dir[CBM_PATH_MAX];
|
||||
memcpy(dir, rel_path, dir_len);
|
||||
dir[dir_len] = '\0';
|
||||
return cbm_pipeline_fqn_folder(project, dir);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
return cbm_pipeline_fqn_module(project, rel_path);
|
||||
}
|
||||
|
||||
static int incr_expand_exact_inbound_frontier(cbm_store_t *store, const char *project,
|
||||
const cbm_file_info_t *all_files, int all_file_count,
|
||||
cbm_file_info_t *exact_files, int *exact_count,
|
||||
@@ -1090,28 +1168,117 @@ static int incr_expand_exact_inbound_frontier(cbm_store_t *store, const char *pr
|
||||
cbm_store_free_inbound_edges(edges, edge_count);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
if (incr_file_info_has_rel_path(exact_files, *exact_count, source_rel_path)) {
|
||||
continue;
|
||||
}
|
||||
if (*exact_count >= max_exact_files) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_FRONTIER_TOO_LARGE;
|
||||
}
|
||||
rc = incr_append_exact_frontier_path(source_rel_path, all_files, all_file_count,
|
||||
exact_files, exact_count, max_exact_files,
|
||||
out_reason);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
cbm_store_free_inbound_edges(edges, edge_count);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
return rc;
|
||||
}
|
||||
const cbm_file_info_t *source_file =
|
||||
incr_find_file_info_by_rel_path(all_files, all_file_count, source_rel_path);
|
||||
if (!source_file) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_FRONTIER_REQUIRES_BATCH;
|
||||
}
|
||||
cbm_store_free_inbound_edges(edges, edge_count);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
exact_files[(*exact_count)++] = *source_file;
|
||||
}
|
||||
cbm_store_free_inbound_edges(edges, edge_count);
|
||||
|
||||
char *module_qn = incr_frontier_import_target_qn(project, rel_path);
|
||||
if (!module_qn) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR;
|
||||
}
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
char **importers = NULL;
|
||||
int importer_count = 0;
|
||||
rc = cbm_store_list_import_edge_source_paths_by_target_qn(store, project, module_qn,
|
||||
&importers, &importer_count);
|
||||
free(module_qn);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
if (out_reason) {
|
||||
*out_reason = CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
for (int i = 0; i < importer_count; i++) {
|
||||
rc = incr_append_exact_frontier_path(importers[i], all_files, all_file_count,
|
||||
exact_files, exact_count, max_exact_files,
|
||||
out_reason);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
incr_free_text_array(importers, importer_count);
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
incr_free_text_array(importers, importer_count);
|
||||
}
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
|
||||
static int incr_expand_regular_changed_frontier(cbm_store_t *store, const char *project,
|
||||
const cbm_file_info_t *all_files,
|
||||
int all_file_count,
|
||||
cbm_incr_classification_t *cls,
|
||||
bool allow_expansion) {
|
||||
if (!store || !project || !all_files || all_file_count <= 0 || !cls ||
|
||||
cls->changed_file_count <= 0) {
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
|
||||
cbm_file_info_t *expanded = malloc((size_t)all_file_count * sizeof(*expanded));
|
||||
if (!expanded) {
|
||||
cbm_log_info("incremental.frontier.fallback", "reason", "alloc");
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
int expanded_count = cls->changed_file_count;
|
||||
for (int i = 0; i < cls->changed_file_count; i++) {
|
||||
expanded[i] = cls->changed_files[i];
|
||||
}
|
||||
|
||||
for (int cursor = 0; cursor < expanded_count; cursor++) {
|
||||
const char *rel_path = expanded[cursor].rel_path;
|
||||
char *module_qn = incr_frontier_import_target_qn(project, rel_path);
|
||||
if (!module_qn) {
|
||||
cbm_log_info("incremental.frontier.fallback", "reason",
|
||||
CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR);
|
||||
free(expanded);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
char **importers = NULL;
|
||||
int importer_count = 0;
|
||||
int rc = cbm_store_list_import_edge_source_paths_by_target_qn(store, project, module_qn,
|
||||
&importers, &importer_count);
|
||||
free(module_qn);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
cbm_log_info("incremental.frontier.fallback", "reason",
|
||||
CBM_PIPELINE_DELTA_REASON_PREFLIGHT_ERROR);
|
||||
free(expanded);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
const char *reason = NULL;
|
||||
for (int i = 0; i < importer_count; i++) {
|
||||
rc = incr_append_exact_frontier_path(importers[i], all_files, all_file_count, expanded,
|
||||
&expanded_count, all_file_count, &reason);
|
||||
if (rc != CBM_STORE_OK) {
|
||||
cbm_log_info("incremental.frontier.fallback", "reason",
|
||||
reason ? reason : CBM_PIPELINE_DELTA_REASON_FRONTIER_ERROR);
|
||||
incr_free_text_array(importers, importer_count);
|
||||
free(expanded);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
}
|
||||
incr_free_text_array(importers, importer_count);
|
||||
}
|
||||
|
||||
if (expanded_count > cls->changed_file_count) {
|
||||
if (!allow_expansion) {
|
||||
cbm_log_info("incremental.frontier.fallback", "reason", "global_derived_edges");
|
||||
free(expanded);
|
||||
return CBM_STORE_NOT_FOUND;
|
||||
}
|
||||
cbm_log_info("incremental.frontier", "changed", itoa_buf_incr(cls->changed_file_count),
|
||||
"expanded", itoa_buf_incr(expanded_count));
|
||||
free(cls->changed_files);
|
||||
cls->changed_files = expanded;
|
||||
cls->changed_file_count = expanded_count;
|
||||
cls->n_changed = expanded_count;
|
||||
} else {
|
||||
free(expanded);
|
||||
}
|
||||
return CBM_STORE_OK;
|
||||
}
|
||||
@@ -1643,6 +1810,20 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool regular_frontier_expansion_ok = cbm_pipeline_get_mode(p) >= CBM_MODE_FAST;
|
||||
if (incr_expand_regular_changed_frontier(store, project, files, file_count, &cls,
|
||||
regular_frontier_expansion_ok) != CBM_STORE_OK) {
|
||||
const char *reason = regular_frontier_expansion_ok
|
||||
? CBM_PIPELINE_DELTA_REASON_FRONTIER_REQUIRES_BATCH
|
||||
: "global_derived_edges";
|
||||
incr_classification_free(&cls);
|
||||
cbm_store_close(store);
|
||||
cbm_log_info("incremental.fallback", "reason", reason);
|
||||
return CBM_NOT_FOUND;
|
||||
}
|
||||
changed_files = cls.changed_files;
|
||||
ci = cls.changed_file_count;
|
||||
|
||||
struct timespec t;
|
||||
|
||||
/* Step 1: Load existing graph into RAM */
|
||||
|
||||
@@ -54,6 +54,10 @@ int64_t cbm_pipeline_upsert_service_route(cbm_gbuf_t *gb, const char *path, cbm_
|
||||
const char *method, const char *broker,
|
||||
const char *source, const char *file_path);
|
||||
|
||||
void cbm_pipeline_detect_url_arg_routes(cbm_gbuf_t *gb, const cbm_gbuf_node_t *source,
|
||||
const CBMCall *call, const char *rel_path,
|
||||
CBMLanguage lang);
|
||||
|
||||
static inline bool cbm_pipeline_label_is_registry_symbol(const char *label) {
|
||||
return label && (strcmp(label, "Function") == 0 || strcmp(label, "Method") == 0 ||
|
||||
cbm_label_is_type_like(label) || strcmp(label, "Variable") == 0 ||
|
||||
|
||||
@@ -161,6 +161,7 @@ struct cbm_store {
|
||||
sqlite3_stmt *stmt_upsert_import_ref;
|
||||
sqlite3_stmt *stmt_delete_import_refs_by_file;
|
||||
sqlite3_stmt *stmt_list_import_ref_paths_by_target;
|
||||
sqlite3_stmt *stmt_list_import_edge_source_paths_by_target_qn;
|
||||
sqlite3_stmt *stmt_list_import_ref_paths_for_export_file;
|
||||
sqlite3_stmt *stmt_delete_owned_edges_by_file;
|
||||
sqlite3_stmt *stmt_delete_owned_nodes_by_file;
|
||||
@@ -1121,6 +1122,7 @@ void cbm_store_close(cbm_store_t *s) {
|
||||
finalize_stmt(&s->stmt_upsert_import_ref);
|
||||
finalize_stmt(&s->stmt_delete_import_refs_by_file);
|
||||
finalize_stmt(&s->stmt_list_import_ref_paths_by_target);
|
||||
finalize_stmt(&s->stmt_list_import_edge_source_paths_by_target_qn);
|
||||
finalize_stmt(&s->stmt_list_import_ref_paths_for_export_file);
|
||||
finalize_stmt(&s->stmt_delete_owned_edges_by_file);
|
||||
finalize_stmt(&s->stmt_delete_owned_nodes_by_file);
|
||||
@@ -2856,6 +2858,40 @@ int cbm_store_list_import_ref_paths_by_target(cbm_store_t *s, const char *projec
|
||||
return store_collect_text_column(s, stmt, "list_import_ref_paths_by_target", out, count);
|
||||
}
|
||||
|
||||
int cbm_store_list_import_edge_source_paths_by_target_qn(cbm_store_t *s, const char *project,
|
||||
const char *target_qn, char ***out,
|
||||
int *count) {
|
||||
if (out) {
|
||||
*out = NULL;
|
||||
}
|
||||
if (count) {
|
||||
*count = 0;
|
||||
}
|
||||
if (!s || !project || !target_qn || !out || !count) {
|
||||
if (s) {
|
||||
store_set_error(s, "list_import_edge_source_paths_by_target_qn: invalid argument");
|
||||
}
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
sqlite3_stmt *stmt = prepare_cached(
|
||||
s, &s->stmt_list_import_edge_source_paths_by_target_qn,
|
||||
"SELECT DISTINCT src.file_path FROM edges e "
|
||||
"JOIN nodes src ON src.project = e.project AND src.id = e.source_id "
|
||||
"JOIN nodes tgt ON tgt.project = e.project AND tgt.id = e.target_id "
|
||||
"WHERE e.project = ?1 AND e.type = 'IMPORTS' AND tgt.qualified_name = ?2 "
|
||||
" AND src.file_path IS NOT NULL AND src.file_path <> '' "
|
||||
"ORDER BY src.file_path;");
|
||||
if (!stmt) {
|
||||
return CBM_STORE_ERR;
|
||||
}
|
||||
|
||||
bind_text(stmt, ST_COL_1, project);
|
||||
bind_text(stmt, ST_COL_2, target_qn);
|
||||
return store_collect_text_column(s, stmt, "list_import_edge_source_paths_by_target_qn", out,
|
||||
count);
|
||||
}
|
||||
|
||||
int cbm_store_list_import_ref_paths_for_export_file(cbm_store_t *s, const char *project,
|
||||
const char *export_rel_path, char ***out,
|
||||
int *count) {
|
||||
|
||||
@@ -587,6 +587,12 @@ int cbm_store_list_import_ref_paths_by_target(cbm_store_t *s, const char *projec
|
||||
const char *target_qn, char ***out,
|
||||
int *count);
|
||||
|
||||
/* Caller frees each returned string and the array. Uses persisted graph IMPORTS
|
||||
* edges, not import_refs metadata, so it also works for full-indexed stores. */
|
||||
int cbm_store_list_import_edge_source_paths_by_target_qn(cbm_store_t *s, const char *project,
|
||||
const char *target_qn, char ***out,
|
||||
int *count);
|
||||
|
||||
/* Caller frees each returned string and the array. */
|
||||
int cbm_store_list_import_ref_paths_for_export_file(cbm_store_t *s, const char *project,
|
||||
const char *export_rel_path, char ***out,
|
||||
|
||||
@@ -8232,6 +8232,30 @@ static int write_incremental_frontier_fixture(int leaf_value) {
|
||||
return write_incremental_frontier_callers();
|
||||
}
|
||||
|
||||
static int write_incremental_arg_url_route_file(const char *route_path, int marker) {
|
||||
char path[CBM_PATH_MAX];
|
||||
int n = snprintf(path, sizeof(path), "%s/http_routes.c", g_incr_tmpdir);
|
||||
if (n < 0 || (size_t)n >= sizeof(path)) {
|
||||
return -1;
|
||||
}
|
||||
char body[CBM_SZ_1K];
|
||||
n = snprintf(body, sizeof(body),
|
||||
"static int cbm_http_path_match(const char *path, const char *pattern) {\n"
|
||||
" return path && pattern;\n"
|
||||
"}\n\n"
|
||||
"int dispatch_request(const char *path) {\n"
|
||||
" if (cbm_http_path_match(path, \"%s\")) {\n"
|
||||
" return %d;\n"
|
||||
" }\n"
|
||||
" return 0;\n"
|
||||
"}\n",
|
||||
route_path, marker);
|
||||
if (n < 0 || (size_t)n >= sizeof(body)) {
|
||||
return -1;
|
||||
}
|
||||
return th_write_file(path, body);
|
||||
}
|
||||
|
||||
static int pipeline_store_insert_file_owned_unowned_source_edge(const char *db_path,
|
||||
const char *project,
|
||||
const char *rel_path,
|
||||
@@ -8823,6 +8847,55 @@ TEST(import_edge_helper_preserves_long_local_name) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(import_map_from_edges_follows_package_reexport) {
|
||||
cbm_gbuf_t *gb = cbm_gbuf_new("proj", "/tmp/proj");
|
||||
ASSERT_NOT_NULL(gb);
|
||||
|
||||
int64_t source_file =
|
||||
cbm_gbuf_upsert_node(gb, "File", "main.py", "proj.app.main.__file__", "app/main.py", 1,
|
||||
1, "{}");
|
||||
int64_t package_module =
|
||||
cbm_gbuf_upsert_node(gb, "Folder", "fastapi", "proj.fastapi", "fastapi", 1,
|
||||
1, "{}");
|
||||
int64_t package_file = cbm_gbuf_upsert_node(gb, "File", "__init__.py", "proj.fastapi.__file__",
|
||||
"fastapi/__init__.py", 1, 1, "{}");
|
||||
int64_t wrong_header = cbm_gbuf_upsert_node(gb, "Class", "Header",
|
||||
"proj.fastapi.openapi.models.Header",
|
||||
"fastapi/openapi/models.py", 1, 1, "{}");
|
||||
int64_t exported_header =
|
||||
cbm_gbuf_upsert_node(gb, "Function", "Header", "proj.fastapi.param_functions.Header",
|
||||
"fastapi/param_functions.py", 1, 1, "{}");
|
||||
ASSERT_GT(source_file, 0);
|
||||
ASSERT_GT(package_module, 0);
|
||||
ASSERT_GT(package_file, 0);
|
||||
ASSERT_GT(wrong_header, 0);
|
||||
ASSERT_GT(exported_header, 0);
|
||||
|
||||
cbm_pipeline_ctx_t ctx = {
|
||||
.gbuf = gb,
|
||||
.project_name = "proj",
|
||||
};
|
||||
ASSERT_EQ(cbm_pipeline_insert_import_edge(&ctx, source_file,
|
||||
cbm_gbuf_find_by_id(gb, package_module), "Header"),
|
||||
1);
|
||||
cbm_gbuf_insert_edge(gb, package_file, exported_header, "IMPORTS",
|
||||
"{\"local_name\":\"Header\"}");
|
||||
|
||||
const char **keys = NULL;
|
||||
const char **vals = NULL;
|
||||
int import_count = 0;
|
||||
ASSERT_EQ(cbm_pipeline_build_import_map_from_edges(gb, "proj", "app/main.py", &keys, &vals,
|
||||
&import_count),
|
||||
0);
|
||||
ASSERT_EQ(import_count, 1);
|
||||
ASSERT_STR_EQ(keys[0], "Header");
|
||||
ASSERT_STR_EQ(vals[0], "proj.fastapi.param_functions.Header");
|
||||
|
||||
cbm_pipeline_free_import_map(keys, vals, import_count);
|
||||
cbm_gbuf_free(gb);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(import_reexport_falls_back_when_pkgmap_target_missing) {
|
||||
cbm_gbuf_t *gb = cbm_gbuf_new("proj", "/tmp/proj");
|
||||
ASSERT_NOT_NULL(gb);
|
||||
@@ -10068,6 +10141,69 @@ TEST(incremental_fast_route_decorator_change_matches_full_rebuild) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(incremental_fast_arg_url_route_change_matches_parallel_full_rebuild) {
|
||||
enum { ARG_URL_FILLER_FILES = 52, ARG_URL_WORKERS = 4 };
|
||||
pipeline_env_snapshot_t workers_env = pipeline_env_save("CBM_WORKERS");
|
||||
if (setup_incremental_repo() != 0) {
|
||||
FAIL("setup failed");
|
||||
}
|
||||
|
||||
char worker_buf[CBM_SZ_32];
|
||||
int n = snprintf(worker_buf, sizeof(worker_buf), "%d", ARG_URL_WORKERS);
|
||||
ASSERT(n > 0 && (size_t)n < sizeof(worker_buf));
|
||||
ASSERT_EQ(cbm_setenv("CBM_WORKERS", worker_buf, 1), 0);
|
||||
|
||||
for (int i = 0; i < ARG_URL_FILLER_FILES; i++) {
|
||||
char path[CBM_PATH_MAX];
|
||||
char body[CBM_SZ_256];
|
||||
n = snprintf(path, sizeof(path), "%s/filler_%02d.c", g_incr_tmpdir, i);
|
||||
ASSERT(n > 0 && (size_t)n < sizeof(path));
|
||||
n = snprintf(body, sizeof(body), "int filler_%02d(void) { return %d; }\n", i, i);
|
||||
ASSERT(n > 0 && (size_t)n < sizeof(body));
|
||||
ASSERT_EQ(th_write_file(path, body), 0);
|
||||
}
|
||||
ASSERT_EQ(write_incremental_arg_url_route_file("/api/index", 1), 0);
|
||||
|
||||
cbm_config_t *cfg = incremental_test_config(g_incr_tmpdir);
|
||||
ASSERT_NOT_NULL(cfg);
|
||||
cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_pipeline_apply_config(p, cfg);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
char *project = cbm_strdup(cbm_pipeline_project_name(p));
|
||||
cbm_pipeline_free(p);
|
||||
ASSERT_NOT_NULL(project);
|
||||
ASSERT(pipeline_store_has_route_name(g_incr_dbpath, project, "/api/index"));
|
||||
|
||||
ASSERT_EQ(write_incremental_arg_url_route_file("/api/index-status", 2), 0);
|
||||
|
||||
p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FAST);
|
||||
ASSERT_NOT_NULL(p);
|
||||
cbm_pipeline_apply_config(p, cfg);
|
||||
ASSERT_EQ(cbm_pipeline_run(p), 0);
|
||||
cbm_pipeline_publish_kind_t kind = cbm_pipeline_publish_kind(p);
|
||||
ASSERT(kind == CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT ||
|
||||
kind == CBM_PIPELINE_PUBLISH_INCREMENTAL_CONTAINMENT);
|
||||
cbm_pipeline_free(p);
|
||||
|
||||
ASSERT(!pipeline_store_has_route_name(g_incr_dbpath, project, "/api/index"));
|
||||
ASSERT(pipeline_store_has_route_name(g_incr_dbpath, project, "/api/index-status"));
|
||||
|
||||
char diff_err[CBM_SZ_8K] = {0};
|
||||
int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild(
|
||||
g_incr_tmpdir, g_incr_dbpath, project, cfg, diff_err, sizeof(diff_err));
|
||||
if (diff_rc != 0) {
|
||||
FAIL(diff_err[0] ? diff_err : "arg-url route incremental differed from fresh FAST rebuild");
|
||||
}
|
||||
ASSERT_EQ(diff_rc, 0);
|
||||
|
||||
free(project);
|
||||
cbm_config_close(cfg);
|
||||
cleanup_incremental_repo();
|
||||
pipeline_env_restore(&workers_env);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(incremental_fast_exact_scratch_multifile_usage_edges_match_fresh) {
|
||||
if (setup_incremental_repo() != 0) {
|
||||
FAIL("setup failed");
|
||||
@@ -12357,6 +12493,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(pipeline_fastapi_depends_edges);
|
||||
RUN_TEST(import_edge_helper_escapes_local_name_once);
|
||||
RUN_TEST(import_edge_helper_preserves_long_local_name);
|
||||
RUN_TEST(import_map_from_edges_follows_package_reexport);
|
||||
RUN_TEST(import_reexport_falls_back_when_pkgmap_target_missing);
|
||||
RUN_TEST(import_symbol_fallback_prefers_import_path_over_insertion_order);
|
||||
/* Incremental */
|
||||
@@ -12376,6 +12513,7 @@ SUITE(pipeline) {
|
||||
RUN_TEST(incremental_fast_rename_like_batch_falls_back_to_full_rebuild_parity);
|
||||
RUN_TEST(incremental_fast_new_folder_exact_delta_parity);
|
||||
RUN_TEST(incremental_fast_route_decorator_change_matches_full_rebuild);
|
||||
RUN_TEST(incremental_fast_arg_url_route_change_matches_parallel_full_rebuild);
|
||||
RUN_TEST(incremental_fast_exact_scratch_multifile_usage_edges_match_fresh);
|
||||
RUN_TEST(incremental_fast_exact_batch_publish_matches_fresh_rebuild_for_two_file_go);
|
||||
RUN_TEST(incremental_full_mode_keeps_exact_upsert_disabled);
|
||||
|
||||
Reference in New Issue
Block a user