trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-12 09:47:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(pipeline): retain ObjectScript macros during incremental extraction

Browse Source 
Previously, src/pipeline/pass_definitions.c populated the sequential result cache without ctx->macro_table, so pass_calls reused unexpanded $$$ calls. Exact incremental runs also extracted only the changed .cls slice without unchanged .inc definitions, dropping the CALLS edge that a clean rebuild produced.

Route sequential extraction through cbm_extract_file_with_options_ex, declare cbm_build_macro_table_from_files once in pipeline_internal.h, and let parallel extraction borrow an immutable caller-owned table. pipeline_incremental.c now builds repository macro context only when the changed slice contains ObjectScript, releases it on every return path, and otherwise preserves changed-file setup complexity.

tests/test_extraction.c verifies the expanded callee remains scoped to MyApp.Caller.Run. tests/test_pipeline.c verifies full resolution, exact incremental publication, CALLS-edge retention, clean-rebuild graph equality, and default fixture cleanup.

Verification: ASan/UBSan pipeline 382/382, extraction 313/313, parallel 36/36; focused TSan passed; macOS leaks reported 0 leaks for 0 total leaked bytes; bash scripts/check-source-safety.sh; git diff --check.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>
This commit is contained in:
Andrew Hundt
2026-07-19 19:04:23 -04:00
parent b3a7dc5dc1
commit 8f0f6fc834
6 changed files with 262 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline/pass_definitions.c
+6 -2
View File
@@ -575,10 +575,14 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
}
/* Extract */
CBMFileResult *result = cbm_extract_file_with_options(
/* The sequential definitions pass owns the result cache later reused by
* pass_calls. Pass the repository-wide ObjectScript table here; waiting
* until pass_calls is too late because a cache hit skips re-extraction
* and would retain unexpanded $$$ macro calls. */
CBMFileResult *result = cbm_extract_file_with_options_ex(
source, source_len, lang, ctx->project_name, rel, cbm_pipeline_ctx_extract_timeout(ctx),
NULL, NULL /* no extra defines or include paths */,
cbm_pipeline_mode_extracts_macro_nodes(ctx->mode));
cbm_pipeline_mode_extracts_macro_nodes(ctx->mode), ctx->macro_table, NULL);
free(source);
if (!result || result->has_error) {
src/pipeline/pass_parallel.c
+9 -8
View File
@@ -1046,10 +1046,6 @@ static void log_extract_mem_stats(int worker_count) {
}
}
/* Forward declaration: macro table builder lives in pipeline.c (shared path). */
CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count,
const char *repo_path);
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count, const cbm_parallel_extract_opts_t *opts) {
@@ -1139,9 +1135,14 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
return CBM_NOT_FOUND;
}
/* ObjectScript macro table (NULL when no .inc include files present). */
CBMMacroTable *pp_macro_table =
cbm_build_macro_table_from_files(files, file_count, ctx->repo_path);
/* Incremental callers may supply a repository-wide table because `files`
* is only the changed slice. Full indexing builds and owns its table here. */
CBMMacroTable *owned_macro_table = NULL;
const CBMMacroTable *pp_macro_table = (const CBMMacroTable *)ctx->macro_table;
if (!pp_macro_table) {
owned_macro_table = cbm_build_macro_table_from_files(files, file_count, ctx->repo_path);
pp_macro_table = owned_macro_table;
}
extract_ctx_t ec = {
.files = files,
@@ -1219,7 +1220,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
cbm_aligned_free(workers);
free(sorted);
cbm_macro_table_free(pp_macro_table); /* ObjectScript macro table (NULL-safe) */
cbm_macro_table_free(owned_macro_table); /* NULL when the caller owns the table */
if (atomic_load(ctx->cancelled) ||
atomic_load_explicit(&ec.worker_failed, memory_order_relaxed) != 0 || merge_rc != 0) {
src/pipeline/pipeline_incremental.c
+38 -12
View File
@@ -1031,7 +1031,8 @@ static void incr_release_seq_cross_arena(cbm_pipeline_ctx_t *ctx) {
}
/* Run parallel or sequential extract+resolve for changed files. */
static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) {
static int run_extract_resolve_inner(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files,
int ci) {
struct timespec t;
/* Per-file LSP always runs. Sequential scoped increments run the reusable
@@ -1167,11 +1168,8 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
}
sequential_cleanup:
/* Cross-language registries own interned names borrowed by the calls,
* usages, and semantic passes. Release them only after the final
* borrower, including every injected-failure path. */
incr_release_seq_cross_arena(ctx);
cbm_pipeline_release_objectscript_tables(ctx);
/* The outer wrapper releases cross-language registries and ObjectScript
* tables after this final borrower on both success and failure. */
if (rc != 0) {
return rc;
}
@@ -1179,6 +1177,33 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
return 0;
}
static bool incr_changed_uses_objectscript(const cbm_file_info_t *files, int count) {
for (int i = 0; files && i < count; i++) {
CBMLanguage lang = files[i].language;
if (lang == CBM_LANG_OBJECTSCRIPT_UDL || lang == CBM_LANG_OBJECTSCRIPT_ROUTINE ||
lang == CBM_LANG_OBJECTSCRIPT_EXPORT) {
return true;
}
}
return false;
}
/* A changed ObjectScript file can expand a $$$ macro from an unchanged .inc.
* Build that context from the full discovered file set, but only for
* ObjectScript increments so other languages retain O(changed-files) setup.
* This wrapper owns every table and cross-language arena on all return paths. */
static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci,
cbm_file_info_t *all_files, int all_file_count) {
if (incr_changed_uses_objectscript(changed_files, ci)) {
ctx->macro_table =
cbm_build_macro_table_from_files(all_files, all_file_count, ctx->repo_path);
}
int rc = run_extract_resolve_inner(ctx, changed_files, ci);
incr_release_seq_cross_arena(ctx);
cbm_pipeline_release_objectscript_tables(ctx);
return rc;
}
/* Run post-extraction passes (tests, decorator tags, configlink). */
static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci,
const char *project, bool refresh_global_semantic_edges) {
@@ -1740,7 +1765,8 @@ static bool incr_overlay_requires_canonical_structure_publish(cbm_store_t *store
static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
const char *project, cbm_file_info_t *changed_files,
int changed_count, int deleted_count,
int changed_count, cbm_file_info_t *all_files,
int all_file_count, int deleted_count,
const char *pass_fingerprint, int *applied) {
if (applied) {
*applied = 0;
@@ -1869,7 +1895,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
}
}
rc = run_extract_resolve(&ctx, changed_files, changed_count);
rc = run_extract_resolve(&ctx, changed_files, changed_count, all_files, all_file_count);
if (rc != 0) {
cbm_pipeline_set_publish_reason(p, "overlay_extract_resolve");
cbm_log_info("incremental.overlay.fallback", "reason", "extract_resolve", "rc",
@@ -2246,7 +2272,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co
}
CBM_PROF_END_N("incremental_exact", "2_ensure_structure", t_exact_structure, exact_count);
CBM_PROF_START(t_exact_extract_resolve);
rc = run_extract_resolve(&ctx, exact_files, exact_count);
rc = run_extract_resolve(&ctx, exact_files, exact_count, all_files, all_file_count);
CBM_PROF_END_N("incremental_exact", "3_extract_resolve", t_exact_extract_resolve, exact_count);
if (rc != 0) {
cbm_pipeline_set_publish_reason(p, "extract_resolve");
@@ -2834,8 +2860,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
return 0;
}
(void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, cls.deleted_count,
pass_fingerprint, &exact_applied);
(void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, files, file_count,
cls.deleted_count, pass_fingerprint, &exact_applied);
if (exact_applied) {
int coverage_rc = incr_refresh_coverage(store, p, project, changed_files, ci);
if (coverage_rc != CBM_STORE_OK) {
@@ -3074,7 +3100,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
excluded_dirs, excluded_count);
cbm_pipeline_set_pkgmap(incremental_pkgmap);
ctx.pkgmap_preseeded = true;
pipeline_rc = run_extract_resolve(&ctx, changed_files, ci);
pipeline_rc = run_extract_resolve(&ctx, changed_files, ci, files, file_count);
ctx.pkgmap_preseeded = false;
if (cbm_pipeline_get_pkgmap() == incremental_pkgmap) {
cbm_pipeline_set_pkgmap(NULL);
src/pipeline/pipeline_internal.h
+5
View File
@@ -1089,6 +1089,11 @@ typedef struct {
size_t retain_per_file_max_bytes;
} cbm_parallel_extract_opts_t;
/* Shared ObjectScript include context for sequential, parallel, and
* incremental extraction. The caller owns the returned table. */
CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count,
const char *repo_path);
int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
int worker_count, const cbm_parallel_extract_opts_t *opts);