fix(pipeline): retain ObjectScript macros during incremental extraction

Previously, src/pipeline/pass_definitions.c populated the sequential result cache without ctx->macro_table, so pass_calls reused unexpanded $$$ calls. Exact incremental runs also extracted only the changed .cls slice without unchanged .inc definitions, dropping the CALLS edge that a clean rebuild produced.

Route sequential extraction through cbm_extract_file_with_options_ex, declare cbm_build_macro_table_from_files once in pipeline_internal.h, and let parallel extraction borrow an immutable caller-owned table. pipeline_incremental.c now builds repository macro context only when the changed slice contains ObjectScript, releases it on every return path, and otherwise preserves changed-file setup complexity.

tests/test_extraction.c verifies the expanded callee remains scoped to MyApp.Caller.Run. tests/test_pipeline.c verifies full resolution, exact incremental publication, CALLS-edge retention, clean-rebuild graph equality, and default fixture cleanup.

Verification: ASan/UBSan pipeline 382/382, extraction 313/313, parallel 36/36; focused TSan passed; macOS leaks reported 0 leaks for 0 total leaked bytes; bash scripts/check-source-safety.sh; git diff --check.
Signed-off-by: Andrew Hundt <ATHundt@gmail.com>

src/pipeline/pass_definitions.c

@@ -575,10 +575,14 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
         }
 
         /* Extract */
-        CBMFileResult *result = cbm_extract_file_with_options(
+        /* The sequential definitions pass owns the result cache later reused by
+         * pass_calls. Pass the repository-wide ObjectScript table here; waiting
+         * until pass_calls is too late because a cache hit skips re-extraction
+         * and would retain unexpanded $$$ macro calls. */
+        CBMFileResult *result = cbm_extract_file_with_options_ex(
             source, source_len, lang, ctx->project_name, rel, cbm_pipeline_ctx_extract_timeout(ctx),
             NULL, NULL /* no extra defines or include paths */,
-            cbm_pipeline_mode_extracts_macro_nodes(ctx->mode));
+            cbm_pipeline_mode_extracts_macro_nodes(ctx->mode), ctx->macro_table, NULL);
         free(source);
 
         if (!result || result->has_error) {

src/pipeline/pass_parallel.c

@@ -1046,10 +1046,6 @@ static void log_extract_mem_stats(int worker_count) {
     }
 }
 
-/* Forward declaration: macro table builder lives in pipeline.c (shared path). */
-CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count,
-                                                const char *repo_path);
-
 int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
                             CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
                             int worker_count, const cbm_parallel_extract_opts_t *opts) {
@@ -1139,9 +1135,14 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
         return CBM_NOT_FOUND;
     }
 
-    /* ObjectScript macro table (NULL when no .inc include files present). */
-    CBMMacroTable *pp_macro_table =
-        cbm_build_macro_table_from_files(files, file_count, ctx->repo_path);
+    /* Incremental callers may supply a repository-wide table because `files`
+     * is only the changed slice. Full indexing builds and owns its table here. */
+    CBMMacroTable *owned_macro_table = NULL;
+    const CBMMacroTable *pp_macro_table = (const CBMMacroTable *)ctx->macro_table;
+    if (!pp_macro_table) {
+        owned_macro_table = cbm_build_macro_table_from_files(files, file_count, ctx->repo_path);
+        pp_macro_table = owned_macro_table;
+    }
 
     extract_ctx_t ec = {
         .files = files,
@@ -1219,7 +1220,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
 
     cbm_aligned_free(workers);
     free(sorted);
-    cbm_macro_table_free(pp_macro_table); /* ObjectScript macro table (NULL-safe) */
+    cbm_macro_table_free(owned_macro_table); /* NULL when the caller owns the table */
 
     if (atomic_load(ctx->cancelled) ||
         atomic_load_explicit(&ec.worker_failed, memory_order_relaxed) != 0 || merge_rc != 0) {

src/pipeline/pipeline_incremental.c

@@ -1031,7 +1031,8 @@ static void incr_release_seq_cross_arena(cbm_pipeline_ctx_t *ctx) {
 }
 
 /* Run parallel or sequential extract+resolve for changed files. */
-static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) {
+static int run_extract_resolve_inner(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files,
+                                     int ci) {
     struct timespec t;
 
     /* Per-file LSP always runs. Sequential scoped increments run the reusable
@@ -1167,11 +1168,8 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
         }
 
     sequential_cleanup:
-        /* Cross-language registries own interned names borrowed by the calls,
-         * usages, and semantic passes. Release them only after the final
-         * borrower, including every injected-failure path. */
-        incr_release_seq_cross_arena(ctx);
-        cbm_pipeline_release_objectscript_tables(ctx);
+        /* The outer wrapper releases cross-language registries and ObjectScript
+         * tables after this final borrower on both success and failure. */
         if (rc != 0) {
             return rc;
         }
@@ -1179,6 +1177,33 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
     return 0;
 }
 
+static bool incr_changed_uses_objectscript(const cbm_file_info_t *files, int count) {
+    for (int i = 0; files && i < count; i++) {
+        CBMLanguage lang = files[i].language;
+        if (lang == CBM_LANG_OBJECTSCRIPT_UDL || lang == CBM_LANG_OBJECTSCRIPT_ROUTINE ||
+            lang == CBM_LANG_OBJECTSCRIPT_EXPORT) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* A changed ObjectScript file can expand a $$$ macro from an unchanged .inc.
+ * Build that context from the full discovered file set, but only for
+ * ObjectScript increments so other languages retain O(changed-files) setup.
+ * This wrapper owns every table and cross-language arena on all return paths. */
+static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci,
+                               cbm_file_info_t *all_files, int all_file_count) {
+    if (incr_changed_uses_objectscript(changed_files, ci)) {
+        ctx->macro_table =
+            cbm_build_macro_table_from_files(all_files, all_file_count, ctx->repo_path);
+    }
+    int rc = run_extract_resolve_inner(ctx, changed_files, ci);
+    incr_release_seq_cross_arena(ctx);
+    cbm_pipeline_release_objectscript_tables(ctx);
+    return rc;
+}
+
 /* Run post-extraction passes (tests, decorator tags, configlink). */
 static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci,
                           const char *project, bool refresh_global_semantic_edges) {
@@ -1740,7 +1765,8 @@ static bool incr_overlay_requires_canonical_structure_publish(cbm_store_t *store
 
 static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
                                          const char *project, cbm_file_info_t *changed_files,
-                                         int changed_count, int deleted_count,
+                                         int changed_count, cbm_file_info_t *all_files,
+                                         int all_file_count, int deleted_count,
                                          const char *pass_fingerprint, int *applied) {
     if (applied) {
         *applied = 0;
@@ -1869,7 +1895,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store,
         }
     }
 
-    rc = run_extract_resolve(&ctx, changed_files, changed_count);
+    rc = run_extract_resolve(&ctx, changed_files, changed_count, all_files, all_file_count);
     if (rc != 0) {
         cbm_pipeline_set_publish_reason(p, "overlay_extract_resolve");
         cbm_log_info("incremental.overlay.fallback", "reason", "extract_resolve", "rc",
@@ -2246,7 +2272,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co
     }
     CBM_PROF_END_N("incremental_exact", "2_ensure_structure", t_exact_structure, exact_count);
     CBM_PROF_START(t_exact_extract_resolve);
-    rc = run_extract_resolve(&ctx, exact_files, exact_count);
+    rc = run_extract_resolve(&ctx, exact_files, exact_count, all_files, all_file_count);
     CBM_PROF_END_N("incremental_exact", "3_extract_resolve", t_exact_extract_resolve, exact_count);
     if (rc != 0) {
         cbm_pipeline_set_publish_reason(p, "extract_resolve");
@@ -2834,8 +2860,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
         return 0;
     }
 
-    (void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, cls.deleted_count,
-                                        pass_fingerprint, &exact_applied);
+    (void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, files, file_count,
+                                        cls.deleted_count, pass_fingerprint, &exact_applied);
     if (exact_applied) {
         int coverage_rc = incr_refresh_coverage(store, p, project, changed_files, ci);
         if (coverage_rc != CBM_STORE_OK) {
@@ -3074,7 +3100,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
                                        excluded_dirs, excluded_count);
         cbm_pipeline_set_pkgmap(incremental_pkgmap);
         ctx.pkgmap_preseeded = true;
-        pipeline_rc = run_extract_resolve(&ctx, changed_files, ci);
+        pipeline_rc = run_extract_resolve(&ctx, changed_files, ci, files, file_count);
         ctx.pkgmap_preseeded = false;
         if (cbm_pipeline_get_pkgmap() == incremental_pkgmap) {
             cbm_pipeline_set_pkgmap(NULL);

src/pipeline/pipeline_internal.h

@@ -1089,6 +1089,11 @@ typedef struct {
     size_t retain_per_file_max_bytes;
 } cbm_parallel_extract_opts_t;
 
+/* Shared ObjectScript include context for sequential, parallel, and
+ * incremental extraction. The caller owns the returned table. */
+CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count,
+                                                const char *repo_path);
+
 int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
                             CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
                             int worker_count, const cbm_parallel_extract_opts_t *opts);

tests/test_extraction.c

@@ -4878,6 +4878,7 @@ TEST(objectscript_macro_expand_local) {
     ASSERT_NOT_NULL(r);
     ASSERT_FALSE(r->has_error);
     ASSERT(has_call(r, "MyApp.Utils.Validate"));
+    ASSERT(has_call_enclosing(r, "MyApp.Utils.Validate", "MyApp.Caller.Run", NULL));
     cbm_free_result(r);
     cbm_arena_destroy(&arena);
     PASS();

tests/test_pipeline.c

@@ -14071,6 +14071,208 @@ TEST(incremental_cross_lsp_language_matrix_matches_fresh_rebuild) {
     PASS();
 }
 
+/* ObjectScript calls can be introduced by $$$ macros declared in an unchanged
+ * .inc file. Incremental extraction must therefore use the same repository-wide
+ * macro context as a fresh build, even when only the consuming .cls changed.
+ * Return through one cleanup block so a red assertion never leaks its store. */
+static int run_incremental_objectscript_macro_oracle(char *err, size_t err_sz) {
+    int rc = -1;
+    cbm_config_t *cfg = NULL;
+    cbm_pipeline_t *pipeline = NULL;
+    cbm_store_t *store = NULL;
+    char *project = NULL;
+    bool initial_call = false;
+    bool incremental_call = false;
+    char *created = th_mktempdir("cbm_incr_objectscript_macro");
+    if (!created) {
+        snprintf(err, err_sz, "ObjectScript fixture directory creation failed");
+        return -1;
+    }
+    char root[CBM_PATH_MAX];
+    int n = snprintf(root, sizeof(root), "%s", created);
+    if (n <= 0 || (size_t)n >= sizeof(root)) {
+        snprintf(err, err_sz, "ObjectScript fixture path overflow");
+        th_rmtree(created);
+        return -1;
+    }
+
+    char include_path[CBM_PATH_MAX];
+    char caller_path[CBM_PATH_MAX];
+    char utils_path[CBM_PATH_MAX];
+    char db_path[CBM_PATH_MAX];
+    n = snprintf(include_path, sizeof(include_path), "%s/Macros.inc", root);
+    if (n <= 0 || (size_t)n >= sizeof(include_path)) {
+        snprintf(err, err_sz, "ObjectScript include path overflow");
+        goto cleanup;
+    }
+    n = snprintf(caller_path, sizeof(caller_path), "%s/Caller.cls", root);
+    if (n <= 0 || (size_t)n >= sizeof(caller_path)) {
+        snprintf(err, err_sz, "ObjectScript class path overflow");
+        goto cleanup;
+    }
+    n = snprintf(utils_path, sizeof(utils_path), "%s/Utils.cls", root);
+    if (n <= 0 || (size_t)n >= sizeof(utils_path)) {
+        snprintf(err, err_sz, "ObjectScript utility path overflow");
+        goto cleanup;
+    }
+    n = snprintf(db_path, sizeof(db_path), "%s/graph.db", root);
+    if (n <= 0 || (size_t)n >= sizeof(db_path)) {
+        snprintf(err, err_sz, "ObjectScript database path overflow");
+        goto cleanup;
+    }
+
+    if (th_write_file(include_path,
+                      "ROUTINE MyApp.Macros [Type=INC]\n"
+                      "#define MyCheck(%sc) ##class(MyApp.Utils).Validate(%sc)\n") != 0) {
+        snprintf(err, err_sz, "ObjectScript include write failed");
+        goto cleanup;
+    }
+    if (th_write_file(utils_path,
+                      "Class MyApp.Utils Extends %RegisteredObject\n"
+                      "{\n"
+                      "ClassMethod Validate(sc As %Status) As %Status\n"
+                      "{\n"
+                      "    Quit sc\n"
+                      "}\n"
+                      "}\n") != 0) {
+        snprintf(err, err_sz, "ObjectScript utility class write failed");
+        goto cleanup;
+    }
+    const char *caller_initial =
+        "Include Macros\n"
+        "Class MyApp.Caller Extends %RegisteredObject\n"
+        "{\n"
+        "Method Run(sc As %Status) As %Status\n"
+        "{\n"
+        "    If $$$MyCheck(sc) { Quit sc }\n"
+        "    Quit $$$OK\n"
+        "}\n"
+        "}\n";
+    const char *caller_updated =
+        "Include Macros\n"
+        "Class MyApp.Caller Extends %RegisteredObject\n"
+        "{\n"
+        "Method Run(sc As %Status) As %Status\n"
+        "{\n"
+        "    Set touched = 1\n"
+        "    If $$$MyCheck(sc) { Quit sc }\n"
+        "    Quit $$$OK\n"
+        "}\n"
+        "}\n";
+    if (th_write_file(caller_path, caller_initial) != 0) {
+        snprintf(err, err_sz, "initial ObjectScript class write failed");
+        goto cleanup;
+    }
+
+    cfg = incremental_test_config(root);
+    if (!cfg || cbm_config_set(cfg, CBM_CONFIG_INCREMENTAL_DERIVED_REFRESH,
+                               CBM_CONFIG_INCREMENTAL_DERIVED_REFRESH_EAGER) != 0) {
+        snprintf(err, err_sz, "ObjectScript incremental config setup failed");
+        goto cleanup;
+    }
+    pipeline = cbm_pipeline_new(root, db_path, CBM_MODE_FAST);
+    if (!pipeline) {
+        snprintf(err, err_sz, "initial ObjectScript pipeline allocation failed");
+        goto cleanup;
+    }
+    cbm_pipeline_apply_config(pipeline, cfg);
+    if (cbm_pipeline_run(pipeline) != 0) {
+        snprintf(err, err_sz, "initial ObjectScript full publication failed");
+        goto cleanup;
+    }
+    project = cbm_strdup(cbm_pipeline_project_name(pipeline));
+    cbm_pipeline_free(pipeline);
+    pipeline = NULL;
+    if (!project) {
+        snprintf(err, err_sz, "ObjectScript project allocation failed");
+        goto cleanup;
+    }
+
+    store = cbm_store_open_path(db_path);
+    if (!store) {
+        snprintf(err, err_sz, "initial ObjectScript store open failed");
+        goto cleanup;
+    }
+    initial_call = cross_file_call_exists(store, project, "Run", "Validate");
+    cbm_store_close(store);
+    store = NULL;
+    if (!initial_call) {
+        snprintf(err, err_sz,
+                 "full ObjectScript pipeline did not resolve the macro-supplied local call");
+        goto cleanup;
+    }
+
+    if (th_write_file(caller_path, caller_updated) != 0) {
+        snprintf(err, err_sz, "updated ObjectScript class write failed");
+        goto cleanup;
+    }
+    pipeline = cbm_pipeline_new(root, db_path, CBM_MODE_FAST);
+    if (!pipeline) {
+        snprintf(err, err_sz, "incremental ObjectScript pipeline allocation failed");
+        goto cleanup;
+    }
+    cbm_pipeline_apply_config(pipeline, cfg);
+    if (cbm_pipeline_run(pipeline) != 0) {
+        snprintf(err, err_sz, "incremental ObjectScript publication failed");
+        goto cleanup;
+    }
+    if (cbm_pipeline_publish_kind(pipeline) != CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT) {
+        snprintf(err, err_sz, "ObjectScript publication kind=%d reason=%s, expected exact",
+                 cbm_pipeline_publish_kind(pipeline),
+                 cbm_pipeline_publish_reason(pipeline) ? cbm_pipeline_publish_reason(pipeline)
+                                                       : "");
+        goto cleanup;
+    }
+    cbm_pipeline_free(pipeline);
+    pipeline = NULL;
+
+    store = cbm_store_open_path(db_path);
+    if (!store) {
+        snprintf(err, err_sz, "incremental ObjectScript store open failed");
+        goto cleanup;
+    }
+    incremental_call = cross_file_call_exists(store, project, "Run", "Validate");
+    cbm_store_close(store);
+    store = NULL;
+    if (!incremental_call) {
+        snprintf(err, err_sz,
+                 "incremental ObjectScript extraction lost an unchanged .inc macro call");
+        goto cleanup;
+    }
+
+    char diff_err[CBM_SZ_8K] = {0};
+    int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild(
+        root, db_path, project, cfg, diff_err, sizeof(diff_err));
+    if (diff_rc != 0) {
+        snprintf(err, err_sz, "%s",
+                 diff_err[0] ? diff_err
+                             : "incremental ObjectScript macro graph differed from fresh rebuild");
+        goto cleanup;
+    }
+    rc = 0;
+
+cleanup:
+    cbm_store_close(store);
+    cbm_pipeline_free(pipeline);
+    free(project);
+    cbm_config_close(cfg);
+    const char *artifact_dir = getenv("CBM_TEST_ARTIFACT_DIR");
+    if (rc != 0 && artifact_dir && artifact_dir[0] != '\0') {
+        printf("    [incremental-objectscript-artifact] %s\n", root);
+    } else {
+        th_rmtree(root);
+    }
+    return rc;
+}
+
+TEST(incremental_objectscript_unchanged_include_macro_matches_fresh_rebuild) {
+    char err[CBM_SZ_8K] = {0};
+    if (run_incremental_objectscript_macro_oracle(err, sizeof(err)) != 0) {
+        FAIL(err[0] ? err : "ObjectScript incremental macro oracle failed");
+    }
+    PASS();
+}
+
 TEST(incremental_mixed_python_rust_edits_match_fresh_rebuild) {
     if (setup_incremental_repo() != 0) {
         FAIL("setup failed");
@@ -17877,6 +18079,7 @@ SUITE(pipeline) {
     RUN_TEST(incremental_exact_python_scoped_lsp_gap_matches_full_rebuild);
     RUN_TEST(incremental_javascript_scoped_lsp_gap_reports_full_rebuild_not_cap_overflow);
     RUN_TEST(incremental_cross_lsp_language_matrix_matches_fresh_rebuild);
+    RUN_TEST(incremental_objectscript_unchanged_include_macro_matches_fresh_rebuild);
     RUN_TEST(incremental_mixed_python_rust_edits_match_fresh_rebuild);
     RUN_TEST(incremental_mixed_rust_typescript_javascript_matches_fresh_rebuild);
     RUN_TEST(incremental_exact_python_receiver_type_gap_matches_full_rebuild);