diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 8d4ea74bb..86a72b0ac 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -575,10 +575,14 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } /* Extract */ - CBMFileResult *result = cbm_extract_file_with_options( + /* The sequential definitions pass owns the result cache later reused by + * pass_calls. Pass the repository-wide ObjectScript table here; waiting + * until pass_calls is too late because a cache hit skips re-extraction + * and would retain unexpanded $$$ macro calls. */ + CBMFileResult *result = cbm_extract_file_with_options_ex( source, source_len, lang, ctx->project_name, rel, cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL /* no extra defines or include paths */, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode), ctx->macro_table, NULL); free(source); if (!result || result->has_error) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 51d314267..2c3937e92 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -1046,10 +1046,6 @@ static void log_extract_mem_stats(int worker_count) { } } -/* Forward declaration: macro table builder lives in pipeline.c (shared path). */ -CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count, - const char *repo_path); - int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, CBMFileResult **result_cache, _Atomic int64_t *shared_ids, int worker_count, const cbm_parallel_extract_opts_t *opts) { @@ -1139,9 +1135,14 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file return CBM_NOT_FOUND; } - /* ObjectScript macro table (NULL when no .inc include files present). */ - CBMMacroTable *pp_macro_table = - cbm_build_macro_table_from_files(files, file_count, ctx->repo_path); + /* Incremental callers may supply a repository-wide table because `files` + * is only the changed slice. Full indexing builds and owns its table here. */ + CBMMacroTable *owned_macro_table = NULL; + const CBMMacroTable *pp_macro_table = (const CBMMacroTable *)ctx->macro_table; + if (!pp_macro_table) { + owned_macro_table = cbm_build_macro_table_from_files(files, file_count, ctx->repo_path); + pp_macro_table = owned_macro_table; + } extract_ctx_t ec = { .files = files, @@ -1219,7 +1220,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file cbm_aligned_free(workers); free(sorted); - cbm_macro_table_free(pp_macro_table); /* ObjectScript macro table (NULL-safe) */ + cbm_macro_table_free(owned_macro_table); /* NULL when the caller owns the table */ if (atomic_load(ctx->cancelled) || atomic_load_explicit(&ec.worker_failed, memory_order_relaxed) != 0 || merge_rc != 0) { diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 10daa5663..137eb525e 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1031,7 +1031,8 @@ static void incr_release_seq_cross_arena(cbm_pipeline_ctx_t *ctx) { } /* Run parallel or sequential extract+resolve for changed files. */ -static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci) { +static int run_extract_resolve_inner(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, + int ci) { struct timespec t; /* Per-file LSP always runs. Sequential scoped increments run the reusable @@ -1167,11 +1168,8 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed } sequential_cleanup: - /* Cross-language registries own interned names borrowed by the calls, - * usages, and semantic passes. Release them only after the final - * borrower, including every injected-failure path. */ - incr_release_seq_cross_arena(ctx); - cbm_pipeline_release_objectscript_tables(ctx); + /* The outer wrapper releases cross-language registries and ObjectScript + * tables after this final borrower on both success and failure. */ if (rc != 0) { return rc; } @@ -1179,6 +1177,33 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed return 0; } +static bool incr_changed_uses_objectscript(const cbm_file_info_t *files, int count) { + for (int i = 0; files && i < count; i++) { + CBMLanguage lang = files[i].language; + if (lang == CBM_LANG_OBJECTSCRIPT_UDL || lang == CBM_LANG_OBJECTSCRIPT_ROUTINE || + lang == CBM_LANG_OBJECTSCRIPT_EXPORT) { + return true; + } + } + return false; +} + +/* A changed ObjectScript file can expand a $$$ macro from an unchanged .inc. + * Build that context from the full discovered file set, but only for + * ObjectScript increments so other languages retain O(changed-files) setup. + * This wrapper owns every table and cross-language arena on all return paths. */ +static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci, + cbm_file_info_t *all_files, int all_file_count) { + if (incr_changed_uses_objectscript(changed_files, ci)) { + ctx->macro_table = + cbm_build_macro_table_from_files(all_files, all_file_count, ctx->repo_path); + } + int rc = run_extract_resolve_inner(ctx, changed_files, ci); + incr_release_seq_cross_arena(ctx); + cbm_pipeline_release_objectscript_tables(ctx); + return rc; +} + /* Run post-extraction passes (tests, decorator tags, configlink). */ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_files, int ci, const char *project, bool refresh_global_semantic_edges) { @@ -1740,7 +1765,8 @@ static bool incr_overlay_requires_canonical_structure_publish(cbm_store_t *store static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, const char *project, cbm_file_info_t *changed_files, - int changed_count, int deleted_count, + int changed_count, cbm_file_info_t *all_files, + int all_file_count, int deleted_count, const char *pass_fingerprint, int *applied) { if (applied) { *applied = 0; @@ -1869,7 +1895,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, } } - rc = run_extract_resolve(&ctx, changed_files, changed_count); + rc = run_extract_resolve(&ctx, changed_files, changed_count, all_files, all_file_count); if (rc != 0) { cbm_pipeline_set_publish_reason(p, "overlay_extract_resolve"); cbm_log_info("incremental.overlay.fallback", "reason", "extract_resolve", "rc", @@ -2246,7 +2272,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co } CBM_PROF_END_N("incremental_exact", "2_ensure_structure", t_exact_structure, exact_count); CBM_PROF_START(t_exact_extract_resolve); - rc = run_extract_resolve(&ctx, exact_files, exact_count); + rc = run_extract_resolve(&ctx, exact_files, exact_count, all_files, all_file_count); CBM_PROF_END_N("incremental_exact", "3_extract_resolve", t_exact_extract_resolve, exact_count); if (rc != 0) { cbm_pipeline_set_publish_reason(p, "extract_resolve"); @@ -2834,8 +2860,8 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil return 0; } - (void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, cls.deleted_count, - pass_fingerprint, &exact_applied); + (void)incr_try_overlay_upsert_route(p, store, project, changed_files, ci, files, file_count, + cls.deleted_count, pass_fingerprint, &exact_applied); if (exact_applied) { int coverage_rc = incr_refresh_coverage(store, p, project, changed_files, ci); if (coverage_rc != CBM_STORE_OK) { @@ -3074,7 +3100,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil excluded_dirs, excluded_count); cbm_pipeline_set_pkgmap(incremental_pkgmap); ctx.pkgmap_preseeded = true; - pipeline_rc = run_extract_resolve(&ctx, changed_files, ci); + pipeline_rc = run_extract_resolve(&ctx, changed_files, ci, files, file_count); ctx.pkgmap_preseeded = false; if (cbm_pipeline_get_pkgmap() == incremental_pkgmap) { cbm_pipeline_set_pkgmap(NULL); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index c4d8af656..cccf59300 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -1089,6 +1089,11 @@ typedef struct { size_t retain_per_file_max_bytes; } cbm_parallel_extract_opts_t; +/* Shared ObjectScript include context for sequential, parallel, and + * incremental extraction. The caller owns the returned table. */ +CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count, + const char *repo_path); + int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, CBMFileResult **result_cache, _Atomic int64_t *shared_ids, int worker_count, const cbm_parallel_extract_opts_t *opts); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index e6e3eeed9..6aeb97a6e 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -4878,6 +4878,7 @@ TEST(objectscript_macro_expand_local) { ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_call(r, "MyApp.Utils.Validate")); + ASSERT(has_call_enclosing(r, "MyApp.Utils.Validate", "MyApp.Caller.Run", NULL)); cbm_free_result(r); cbm_arena_destroy(&arena); PASS(); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 2a847e8b7..248e1cc32 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14071,6 +14071,208 @@ TEST(incremental_cross_lsp_language_matrix_matches_fresh_rebuild) { PASS(); } +/* ObjectScript calls can be introduced by $$$ macros declared in an unchanged + * .inc file. Incremental extraction must therefore use the same repository-wide + * macro context as a fresh build, even when only the consuming .cls changed. + * Return through one cleanup block so a red assertion never leaks its store. */ +static int run_incremental_objectscript_macro_oracle(char *err, size_t err_sz) { + int rc = -1; + cbm_config_t *cfg = NULL; + cbm_pipeline_t *pipeline = NULL; + cbm_store_t *store = NULL; + char *project = NULL; + bool initial_call = false; + bool incremental_call = false; + char *created = th_mktempdir("cbm_incr_objectscript_macro"); + if (!created) { + snprintf(err, err_sz, "ObjectScript fixture directory creation failed"); + return -1; + } + char root[CBM_PATH_MAX]; + int n = snprintf(root, sizeof(root), "%s", created); + if (n <= 0 || (size_t)n >= sizeof(root)) { + snprintf(err, err_sz, "ObjectScript fixture path overflow"); + th_rmtree(created); + return -1; + } + + char include_path[CBM_PATH_MAX]; + char caller_path[CBM_PATH_MAX]; + char utils_path[CBM_PATH_MAX]; + char db_path[CBM_PATH_MAX]; + n = snprintf(include_path, sizeof(include_path), "%s/Macros.inc", root); + if (n <= 0 || (size_t)n >= sizeof(include_path)) { + snprintf(err, err_sz, "ObjectScript include path overflow"); + goto cleanup; + } + n = snprintf(caller_path, sizeof(caller_path), "%s/Caller.cls", root); + if (n <= 0 || (size_t)n >= sizeof(caller_path)) { + snprintf(err, err_sz, "ObjectScript class path overflow"); + goto cleanup; + } + n = snprintf(utils_path, sizeof(utils_path), "%s/Utils.cls", root); + if (n <= 0 || (size_t)n >= sizeof(utils_path)) { + snprintf(err, err_sz, "ObjectScript utility path overflow"); + goto cleanup; + } + n = snprintf(db_path, sizeof(db_path), "%s/graph.db", root); + if (n <= 0 || (size_t)n >= sizeof(db_path)) { + snprintf(err, err_sz, "ObjectScript database path overflow"); + goto cleanup; + } + + if (th_write_file(include_path, + "ROUTINE MyApp.Macros [Type=INC]\n" + "#define MyCheck(%sc) ##class(MyApp.Utils).Validate(%sc)\n") != 0) { + snprintf(err, err_sz, "ObjectScript include write failed"); + goto cleanup; + } + if (th_write_file(utils_path, + "Class MyApp.Utils Extends %RegisteredObject\n" + "{\n" + "ClassMethod Validate(sc As %Status) As %Status\n" + "{\n" + " Quit sc\n" + "}\n" + "}\n") != 0) { + snprintf(err, err_sz, "ObjectScript utility class write failed"); + goto cleanup; + } + const char *caller_initial = + "Include Macros\n" + "Class MyApp.Caller Extends %RegisteredObject\n" + "{\n" + "Method Run(sc As %Status) As %Status\n" + "{\n" + " If $$$MyCheck(sc) { Quit sc }\n" + " Quit $$$OK\n" + "}\n" + "}\n"; + const char *caller_updated = + "Include Macros\n" + "Class MyApp.Caller Extends %RegisteredObject\n" + "{\n" + "Method Run(sc As %Status) As %Status\n" + "{\n" + " Set touched = 1\n" + " If $$$MyCheck(sc) { Quit sc }\n" + " Quit $$$OK\n" + "}\n" + "}\n"; + if (th_write_file(caller_path, caller_initial) != 0) { + snprintf(err, err_sz, "initial ObjectScript class write failed"); + goto cleanup; + } + + cfg = incremental_test_config(root); + if (!cfg || cbm_config_set(cfg, CBM_CONFIG_INCREMENTAL_DERIVED_REFRESH, + CBM_CONFIG_INCREMENTAL_DERIVED_REFRESH_EAGER) != 0) { + snprintf(err, err_sz, "ObjectScript incremental config setup failed"); + goto cleanup; + } + pipeline = cbm_pipeline_new(root, db_path, CBM_MODE_FAST); + if (!pipeline) { + snprintf(err, err_sz, "initial ObjectScript pipeline allocation failed"); + goto cleanup; + } + cbm_pipeline_apply_config(pipeline, cfg); + if (cbm_pipeline_run(pipeline) != 0) { + snprintf(err, err_sz, "initial ObjectScript full publication failed"); + goto cleanup; + } + project = cbm_strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + pipeline = NULL; + if (!project) { + snprintf(err, err_sz, "ObjectScript project allocation failed"); + goto cleanup; + } + + store = cbm_store_open_path(db_path); + if (!store) { + snprintf(err, err_sz, "initial ObjectScript store open failed"); + goto cleanup; + } + initial_call = cross_file_call_exists(store, project, "Run", "Validate"); + cbm_store_close(store); + store = NULL; + if (!initial_call) { + snprintf(err, err_sz, + "full ObjectScript pipeline did not resolve the macro-supplied local call"); + goto cleanup; + } + + if (th_write_file(caller_path, caller_updated) != 0) { + snprintf(err, err_sz, "updated ObjectScript class write failed"); + goto cleanup; + } + pipeline = cbm_pipeline_new(root, db_path, CBM_MODE_FAST); + if (!pipeline) { + snprintf(err, err_sz, "incremental ObjectScript pipeline allocation failed"); + goto cleanup; + } + cbm_pipeline_apply_config(pipeline, cfg); + if (cbm_pipeline_run(pipeline) != 0) { + snprintf(err, err_sz, "incremental ObjectScript publication failed"); + goto cleanup; + } + if (cbm_pipeline_publish_kind(pipeline) != CBM_PIPELINE_PUBLISH_INCREMENTAL_EXACT) { + snprintf(err, err_sz, "ObjectScript publication kind=%d reason=%s, expected exact", + cbm_pipeline_publish_kind(pipeline), + cbm_pipeline_publish_reason(pipeline) ? cbm_pipeline_publish_reason(pipeline) + : ""); + goto cleanup; + } + cbm_pipeline_free(pipeline); + pipeline = NULL; + + store = cbm_store_open_path(db_path); + if (!store) { + snprintf(err, err_sz, "incremental ObjectScript store open failed"); + goto cleanup; + } + incremental_call = cross_file_call_exists(store, project, "Run", "Validate"); + cbm_store_close(store); + store = NULL; + if (!incremental_call) { + snprintf(err, err_sz, + "incremental ObjectScript extraction lost an unchanged .inc macro call"); + goto cleanup; + } + + char diff_err[CBM_SZ_8K] = {0}; + int diff_rc = pipeline_compare_current_db_to_fresh_fast_rebuild( + root, db_path, project, cfg, diff_err, sizeof(diff_err)); + if (diff_rc != 0) { + snprintf(err, err_sz, "%s", + diff_err[0] ? diff_err + : "incremental ObjectScript macro graph differed from fresh rebuild"); + goto cleanup; + } + rc = 0; + +cleanup: + cbm_store_close(store); + cbm_pipeline_free(pipeline); + free(project); + cbm_config_close(cfg); + const char *artifact_dir = getenv("CBM_TEST_ARTIFACT_DIR"); + if (rc != 0 && artifact_dir && artifact_dir[0] != '\0') { + printf(" [incremental-objectscript-artifact] %s\n", root); + } else { + th_rmtree(root); + } + return rc; +} + +TEST(incremental_objectscript_unchanged_include_macro_matches_fresh_rebuild) { + char err[CBM_SZ_8K] = {0}; + if (run_incremental_objectscript_macro_oracle(err, sizeof(err)) != 0) { + FAIL(err[0] ? err : "ObjectScript incremental macro oracle failed"); + } + PASS(); +} + TEST(incremental_mixed_python_rust_edits_match_fresh_rebuild) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); @@ -17877,6 +18079,7 @@ SUITE(pipeline) { RUN_TEST(incremental_exact_python_scoped_lsp_gap_matches_full_rebuild); RUN_TEST(incremental_javascript_scoped_lsp_gap_reports_full_rebuild_not_cap_overflow); RUN_TEST(incremental_cross_lsp_language_matrix_matches_fresh_rebuild); + RUN_TEST(incremental_objectscript_unchanged_include_macro_matches_fresh_rebuild); RUN_TEST(incremental_mixed_python_rust_edits_match_fresh_rebuild); RUN_TEST(incremental_mixed_rust_typescript_javascript_matches_fresh_rebuild); RUN_TEST(incremental_exact_python_receiver_type_gap_matches_full_rebuild);