From cc345f05193b1fb39f8fa44ca6152cf0a5c2f9ec Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 13 Jul 2026 01:35:21 -0400 Subject: [PATCH] fix(indexing): configure bounded parse deadlines Expose extraction deadlines through the existing configuration registry with a documented default and explicit lower and upper bounds. Propagate the selected deadline through sequential, parallel, incremental, semantic, usage, call, and infrastructure extraction paths so every route applies the same policy. Keep sanitizer-specific test headroom in test configuration rather than production constants, and verify default registration plus both clamps. Extraction failures continue to fail closed, preserving the previously published database instead of persisting partial graphs. Validated with the 352-test pipeline suite, 161-test incremental suite including exact 15,956-node/55,790-edge equality, 31-test parallel suite, focused TSan worker-pool and parallel suites, and a complete TSan run with no race report. The complete runner reached 6,418 passes and 25 pre-existing red graph-capability failures, which remain the next tracked blocker. Signed-off-by: Andrew Hundt --- src/cli/cli.c | 8 ++++++++ src/pipeline/pass_calls.c | 7 ++++--- src/pipeline/pass_definitions.c | 4 ++-- src/pipeline/pass_k8s.c | 15 ++++++++------- src/pipeline/pass_parallel.c | 6 ++++-- src/pipeline/pass_semantic.c | 7 ++++--- src/pipeline/pass_usages.c | 5 +++-- src/pipeline/pipeline.c | 16 ++++++++++++++++ src/pipeline/pipeline.h | 5 +++++ src/pipeline/pipeline_incremental.c | 3 +++ src/pipeline/pipeline_internal.h | 8 ++++++++ tests/test_incremental.c | 7 +++++++ tests/test_pipeline.c | 21 +++++++++++++++++++++ 13 files changed, 93 insertions(+), 19 deletions(-) diff --git a/src/cli/cli.c b/src/cli/cli.c index bc2d13189..08b1be678 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -2860,6 +2860,9 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { /* ── Config registry ──────────────────────────────────────────── */ +/* Hand-wrapped for readable help text; automatic formatting makes this table + * substantially narrower and churns unrelated entries. */ +// clang-format off const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { /* ── Indexing ── */ {"auto_index", "true", "CBM_AUTO_INDEX", "Indexing", @@ -2871,6 +2874,10 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { "0-10000000", "Protects against accidentally indexing huge monorepos. Raise for large codebases. " "Set 0 to disable the limit and always index regardless of repo size."}, + {CBM_CONFIG_EXTRACT_TIMEOUT_MS, CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT, NULL, "Indexing", + "Per-file tree-sitter parse deadline in milliseconds", + "100-120000", + "Bounds pathological parses. A timeout fails the indexing transaction and preserves the prior database; raise only for measured large-file workloads or instrumented tests."}, {"reindex_on_startup", "false", "CBM_REINDEX_ON_STARTUP", "Indexing", "Re-index stale projects when server starts", "true|false", @@ -3225,6 +3232,7 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { "Set 0 for unlimited if you need complete large-package analysis."}, {NULL, NULL, NULL, NULL, NULL, NULL, NULL} /* sentinel */ }; +// clang-format on /* Get config value with env var override priority: env > db > default. * Looks up the registry entry for the key to find the env var name. */ diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 1bbc107e0..0fd3e879a 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -410,9 +410,10 @@ static CBMFileResult *calls_get_or_extract(cbm_pipeline_ctx_t *ctx, int idx, if (!src) { return NULL; } - CBMFileResult *r = cbm_extract_file_with_options( - src, slen, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *r = + cbm_extract_file_with_options(src, slen, fi->language, ctx->project_name, fi->rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(src); if (r) { *owned = true; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 0b27e195b..a33d043c6 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -462,8 +462,8 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t /* Extract */ CBMFileResult *result = cbm_extract_file_with_options( - source, source_len, lang, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, - NULL /* no extra defines or include paths */, + source, source_len, lang, ctx->project_name, rel, cbm_pipeline_ctx_extract_timeout(ctx), + NULL, NULL /* no extra defines or include paths */, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 4d48cb7e3..8ef3609a4 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -108,10 +108,10 @@ static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const ch int src_len = 0; char *source = k8s_read_file(path, &src_len); if (source) { - res = cbm_extract_file_with_options( - source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + res = cbm_extract_file_with_options(source, src_len, CBM_LANG_KUSTOMIZE, + ctx->project_name, rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); allocated = true; } @@ -377,9 +377,10 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const (void)path; /* retained for symmetry; source is always provided now */ int resource_count = 0; - CBMFileResult *res = cbm_extract_file_with_options( - source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *res = + cbm_extract_file_with_options(source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); if (!res) { return; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index bb4f5436f..c3b64b02e 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -421,6 +421,7 @@ typedef struct { cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */ _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */ bool extract_macros; + int64_t extract_timeout_micros; } extract_ctx_t; /* Insert one definition node (and its route if present) into the local gbuf. */ @@ -524,8 +525,8 @@ static void extract_worker(int worker_id, void *ctx_ptr) { uint64_t file_t0 = extract_now_ns(); CBMFileResult *result = cbm_extract_file_with_options( - source, source_len, fi->language, ec->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, - NULL, NULL, ec->extract_macros); + source, source_len, fi->language, ec->project_name, fi->rel_path, + ec->extract_timeout_micros, NULL, NULL, ec->extract_macros); uint64_t file_elapsed_ms = (extract_now_ns() - file_t0) / PP_USEC_PER_MS; @@ -717,6 +718,7 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .cancelled = ctx->cancelled, .pkg_entries = pkg_entries, .extract_macros = cbm_pipeline_mode_extracts_macro_nodes(ctx->mode), + .extract_timeout_micros = cbm_pipeline_ctx_extract_timeout(ctx), }; atomic_init(&ec.next_worker_id, 0); atomic_init(&ec.next_file_idx, 0); diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 263092fbb..a045e1ed8 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -381,9 +381,10 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx, if (!source) { return NULL; } - CBMFileResult *r = cbm_extract_file_with_options( - source, source_len, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, - NULL, NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *r = + cbm_extract_file_with_options(source, source_len, fi->language, ctx->project_name, + fi->rel_path, cbm_pipeline_ctx_extract_timeout(ctx), NULL, + NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); if (r) { *owned = true; diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 64bc7eb78..ba2057cfa 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -232,8 +232,9 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil continue; } result = cbm_extract_file_with_options( - source, source_len, files[i].language, ctx->project_name, rel, CBM_EXTRACT_BUDGET, - NULL, NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + source, source_len, files[i].language, ctx->project_name, rel, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); if (!result) { errors++; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index be47c470b..b2abf9a88 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -112,6 +112,7 @@ struct cbm_pipeline { double semantic_threshold; double githistory_min_coupling; double lsp_confidence_floor; + int64_t extract_timeout_micros; cbm_incremental_reindex_policy_t incremental_reindex; cbm_overlay_publish_policy_t overlay_publish; cbm_incremental_derived_refresh_policy_t incremental_derived_refresh; @@ -205,6 +206,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->semantic_threshold = 0.0; p->githistory_min_coupling = 0.0; p->lsp_confidence_floor = 0.0; + p->extract_timeout_micros = CBM_EXTRACT_BUDGET; p->incremental_reindex = CBM_INCREMENTAL_REINDEX_OFF; p->overlay_publish = CBM_OVERLAY_PUBLISH_OFF; p->incremental_derived_refresh = CBM_INCREMENTAL_DERIVED_REFRESH_STALE_ON_INCREMENTAL; @@ -346,6 +348,15 @@ void cbm_pipeline_apply_config(cbm_pipeline_t *p, cbm_config_t *cfg) { cbm_pipeline_set_lsp_confidence_floor(p, lsp_floor); } + int extract_timeout_ms = cbm_config_get_int(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, + CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS); + if (extract_timeout_ms < CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS) { + extract_timeout_ms = CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS; + } else if (extract_timeout_ms > CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS) { + extract_timeout_ms = CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS; + } + p->extract_timeout_micros = (int64_t)extract_timeout_ms * 1000; + const char *incremental = cbm_config_get(cfg, CBM_CONFIG_INCREMENTAL_REINDEX, CBM_CONFIG_INCREMENTAL_REINDEX_OFF); if (incremental && strcmp(incremental, CBM_CONFIG_INCREMENTAL_REINDEX_ALWAYS) == 0) { @@ -405,6 +416,10 @@ double cbm_pipeline_lsp_confidence_floor(const cbm_pipeline_t *p) { return p ? p->lsp_confidence_floor : 0.0; } +int64_t cbm_pipeline_extract_timeout_micros(const cbm_pipeline_t *p) { + return p ? p->extract_timeout_micros : CBM_EXTRACT_BUDGET; +} + int cbm_pipeline_exact_max_changed_paths(const cbm_pipeline_t *p) { return p ? p->exact_delta_max_changed_paths : CBM_PIPELINE_EXACT_DELTA_DEFAULT_MAX_CHANGED_PATHS; @@ -1765,6 +1780,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { .semantic_threshold = p->semantic_threshold, .githistory_min_coupling = p->githistory_min_coupling, .lsp_confidence_floor = p->lsp_confidence_floor, + .extract_timeout_micros = p->extract_timeout_micros, .path_aliases = path_aliases, }; diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 98fdc946a..72871b4d2 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -105,6 +105,11 @@ void cbm_pipeline_set_flush_store(cbm_pipeline_t *p, cbm_store_t *store); #define CBM_CONFIG_SEMANTIC_THRESHOLD "semantic_threshold" #define CBM_CONFIG_GITHISTORY_MIN_COUPLING "githistory_min_coupling" #define CBM_CONFIG_LSP_CONFIDENCE_FLOOR "lsp_confidence_floor" +#define CBM_CONFIG_EXTRACT_TIMEOUT_MS "extract_timeout_ms" +#define CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS 5000 +#define CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT "5000" +#define CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS 100 +#define CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS 120000 #define CBM_CONFIG_INCREMENTAL_REINDEX "incremental_reindex" #define CBM_CONFIG_INCREMENTAL_REINDEX_OFF "off" #define CBM_CONFIG_INCREMENTAL_REINDEX_FAST "fast" diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index c83f4b56d..51c212ac7 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1732,6 +1732,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, .pkgmap_preseeded = true, .result_cache = result_cache, @@ -2096,6 +2097,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, .pkgmap_preseeded = true, .result_cache = result_cache, @@ -2751,6 +2753,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, }; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 775ef6ef1..c177824d6 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -404,6 +404,7 @@ typedef struct { double semantic_threshold; /* <=0 uses semantic default 0.75 */ double githistory_min_coupling; /* <=0 uses git-history default 0.3 */ double lsp_confidence_floor; /* <=0 uses LSP default 0.6 */ + int64_t extract_timeout_micros; /* bounded tree-sitter parse deadline */ /* Extraction result cache (sequential pipeline optimization). * When non-NULL, pass_definitions stores results here instead of freeing, @@ -437,6 +438,13 @@ typedef struct { int store_backed_lsp_scope_cap; } cbm_pipeline_ctx_t; +static inline int64_t cbm_pipeline_ctx_extract_timeout(const cbm_pipeline_ctx_t *ctx) { + return ctx && ctx->extract_timeout_micros > 0 ? ctx->extract_timeout_micros + : CBM_EXTRACT_BUDGET; +} + +int64_t cbm_pipeline_extract_timeout_micros(const cbm_pipeline_t *p); + static inline bool cbm_pipeline_mode_builds_global_semantic_edges(int mode) { return mode == CBM_MODE_FULL || mode == CBM_MODE_MODERATE; } diff --git a/tests/test_incremental.c b/tests/test_incremental.c index a3ada7771..550e964ac 100644 --- a/tests/test_incremental.c +++ b/tests/test_incremental.c @@ -60,6 +60,7 @@ enum { INCR_ACCURACY_EDGE_TOLERANCE = 50, INCR_ACCURACY_CALL_TOLERANCE = 2, INCR_FULL_INDEX_MAX_RSS_DELTA_MB = 2048, + INCR_INSTRUMENTED_TIMEOUT_MULTIPLIER = 4, }; static const char *INCR_TEST_ARTIFACT_ENV = "CBM_TEST_ARTIFACT_DIR"; @@ -592,6 +593,12 @@ static int incremental_setup(void) { return -1; } cbm_config_set(g_cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "always"); + if (incr_memory_instrumentation_active()) { + char timeout_ms[CBM_SZ_32]; + snprintf(timeout_ms, sizeof(timeout_ms), "%d", + CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS * INCR_INSTRUMENTED_TIMEOUT_MULTIPLIER); + cbm_config_set(g_cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, timeout_ms); + } cbm_mcp_server_set_config(g_srv, g_cfg); g_rss_before_full = cbm_mem_rss(); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index abcf52d5a..0c4c189d7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14608,6 +14608,7 @@ TEST(pipeline_apply_config_sets_all_thresholds) { ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_SEMANTIC_THRESHOLD, "0.76"), 0); ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_GITHISTORY_MIN_COUPLING, "0.31"), 0); ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_LSP_CONFIDENCE_FLOOR, "0.61"), 0); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "17000"), 0); char max_changed[CBM_SZ_32]; char max_affected[CBM_SZ_32]; int n = snprintf(max_changed, sizeof(max_changed), "%d", PIPELINE_TEST_EXACT_MAX_CHANGED); @@ -14633,11 +14634,21 @@ TEST(pipeline_apply_config_sets_all_thresholds) { ASSERT_TRUE(cbm_pipeline_githistory_min_coupling(p) < 0.32); ASSERT_TRUE(cbm_pipeline_lsp_confidence_floor(p) > 0.60); ASSERT_TRUE(cbm_pipeline_lsp_confidence_floor(p) < 0.62); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), 17000000); ASSERT_EQ(cbm_pipeline_exact_max_changed_paths(p), PIPELINE_TEST_EXACT_MAX_CHANGED); ASSERT_EQ(cbm_pipeline_exact_max_affected_paths(p), PIPELINE_TEST_EXACT_MAX_AFFECTED); ASSERT_TRUE(cbm_pipeline_incremental_derived_refresh_stale_on_exact(p)); ASSERT_TRUE(cbm_pipeline_incremental_derived_refresh_stale_on_incremental(p)); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "1"), 0); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), + (int64_t)CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS * 1000); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "999999"), 0); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), + (int64_t)CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS * 1000); + cbm_pipeline_free(p); cbm_config_close(cfg); rm_rf(tmpdir); @@ -14900,6 +14911,15 @@ TEST(config_registry_includes_incremental_reindex_policy) { PASS(); } +TEST(config_registry_includes_extract_timeout) { + const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_EXTRACT_TIMEOUT_MS); + ASSERT_NOT_NULL(entry); + ASSERT_STR_EQ(entry->default_val, CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT); + ASSERT_STR_EQ(entry->category, "Indexing"); + ASSERT_STR_EQ(entry->range, "100-120000"); + PASS(); +} + TEST(config_registry_includes_overlay_publish_policy) { const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_OVERLAY_PUBLISH); ASSERT_NOT_NULL(entry); @@ -15670,6 +15690,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_batch_rejects_invalid_token_stride); RUN_TEST(config_registry_includes_mcp_timeout_knobs); RUN_TEST(config_registry_includes_incremental_reindex_policy); + RUN_TEST(config_registry_includes_extract_timeout); RUN_TEST(config_registry_includes_overlay_publish_policy); RUN_TEST(config_registry_includes_overlay_compaction_policy); RUN_TEST(config_registry_includes_incremental_exact_frontier_caps);