diff --git a/src/cli/cli.c b/src/cli/cli.c index bc2d13189..08b1be678 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -2860,6 +2860,9 @@ int cbm_config_delete(cbm_config_t *cfg, const char *key) { /* ── Config registry ──────────────────────────────────────────── */ +/* Hand-wrapped for readable help text; automatic formatting makes this table + * substantially narrower and churns unrelated entries. */ +// clang-format off const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { /* ── Indexing ── */ {"auto_index", "true", "CBM_AUTO_INDEX", "Indexing", @@ -2871,6 +2874,10 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { "0-10000000", "Protects against accidentally indexing huge monorepos. Raise for large codebases. " "Set 0 to disable the limit and always index regardless of repo size."}, + {CBM_CONFIG_EXTRACT_TIMEOUT_MS, CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT, NULL, "Indexing", + "Per-file tree-sitter parse deadline in milliseconds", + "100-120000", + "Bounds pathological parses. A timeout fails the indexing transaction and preserves the prior database; raise only for measured large-file workloads or instrumented tests."}, {"reindex_on_startup", "false", "CBM_REINDEX_ON_STARTUP", "Indexing", "Re-index stale projects when server starts", "true|false", @@ -3225,6 +3232,7 @@ const cbm_config_entry_t CBM_CONFIG_REGISTRY[] = { "Set 0 for unlimited if you need complete large-package analysis."}, {NULL, NULL, NULL, NULL, NULL, NULL, NULL} /* sentinel */ }; +// clang-format on /* Get config value with env var override priority: env > db > default. * Looks up the registry entry for the key to find the env var name. */ diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index 1bbc107e0..0fd3e879a 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -410,9 +410,10 @@ static CBMFileResult *calls_get_or_extract(cbm_pipeline_ctx_t *ctx, int idx, if (!src) { return NULL; } - CBMFileResult *r = cbm_extract_file_with_options( - src, slen, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *r = + cbm_extract_file_with_options(src, slen, fi->language, ctx->project_name, fi->rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(src); if (r) { *owned = true; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 0b27e195b..a33d043c6 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -462,8 +462,8 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t /* Extract */ CBMFileResult *result = cbm_extract_file_with_options( - source, source_len, lang, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, - NULL /* no extra defines or include paths */, + source, source_len, lang, ctx->project_name, rel, cbm_pipeline_ctx_extract_timeout(ctx), + NULL, NULL /* no extra defines or include paths */, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 4d48cb7e3..8ef3609a4 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -108,10 +108,10 @@ static void handle_kustomize(cbm_pipeline_ctx_t *ctx, const char *path, const ch int src_len = 0; char *source = k8s_read_file(path, &src_len); if (source) { - res = cbm_extract_file_with_options( - source, src_len, CBM_LANG_KUSTOMIZE, ctx->project_name, rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + res = cbm_extract_file_with_options(source, src_len, CBM_LANG_KUSTOMIZE, + ctx->project_name, rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); allocated = true; } @@ -377,9 +377,10 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *path, const (void)path; /* retained for symmetry; source is always provided now */ int resource_count = 0; - CBMFileResult *res = cbm_extract_file_with_options( - source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, - cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *res = + cbm_extract_file_with_options(source, src_len, CBM_LANG_K8S, ctx->project_name, rel_path, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); if (!res) { return; } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index bb4f5436f..c3b64b02e 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -421,6 +421,7 @@ typedef struct { cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */ _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */ bool extract_macros; + int64_t extract_timeout_micros; } extract_ctx_t; /* Insert one definition node (and its route if present) into the local gbuf. */ @@ -524,8 +525,8 @@ static void extract_worker(int worker_id, void *ctx_ptr) { uint64_t file_t0 = extract_now_ns(); CBMFileResult *result = cbm_extract_file_with_options( - source, source_len, fi->language, ec->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, - NULL, NULL, ec->extract_macros); + source, source_len, fi->language, ec->project_name, fi->rel_path, + ec->extract_timeout_micros, NULL, NULL, ec->extract_macros); uint64_t file_elapsed_ms = (extract_now_ns() - file_t0) / PP_USEC_PER_MS; @@ -717,6 +718,7 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, .cancelled = ctx->cancelled, .pkg_entries = pkg_entries, .extract_macros = cbm_pipeline_mode_extracts_macro_nodes(ctx->mode), + .extract_timeout_micros = cbm_pipeline_ctx_extract_timeout(ctx), }; atomic_init(&ec.next_worker_id, 0); atomic_init(&ec.next_file_idx, 0); diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 263092fbb..a045e1ed8 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -381,9 +381,10 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx, if (!source) { return NULL; } - CBMFileResult *r = cbm_extract_file_with_options( - source, source_len, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, - NULL, NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + CBMFileResult *r = + cbm_extract_file_with_options(source, source_len, fi->language, ctx->project_name, + fi->rel_path, cbm_pipeline_ctx_extract_timeout(ctx), NULL, + NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); if (r) { *owned = true; diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 64bc7eb78..ba2057cfa 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -232,8 +232,9 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil continue; } result = cbm_extract_file_with_options( - source, source_len, files[i].language, ctx->project_name, rel, CBM_EXTRACT_BUDGET, - NULL, NULL, cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); + source, source_len, files[i].language, ctx->project_name, rel, + cbm_pipeline_ctx_extract_timeout(ctx), NULL, NULL, + cbm_pipeline_mode_extracts_macro_nodes(ctx->mode)); free(source); if (!result) { errors++; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index be47c470b..b2abf9a88 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -112,6 +112,7 @@ struct cbm_pipeline { double semantic_threshold; double githistory_min_coupling; double lsp_confidence_floor; + int64_t extract_timeout_micros; cbm_incremental_reindex_policy_t incremental_reindex; cbm_overlay_publish_policy_t overlay_publish; cbm_incremental_derived_refresh_policy_t incremental_derived_refresh; @@ -205,6 +206,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->semantic_threshold = 0.0; p->githistory_min_coupling = 0.0; p->lsp_confidence_floor = 0.0; + p->extract_timeout_micros = CBM_EXTRACT_BUDGET; p->incremental_reindex = CBM_INCREMENTAL_REINDEX_OFF; p->overlay_publish = CBM_OVERLAY_PUBLISH_OFF; p->incremental_derived_refresh = CBM_INCREMENTAL_DERIVED_REFRESH_STALE_ON_INCREMENTAL; @@ -346,6 +348,15 @@ void cbm_pipeline_apply_config(cbm_pipeline_t *p, cbm_config_t *cfg) { cbm_pipeline_set_lsp_confidence_floor(p, lsp_floor); } + int extract_timeout_ms = cbm_config_get_int(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, + CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS); + if (extract_timeout_ms < CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS) { + extract_timeout_ms = CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS; + } else if (extract_timeout_ms > CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS) { + extract_timeout_ms = CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS; + } + p->extract_timeout_micros = (int64_t)extract_timeout_ms * 1000; + const char *incremental = cbm_config_get(cfg, CBM_CONFIG_INCREMENTAL_REINDEX, CBM_CONFIG_INCREMENTAL_REINDEX_OFF); if (incremental && strcmp(incremental, CBM_CONFIG_INCREMENTAL_REINDEX_ALWAYS) == 0) { @@ -405,6 +416,10 @@ double cbm_pipeline_lsp_confidence_floor(const cbm_pipeline_t *p) { return p ? p->lsp_confidence_floor : 0.0; } +int64_t cbm_pipeline_extract_timeout_micros(const cbm_pipeline_t *p) { + return p ? p->extract_timeout_micros : CBM_EXTRACT_BUDGET; +} + int cbm_pipeline_exact_max_changed_paths(const cbm_pipeline_t *p) { return p ? p->exact_delta_max_changed_paths : CBM_PIPELINE_EXACT_DELTA_DEFAULT_MAX_CHANGED_PATHS; @@ -1765,6 +1780,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { .semantic_threshold = p->semantic_threshold, .githistory_min_coupling = p->githistory_min_coupling, .lsp_confidence_floor = p->lsp_confidence_floor, + .extract_timeout_micros = p->extract_timeout_micros, .path_aliases = path_aliases, }; diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 98fdc946a..72871b4d2 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -105,6 +105,11 @@ void cbm_pipeline_set_flush_store(cbm_pipeline_t *p, cbm_store_t *store); #define CBM_CONFIG_SEMANTIC_THRESHOLD "semantic_threshold" #define CBM_CONFIG_GITHISTORY_MIN_COUPLING "githistory_min_coupling" #define CBM_CONFIG_LSP_CONFIDENCE_FLOOR "lsp_confidence_floor" +#define CBM_CONFIG_EXTRACT_TIMEOUT_MS "extract_timeout_ms" +#define CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS 5000 +#define CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT "5000" +#define CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS 100 +#define CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS 120000 #define CBM_CONFIG_INCREMENTAL_REINDEX "incremental_reindex" #define CBM_CONFIG_INCREMENTAL_REINDEX_OFF "off" #define CBM_CONFIG_INCREMENTAL_REINDEX_FAST "fast" diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index c83f4b56d..51c212ac7 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -1732,6 +1732,7 @@ static int incr_try_overlay_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, .pkgmap_preseeded = true, .result_cache = result_cache, @@ -2096,6 +2097,7 @@ static int incr_try_exact_upsert_route(cbm_pipeline_t *p, cbm_store_t *store, co .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, .pkgmap_preseeded = true, .result_cache = result_cache, @@ -2751,6 +2753,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil .semantic_threshold = cbm_pipeline_semantic_threshold(p), .githistory_min_coupling = cbm_pipeline_githistory_min_coupling(p), .lsp_confidence_floor = cbm_pipeline_lsp_confidence_floor(p), + .extract_timeout_micros = cbm_pipeline_extract_timeout_micros(p), .path_aliases = path_aliases, }; diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 775ef6ef1..c177824d6 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -404,6 +404,7 @@ typedef struct { double semantic_threshold; /* <=0 uses semantic default 0.75 */ double githistory_min_coupling; /* <=0 uses git-history default 0.3 */ double lsp_confidence_floor; /* <=0 uses LSP default 0.6 */ + int64_t extract_timeout_micros; /* bounded tree-sitter parse deadline */ /* Extraction result cache (sequential pipeline optimization). * When non-NULL, pass_definitions stores results here instead of freeing, @@ -437,6 +438,13 @@ typedef struct { int store_backed_lsp_scope_cap; } cbm_pipeline_ctx_t; +static inline int64_t cbm_pipeline_ctx_extract_timeout(const cbm_pipeline_ctx_t *ctx) { + return ctx && ctx->extract_timeout_micros > 0 ? ctx->extract_timeout_micros + : CBM_EXTRACT_BUDGET; +} + +int64_t cbm_pipeline_extract_timeout_micros(const cbm_pipeline_t *p); + static inline bool cbm_pipeline_mode_builds_global_semantic_edges(int mode) { return mode == CBM_MODE_FULL || mode == CBM_MODE_MODERATE; } diff --git a/tests/test_incremental.c b/tests/test_incremental.c index a3ada7771..550e964ac 100644 --- a/tests/test_incremental.c +++ b/tests/test_incremental.c @@ -60,6 +60,7 @@ enum { INCR_ACCURACY_EDGE_TOLERANCE = 50, INCR_ACCURACY_CALL_TOLERANCE = 2, INCR_FULL_INDEX_MAX_RSS_DELTA_MB = 2048, + INCR_INSTRUMENTED_TIMEOUT_MULTIPLIER = 4, }; static const char *INCR_TEST_ARTIFACT_ENV = "CBM_TEST_ARTIFACT_DIR"; @@ -592,6 +593,12 @@ static int incremental_setup(void) { return -1; } cbm_config_set(g_cfg, CBM_CONFIG_INCREMENTAL_REINDEX, "always"); + if (incr_memory_instrumentation_active()) { + char timeout_ms[CBM_SZ_32]; + snprintf(timeout_ms, sizeof(timeout_ms), "%d", + CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT_MS * INCR_INSTRUMENTED_TIMEOUT_MULTIPLIER); + cbm_config_set(g_cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, timeout_ms); + } cbm_mcp_server_set_config(g_srv, g_cfg); g_rss_before_full = cbm_mem_rss(); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index abcf52d5a..0c4c189d7 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -14608,6 +14608,7 @@ TEST(pipeline_apply_config_sets_all_thresholds) { ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_SEMANTIC_THRESHOLD, "0.76"), 0); ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_GITHISTORY_MIN_COUPLING, "0.31"), 0); ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_LSP_CONFIDENCE_FLOOR, "0.61"), 0); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "17000"), 0); char max_changed[CBM_SZ_32]; char max_affected[CBM_SZ_32]; int n = snprintf(max_changed, sizeof(max_changed), "%d", PIPELINE_TEST_EXACT_MAX_CHANGED); @@ -14633,11 +14634,21 @@ TEST(pipeline_apply_config_sets_all_thresholds) { ASSERT_TRUE(cbm_pipeline_githistory_min_coupling(p) < 0.32); ASSERT_TRUE(cbm_pipeline_lsp_confidence_floor(p) > 0.60); ASSERT_TRUE(cbm_pipeline_lsp_confidence_floor(p) < 0.62); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), 17000000); ASSERT_EQ(cbm_pipeline_exact_max_changed_paths(p), PIPELINE_TEST_EXACT_MAX_CHANGED); ASSERT_EQ(cbm_pipeline_exact_max_affected_paths(p), PIPELINE_TEST_EXACT_MAX_AFFECTED); ASSERT_TRUE(cbm_pipeline_incremental_derived_refresh_stale_on_exact(p)); ASSERT_TRUE(cbm_pipeline_incremental_derived_refresh_stale_on_incremental(p)); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "1"), 0); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), + (int64_t)CBM_CONFIG_EXTRACT_TIMEOUT_MIN_MS * 1000); + ASSERT_EQ(cbm_config_set(cfg, CBM_CONFIG_EXTRACT_TIMEOUT_MS, "999999"), 0); + cbm_pipeline_apply_config(p, cfg); + ASSERT_EQ(cbm_pipeline_extract_timeout_micros(p), + (int64_t)CBM_CONFIG_EXTRACT_TIMEOUT_MAX_MS * 1000); + cbm_pipeline_free(p); cbm_config_close(cfg); rm_rf(tmpdir); @@ -14900,6 +14911,15 @@ TEST(config_registry_includes_incremental_reindex_policy) { PASS(); } +TEST(config_registry_includes_extract_timeout) { + const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_EXTRACT_TIMEOUT_MS); + ASSERT_NOT_NULL(entry); + ASSERT_STR_EQ(entry->default_val, CBM_CONFIG_EXTRACT_TIMEOUT_DEFAULT); + ASSERT_STR_EQ(entry->category, "Indexing"); + ASSERT_STR_EQ(entry->range, "100-120000"); + PASS(); +} + TEST(config_registry_includes_overlay_publish_policy) { const cbm_config_entry_t *entry = find_config_entry(CBM_CONFIG_OVERLAY_PUBLISH); ASSERT_NOT_NULL(entry); @@ -15670,6 +15690,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_semantic_batch_rejects_invalid_token_stride); RUN_TEST(config_registry_includes_mcp_timeout_knobs); RUN_TEST(config_registry_includes_incremental_reindex_policy); + RUN_TEST(config_registry_includes_extract_timeout); RUN_TEST(config_registry_includes_overlay_publish_policy); RUN_TEST(config_registry_includes_overlay_compaction_policy); RUN_TEST(config_registry_includes_incremental_exact_frontier_caps);