Merge pull request #854 from DeusData/distill/685-retention-reshape

fix(index): bound + source-cap parallel retention with re-read fallback (low-RAM peak RSS)

src/pipeline/pass_parallel.c

@@ -40,15 +40,32 @@ enum {
 #define PP_FIELD_HINT_CONF 0.85
 enum { PP_CSHARP_M_PREFIX_LEN = 2 };
 
-/* Source-retention caps for the parallel pipeline. The extract worker
- * copies source bytes into result->arena so the fused cross-file LSP
- * step in resolve_worker can run without re-reading from disk. Bound
- * peak RSS with a per-file cap (skip retention for pathological huge
- * generated files) and a total project-wide cap (skip when budget
- * exhausted — cross-file LSP becomes a no-op for those late files,
- * defs/calls already extracted are unaffected). */
-#define PP_RETAIN_PER_FILE_MAX_BYTES (100LL * 1024 * 1024)
-#define PP_RETAIN_TOTAL_BUDGET_BYTES (2LL * 1024 * 1024 * 1024)
+/* Absolute source-retention ceilings for the parallel extract pipeline.
+ *
+ * The extract worker copies each file's source bytes into result->arena so
+ * the fused cross-file LSP step in resolve_worker can re-parse without
+ * re-opening the file. That retention is TRANSIENT (freed at run end) but it
+ * is a PEAK-RSS driver: every retained byte is resident at once across the
+ * extract→resolve handoff.
+ *
+ * A cap here is a FLOOR as much as a ceiling: whatever total we allow, we
+ * WILL hold that much resident at peak on a large repo. rust-analyzer bounds
+ * retained file *text* to a small fixed budget and re-reads source on a miss
+ * rather than scaling the retained set with host RAM — because the re-read is
+ * cheap relative to holding tens of GB resident. We follow the same model:
+ *   - derive the total budget from the process memory budget (budget/8),
+ *   - BUT clamp the RAM-derived DEFAULT to a small absolute ceiling (1 GiB)
+ *     so a 512 GiB host does not retain 64 GiB of source it would re-read
+ *     cheaply anyway, and keep the per-file cap modest (32 MiB) so one
+ *     pathological generated blob cannot monopolise the budget.
+ * A file dropped from retention is NOT lost to cross-file resolution:
+ * resolve_worker re-reads it on demand, bounded and freed immediately
+ * (source_reread fallback). Both caps are env-overridable — CBM_RETAIN_TOTAL_MB
+ * / CBM_RETAIN_PER_FILE_MB raise or lower the auto-derived defaults directly
+ * (the hard ceilings bound only the RAM-derived default, never a deliberate
+ * operator/caller choice). */
+#define PP_RETAIN_TOTAL_HARD_MAX_BYTES (1024ULL * 1024 * 1024)  /* 1 GiB default ceiling */
+#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */
 #include "pipeline/pipeline.h"
 #include "pipeline/pipeline_internal.h"
 #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
@@ -73,6 +90,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
 #include "simhash/minhash.h"
 #include "semantic/ast_profile.h"
 
+#include <errno.h>
 #include <stdatomic.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -80,6 +98,95 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
 #include <string.h>
 #include <time.h>
 
+/* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes.
+ * Follows the limits.c strtol convention: unset / unparseable / non-positive
+ * → return 0 so the caller keeps its derived default. */
+static size_t cbm_retain_env_bytes(const char *name) {
+    const char *raw = getenv(name);
+    if (!raw || !raw[0]) {
+        return 0;
+    }
+    errno = 0;
+    char *end = NULL;
+    long v = strtol(raw, &end, 10);
+    if (errno != 0 || end == raw || *end != '\0' || v <= 0) {
+        return 0;
+    }
+    return (size_t)v * 1024 * 1024;
+}
+
+/* Auto-derived TOTAL retention budget: a fraction of the process memory
+ * budget, clamped to the absolute ceiling. The clamp bounds ONLY this
+ * RAM-derived default — a huge-RAM host must not retain tens of GB it would
+ * re-read cheaply. When the budget is unset (tests / no cgroup) fall back to
+ * the ceiling. */
+static size_t cbm_parallel_extract_default_total_cap(void) {
+    size_t cap = cbm_mem_budget();
+    if (cap > 0) {
+        cap /= 8;
+    } else {
+        cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
+    }
+    if (cap > PP_RETAIN_TOTAL_HARD_MAX_BYTES) {
+        cap = PP_RETAIN_TOTAL_HARD_MAX_BYTES;
+    }
+    return cap;
+}
+
+/* Auto-derived PER-FILE cap: modest absolute ceiling, never above the total. */
+static size_t cbm_parallel_extract_default_per_file_cap(size_t total_cap) {
+    size_t cap = PP_RETAIN_PER_FILE_HARD_MAX_BYTES;
+    if (cap > total_cap) {
+        cap = total_cap;
+    }
+    return cap;
+}
+
+/* Resolve the effective retention options from (in precedence order):
+ * explicit caller opts > CBM_RETAIN_*_MB env knobs > RAM-derived defaults.
+ * The absolute hard ceilings apply only to the RAM-derived defaults; an
+ * operator/caller that sets a value explicitly is trusted. The only invariant
+ * enforced unconditionally is per-file ≤ total. */
+static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts(
+    const cbm_parallel_extract_opts_t *opts) {
+    cbm_parallel_extract_opts_t resolved = {
+        .retain_sources = true,
+        .retain_sources_set = true,
+        .retain_total_budget_bytes = cbm_parallel_extract_default_total_cap(),
+        .retain_per_file_max_bytes = 0,
+    };
+
+    size_t env_total = cbm_retain_env_bytes("CBM_RETAIN_TOTAL_MB");
+    if (env_total > 0) {
+        resolved.retain_total_budget_bytes = env_total;
+    }
+
+    resolved.retain_per_file_max_bytes =
+        cbm_parallel_extract_default_per_file_cap(resolved.retain_total_budget_bytes);
+    size_t env_per_file = cbm_retain_env_bytes("CBM_RETAIN_PER_FILE_MB");
+    if (env_per_file > 0) {
+        resolved.retain_per_file_max_bytes = env_per_file;
+    }
+
+    if (opts) {
+        if (opts->retain_sources_set) {
+            resolved.retain_sources = opts->retain_sources;
+        }
+        if (opts->retain_total_budget_bytes > 0) {
+            resolved.retain_total_budget_bytes = opts->retain_total_budget_bytes;
+        }
+        if (opts->retain_per_file_max_bytes > 0) {
+            resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes;
+        }
+    }
+
+    /* Correctness invariant: a single file can never exceed the total budget. */
+    if (resolved.retain_per_file_max_bytes > resolved.retain_total_budget_bytes) {
+        resolved.retain_per_file_max_bytes = resolved.retain_total_budget_bytes;
+    }
+    return resolved;
+}
+
 static uint64_t extract_now_ns(void) {
     struct timespec ts;
     cbm_clock_gettime(CLOCK_MONOTONIC, &ts);
@@ -569,7 +676,12 @@ typedef struct {
     _Atomic int next_file_idx;
 
     cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
-    _Atomic int64_t retained_bytes; /* total source bytes copied into result arenas */
+
+    bool retain_sources;              /* copy source into result->arena for cross-file LSP */
+    size_t retain_total_budget_bytes; /* project-wide retention cap (peak-RSS bound) */
+    size_t retain_per_file_max_bytes; /* per-file retention cap */
+    _Atomic int64_t retained_bytes;   /* total source bytes copied into result arenas */
+    _Atomic int retain_cap_warned;    /* WARN index.retain_capped emitted once per run */
 
     /* Per-worker skip lists (separate allocation, indexed by worker_id — no hot-
      * path lock). Merged into the pipeline in the sequential merge loop. */
@@ -768,29 +880,44 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
                                  source_len, &ec->pkg_entries[worker_id]);
         }
 
-        /* Retain source bytes in result->arena so the fused cross-file
-         * LSP step in resolve_worker can run without re-reading from
-         * disk. Capped per-file (PP_RETAIN_PER_FILE_MAX_BYTES) and
-         * globally (PP_RETAIN_TOTAL_BUDGET_BYTES) to bound peak RSS.
-         * Skipping retention just means cross-file LSP no-ops for this
-         * file — defs/calls already extracted are unaffected. */
-        if (source_len > 0 && (int64_t)source_len <= PP_RETAIN_PER_FILE_MAX_BYTES) {
-            int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
-                                                      memory_order_relaxed);
-            if (prior + (int64_t)source_len <= PP_RETAIN_TOTAL_BUDGET_BYTES) {
-                char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
-                if (copy) {
-                    memcpy(copy, source, (size_t)source_len);
-                    copy[source_len] = '\0';
-                    result->source = copy;
-                    result->source_len = source_len;
+        /* Retain source bytes in result->arena so the fused cross-file LSP
+         * step in resolve_worker can re-parse without re-reading from disk.
+         * Bounded per-file (retain_per_file_max_bytes) and by a project-wide
+         * budget (retain_total_budget_bytes) to bound peak RSS — see the
+         * retention-cap comment at the top of this file. A file DROPPED here
+         * is NOT lost to cross-file resolution: resolve_worker re-reads it on
+         * demand (bounded, freed immediately). WARN once per run so the
+         * operator knows retention was capped (index.retain_capped). */
+        if (ec->retain_sources && source_len > 0) {
+            bool dropped = false;
+            if ((size_t)source_len > ec->retain_per_file_max_bytes) {
+                dropped = true; /* over the per-file cap */
+            } else {
+                int64_t prior = atomic_fetch_add_explicit(&ec->retained_bytes, (int64_t)source_len,
+                                                          memory_order_relaxed);
+                if ((size_t)(prior + (int64_t)source_len) <= ec->retain_total_budget_bytes) {
+                    char *copy = (char *)cbm_arena_alloc(&result->arena, (size_t)source_len + 1);
+                    if (copy) {
+                        memcpy(copy, source, (size_t)source_len);
+                        copy[source_len] = '\0';
+                        result->source = copy;
+                        result->source_len = source_len;
+                    } else {
+                        /* Arena OOM — not a cap drop; re-read still covers
+                         * cross-file resolution, so don't emit the WARN. */
+                        atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
+                                                  memory_order_relaxed);
+                    }
                 } else {
                     atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
                                               memory_order_relaxed);
+                    dropped = true; /* project-wide budget exhausted */
                 }
-            } else {
-                atomic_fetch_sub_explicit(&ec->retained_bytes, (int64_t)source_len,
-                                          memory_order_relaxed);
+            }
+            if (dropped &&
+                atomic_exchange_explicit(&ec->retain_cap_warned, 1, memory_order_relaxed) == 0) {
+                cbm_log_warn("index.retain_capped", "path", fi->rel_path ? fi->rel_path : "?",
+                             "bytes", itoa_log((int)source_len));
             }
         }
 
@@ -856,15 +983,24 @@ static void log_extract_mem_stats(int worker_count) {
     }
 }
 
-int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
-                         CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
-                         int worker_count) {
+int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
+                            CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
+                            int worker_count, const cbm_parallel_extract_opts_t *opts) {
+    cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts);
+
     if (file_count == 0) {
         return 0;
     }
 
     cbm_log_info("parallel.extract.start", "files", itoa_log(file_count), "workers",
                  itoa_log(worker_count));
+    {
+        size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K;
+        cbm_log_info("parallel.extract.retention", "retain_sources",
+                     resolved_opts.retain_sources ? "true" : "false", "total_mb",
+                     itoa_log((int)(resolved_opts.retain_total_budget_bytes / mb)), "per_file_mb",
+                     itoa_log((int)(resolved_opts.retain_per_file_max_bytes / mb)));
+    }
 
     /* Log per-worker memory budget */
     if (cbm_mem_budget() > 0) {
@@ -885,7 +1021,10 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
 
     /* Sub-phase: Sort files by descending size for tail-latency reduction */
     CBM_PROF_START(t_sort);
-    file_sort_entry_t *sorted = malloc(file_count * sizeof(file_sort_entry_t));
+    file_sort_entry_t *sorted = malloc((size_t)file_count * sizeof(file_sort_entry_t));
+    if (!sorted) {
+        return CBM_NOT_FOUND;
+    }
     for (int i = 0; i < file_count; i++) {
         sorted[i].idx = i;
         sorted[i].size = files[i].size;
@@ -903,11 +1042,16 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
     memset(workers, 0, (size_t)worker_count * sizeof(extract_worker_state_t));
 
     /* Per-worker manifest entry arrays (separate from cache-line-aligned worker state) */
-    cbm_pkg_entries_t *pkg_entries = calloc(worker_count, sizeof(cbm_pkg_entries_t));
+    cbm_pkg_entries_t *pkg_entries = calloc((size_t)worker_count, sizeof(cbm_pkg_entries_t));
+    if (!pkg_entries) {
+        cbm_aligned_free(workers);
+        free(sorted);
+        return CBM_NOT_FOUND;
+    }
 
     /* Per-worker skip lists (separate allocation; merged into the pipeline in the
      * sequential merge loop below). */
-    pp_err_list_t *err_lists = calloc(worker_count, sizeof(pp_err_list_t));
+    pp_err_list_t *err_lists = calloc((size_t)worker_count, sizeof(pp_err_list_t));
 
     extract_ctx_t ec = {
         .files = files,
@@ -922,16 +1066,20 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
         .cancelled = ctx->cancelled,
         .pkg_entries = pkg_entries,
         .err_lists = err_lists,
+        .retain_sources = resolved_opts.retain_sources,
+        .retain_total_budget_bytes = resolved_opts.retain_total_budget_bytes,
+        .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes,
     };
     atomic_init(&ec.next_worker_id, 0);
     atomic_init(&ec.next_file_idx, 0);
     atomic_init(&ec.retained_bytes, 0);
+    atomic_init(&ec.retain_cap_warned, 0);
     atomic_init(&ec.oversized_warned, 0);
 
     /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
     CBM_PROF_START(t_dispatch);
-    cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
-    cbm_parallel_for(worker_count, extract_worker, &ec, opts);
+    cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
+    cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
     CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
 
     /* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
@@ -982,6 +1130,13 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
     return 0;
 }
 
+int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
+                         CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
+                         int worker_count) {
+    return cbm_parallel_extract_ex(ctx, files, file_count, result_cache, shared_ids, worker_count,
+                                   NULL);
+}
+
 /* ── Phase 3B: Serial Registry Build ─────────────────────────────── */
 
 /* Register one definition and create DEFINES + DEFINES_METHOD edges. Returns edge count. */
@@ -2466,18 +2621,31 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
          * Runs BEFORE resolve_file_calls so its additions to
          * result->resolved_calls are picked up by
          * cbm_pipeline_find_lsp_resolution when calls become CALLS
-         * edges. Requires result->source to have been retained in
-         * result->arena during extract (PP_RETAIN_*); files over the
-         * cap or past the budget have result->source==NULL and are
-         * counted as skipped_no_source — defs/calls already in the
-         * extract are unaffected.
+         * edges. Prefers source bytes retained in result->arena during
+         * extract; when the low-RAM retention cap dropped this file
+         * (result->source==NULL) it FALLS BACK to a bounded per-file
+         * read from disk, freed immediately after the LSP call. This is
+         * the correctness guarantee: lowering the retention cap trades
+         * retained RAM for a bounded re-read, it NEVER drops a cross-file
+         * edge. Only a genuine read failure (deleted/unreadable/oversized)
+         * leaves source NULL and is counted as skipped_no_source; defs/calls
+         * already in the extract are unaffected either way.
          *
          * Slab reclaim afterward: the LSP re-parses via tree-sitter,
          * which allocates through this worker's TLS slab. Reclaiming
          * here keeps the slab high-water bounded as the resolve phase
          * walks across thousands of files in a single worker thread. */
         if (cross_lsp_eligible) {
-            if (result->source && result->source_len > 0) {
+            char *lsp_source_owned = NULL;
+            const char *lsp_source = result->source;
+            int lsp_source_len = result->source_len;
+            if ((!lsp_source || lsp_source_len <= 0) && rc->files[file_idx].path) {
+                /* Retention cap skipped this file — re-read on demand (bounded
+                 * by read_file's cbm_max_file_bytes cap), freed below. */
+                lsp_source_owned = read_file(rc->files[file_idx].path, &lsp_source_len, NULL, NULL);
+                lsp_source = lsp_source_owned;
+            }
+            if (lsp_source && lsp_source_len > 0) {
                 const char *def_module = rc->def_modules ? rc->def_modules[file_idx] : module_qn;
 
                 uint64_t lsp_t0 = extract_now_ns();
@@ -2511,8 +2679,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
                     }
                     case CBM_LANG_PYTHON:
                         cbm_run_py_lsp_cross_with_registry(
-                            &result->arena, result->source, result->source_len, def_module,
-                            prebuilt, imp_keys, imp_vals, imp_count, result->cached_tree,
+                            &result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
+                            imp_keys, imp_vals, imp_count, result->cached_tree,
                             &result->resolved_calls);
                         used_prebuilt = true;
                         break;
@@ -2520,16 +2688,15 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
                     case CBM_LANG_CPP:
                     case CBM_LANG_CUDA:
                         cbm_run_c_lsp_cross_with_registry(
-                            &result->arena, result->source, result->source_len, def_module,
+                            &result->arena, lsp_source, lsp_source_len, def_module,
                             (lang != CBM_LANG_C), prebuilt, imp_keys, imp_vals, imp_count,
                             result->cached_tree, &result->resolved_calls);
                         used_prebuilt = true;
                         break;
                     case CBM_LANG_CSHARP:
-                        cbm_run_cs_lsp_cross_with_registry(&result->arena, result->source,
-                                                           result->source_len, def_module, prebuilt,
-                                                           imp_vals, imp_count, result->cached_tree,
-                                                           &result->resolved_calls);
+                        cbm_run_cs_lsp_cross_with_registry(
+                            &result->arena, lsp_source, lsp_source_len, def_module, prebuilt,
+                            imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
                         used_prebuilt = true;
                         break;
                     case CBM_LANG_JAVASCRIPT:
@@ -2557,8 +2724,8 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
                             }
                         }
                         cbm_run_ts_lsp_cross_with_registry(
-                            &result->arena, result->source, result->source_len, def_module, js, jsx,
-                            dts, prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
+                            &result->arena, lsp_source, lsp_source_len, def_module, js, jsx, dts,
+                            prebuilt, ts_defs, ts_def_count, imp_keys, imp_vals, imp_count,
                             result->cached_tree, &result->resolved_calls);
                         free(ts_filtered);
                         used_prebuilt = true;
@@ -2590,16 +2757,17 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
                         lang == CBM_LANG_TSX) {
                         bool js, jsx, dts;
                         cbm_pxc_ts_modes(lang, rel, &js, &jsx, &dts);
-                        cbm_pxc_run_one_ts(result, result->source, result->source_len, def_module,
+                        cbm_pxc_run_one_ts(result, lsp_source, lsp_source_len, def_module,
                                            file_defs, file_def_count, imp_keys, imp_vals, imp_count,
                                            js, jsx, dts);
                     } else {
-                        cbm_pxc_run_one(lang, result, result->source, result->source_len,
-                                        def_module, file_defs, file_def_count, imp_keys, imp_vals,
-                                        imp_count);
+                        cbm_pxc_run_one(lang, result, lsp_source, lsp_source_len, def_module,
+                                        file_defs, file_def_count, imp_keys, imp_vals, imp_count);
                     }
                 }
                 free(filtered);
+                /* Free the on-demand re-read (no-op when source was retained). */
+                free_source(lsp_source_owned);
                 /* Contract: cbm_slab_reclaim() requires the thread parser to be
                  * destroyed first; otherwise its lexer holds slab pointers
                  * (lexer.included_ranges) that get freed underneath it, causing
@@ -2617,13 +2785,14 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
                 }
                 atomic_fetch_add_explicit(&rc->lsp_cross_processed, SKIP_ONE, memory_order_relaxed);
             } else {
-                /* No retained source → the cross-file LSP refinement no-ops for
-                 * this file. This is a bounded OPTIMIZATION skip, NOT a file
-                 * failure: defs/calls were already extracted and are unaffected.
-                 * Deliberately NOT recorded as a cbm_file_error — doing so would
-                 * flood skipped[] with false positives (itself a false-guard
-                 * bug). The "cross_lsp" phase string is reserved for Track C's
-                 * real crash-attribution signal; leave it unwired here. */
+                /* Source unavailable even after the re-read fallback (file
+                 * deleted / unreadable / oversized) → the cross-file LSP
+                 * refinement no-ops for this file. This is a bounded skip, NOT
+                 * a file failure: defs/calls were already extracted and are
+                 * unaffected. Deliberately NOT recorded as a cbm_file_error —
+                 * doing so would flood skipped[] with false positives (itself a
+                 * false-guard bug). The "cross_lsp" phase string is reserved for
+                 * Track C's real crash-attribution signal; leave it unwired. */
                 atomic_fetch_add_explicit(&rc->lsp_cross_skipped_no_source, SKIP_ONE,
                                           memory_order_relaxed);
             }

src/pipeline/pipeline_internal.h

@@ -444,6 +444,21 @@ char *cbm_infra_qn(const char *project_name, const char *rel_path, const char *i
  * Each worker creates nodes in a per-worker gbuf, then merges into ctx->gbuf.
  * Caches CBMFileResult* in result_cache[file_idx] for reuse in Phase 3B/4.
  * shared_ids provides globally unique node/edge IDs across workers. */
+
+/* Source-retention tuning for cbm_parallel_extract_ex. Zero-valued byte caps
+ * mean "use the derived default" (RAM-fraction total, clamped to an absolute
+ * ceiling; modest per-file cap); CBM_RETAIN_TOTAL_MB / CBM_RETAIN_PER_FILE_MB
+ * override those. retain_sources_set=false keeps the default retain policy. */
+typedef struct {
+    bool retain_sources;
+    bool retain_sources_set; /* false keeps the default retain_sources policy */
+    size_t retain_total_budget_bytes;
+    size_t retain_per_file_max_bytes;
+} cbm_parallel_extract_opts_t;
+
+int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
+                            CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
+                            int worker_count, const cbm_parallel_extract_opts_t *opts);
 int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count,
                          CBMFileResult **result_cache, _Atomic int64_t *shared_ids,
                          int worker_count);

tests/test_mem.c

@@ -565,6 +565,167 @@ static void teardown_mem_test_repo(void) {
     }
 }
 
+static size_t count_retained_source_bytes(CBMFileResult **result_cache, int file_count,
+                                          int *retained_count) {
+    size_t retained_bytes = 0;
+    int count = 0;
+
+    for (int i = 0; i < file_count; i++) {
+        CBMFileResult *result = result_cache[i];
+        if (result && result->source) {
+            retained_bytes += (size_t)result->source_len;
+            count++;
+        }
+    }
+
+    if (retained_count) {
+        *retained_count = count;
+    }
+    return retained_bytes;
+}
+
+/* retain_sources=false disables source retention entirely: no result->source is
+ * kept, yet extraction still produces defs/nodes. Guards the low-RAM opt-out. */
+TEST(parallel_extract_without_source_retention) {
+    if (setup_mem_test_repo() != 0) {
+        FAIL("tmpdir setup failed");
+    }
+
+    cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
+    cbm_file_info_t *files = NULL;
+    int file_count = 0;
+    if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
+        teardown_mem_test_repo();
+        FAIL("discover failed");
+    }
+
+    cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
+    cbm_registry_t *reg = cbm_registry_new();
+    atomic_int cancelled;
+    atomic_init(&cancelled, 0);
+
+    cbm_pipeline_ctx_t ctx = {
+        .project_name = "mem-test",
+        .repo_path = g_mem_tmpdir,
+        .gbuf = gbuf,
+        .registry = reg,
+        .cancelled = &cancelled,
+    };
+
+    _Atomic int64_t shared_ids;
+    atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
+
+    CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
+    ASSERT_NOT_NULL(result_cache);
+
+    cbm_parallel_extract_opts_t extract_opts = {
+        .retain_sources = false,
+        .retain_sources_set = true,
+        .retain_total_budget_bytes = 0,
+        .retain_per_file_max_bytes = 0,
+    };
+    int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
+                                     &extract_opts);
+    ASSERT_EQ(rc, 0);
+
+    int defs_seen = 0;
+    for (int i = 0; i < file_count; i++) {
+        if (result_cache[i]) {
+            ASSERT_EQ(result_cache[i]->source, NULL);
+            defs_seen += result_cache[i]->defs.count;
+        }
+    }
+    ASSERT_GT(defs_seen, 0);
+    ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
+
+    for (int i = 0; i < file_count; i++) {
+        if (result_cache[i]) {
+            cbm_free_result(result_cache[i]);
+        }
+    }
+    free(result_cache);
+    cbm_registry_free(reg);
+    cbm_gbuf_free(gbuf);
+    cbm_discover_free(files, file_count);
+    teardown_mem_test_repo();
+    PASS();
+}
+
+/* Guard B (peak bound): a tiny total retention budget must actually bound the
+ * retained source bytes — retained_bytes <= budget — while extraction still
+ * produces defs/nodes. Over-budget files fall back to a bounded re-read during
+ * cross-file resolution (exercised in test_parallel.c), so the cap trades
+ * retained RAM, never correctness. */
+TEST(parallel_extract_tiny_source_retention_budget) {
+    if (setup_mem_test_repo() != 0) {
+        FAIL("tmpdir setup failed");
+    }
+
+    cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
+    cbm_file_info_t *files = NULL;
+    int file_count = 0;
+    if (cbm_discover(g_mem_tmpdir, &opts, &files, &file_count) != 0) {
+        teardown_mem_test_repo();
+        FAIL("discover failed");
+    }
+
+    cbm_gbuf_t *gbuf = cbm_gbuf_new("mem-test", g_mem_tmpdir);
+    cbm_registry_t *reg = cbm_registry_new();
+    atomic_int cancelled;
+    atomic_init(&cancelled, 0);
+
+    cbm_pipeline_ctx_t ctx = {
+        .project_name = "mem-test",
+        .repo_path = g_mem_tmpdir,
+        .gbuf = gbuf,
+        .registry = reg,
+        .cancelled = &cancelled,
+    };
+
+    _Atomic int64_t shared_ids;
+    atomic_init(&shared_ids, cbm_gbuf_next_id(gbuf));
+
+    CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
+    ASSERT_NOT_NULL(result_cache);
+
+    const size_t retain_total_budget_bytes = 256;
+    cbm_parallel_extract_opts_t extract_opts = {
+        .retain_sources = true,
+        .retain_sources_set = true,
+        .retain_total_budget_bytes = retain_total_budget_bytes,
+        .retain_per_file_max_bytes = 100U * 1024U * 1024U,
+    };
+    int rc = cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, 2,
+                                     &extract_opts);
+    ASSERT_EQ(rc, 0);
+
+    int retained_count = 0;
+    size_t retained_bytes = count_retained_source_bytes(result_cache, file_count, &retained_count);
+    int defs_seen = 0;
+    for (int i = 0; i < file_count; i++) {
+        if (result_cache[i]) {
+            defs_seen += result_cache[i]->defs.count;
+        }
+    }
+
+    ASSERT_GT(defs_seen, 0);
+    ASSERT_GT(retained_count, 0);
+    ASSERT_LTE(retained_bytes, retain_total_budget_bytes);
+    ASSERT_GT(cbm_gbuf_node_count(gbuf), 0);
+
+    for (int i = 0; i < file_count; i++) {
+        if (result_cache[i]) {
+            cbm_free_result(result_cache[i]);
+        }
+    }
+    free(result_cache);
+    cbm_registry_free(reg);
+    cbm_gbuf_free(gbuf);
+    cbm_discover_free(files, file_count);
+    teardown_mem_test_repo();
+    PASS();
+}
+
 TEST(parallel_extract_with_slab) {
     cbm_mem_init(0.5);
 
@@ -666,5 +827,7 @@ SUITE(mem) {
     RUN_TEST(slab_calloc_zeroed);
     RUN_TEST(slab_mixed_alloc_free_stress);
     /* Integration */
+    RUN_TEST(parallel_extract_without_source_retention);
+    RUN_TEST(parallel_extract_tiny_source_retention_budget);
     RUN_TEST(parallel_extract_with_slab);
 }

tests/test_parallel.c

@@ -112,8 +112,10 @@ static cbm_gbuf_t *run_sequential(const char *project, const char *repo_path,
 
 /* ── Run parallel pipeline on files, returning gbuf ───────────────── */
 
-static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
-                                int file_count, int worker_count) {
+static cbm_gbuf_t *run_parallel_with_extract_opts(const char *project, const char *repo_path,
+                                                  cbm_file_info_t *files, int file_count,
+                                                  int worker_count,
+                                                  const cbm_parallel_extract_opts_t *extract_opts) {
     cbm_gbuf_t *gbuf = cbm_gbuf_new(project, repo_path);
     cbm_registry_t *reg = cbm_registry_new();
     atomic_int cancelled;
@@ -131,10 +133,15 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
     int64_t gbuf_next = cbm_gbuf_next_id(gbuf);
     atomic_init(&shared_ids, gbuf_next);
 
-    CBMFileResult **result_cache = calloc(file_count, sizeof(CBMFileResult *));
+    CBMFileResult **result_cache = calloc((size_t)file_count, sizeof(CBMFileResult *));
 
     cbm_init();
-    cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
+    if (extract_opts) {
+        cbm_parallel_extract_ex(&ctx, files, file_count, result_cache, &shared_ids, worker_count,
+                                extract_opts);
+    } else {
+        cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
+    }
     cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
 
     cbm_build_registry_from_cache(&ctx, files, file_count, result_cache);
@@ -175,6 +182,12 @@ static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_
     return gbuf;
 }
 
+static cbm_gbuf_t *run_parallel(const char *project, const char *repo_path, cbm_file_info_t *files,
+                                int file_count, int worker_count) {
+    return run_parallel_with_extract_opts(project, repo_path, files, file_count, worker_count,
+                                          NULL);
+}
+
 /* ── Parity Tests ─────────────────────────────────────────────────── */
 
 static cbm_gbuf_t *g_seq_gbuf = NULL;
@@ -741,8 +754,7 @@ TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm) {
     int64_t nid = cbm_gbuf_upsert_node(tgbuf, "Method", "run", "proj.zeta.Helper.run",
                                        "zeta/helper.py", 1, 3, NULL);
     ASSERT_TRUE(nid != 0);
-    ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) ==
-                NULL);
+    ASSERT_TRUE(cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", false) == NULL);
     const cbm_gbuf_node_t *jvm_hit =
         cbm_pipeline_lsp_target_node(tgbuf, "proj", "com.other.Helper.run", true);
     ASSERT_NOT_NULL(jvm_hit);
@@ -884,6 +896,129 @@ TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges) {
     PASS();
 }
 
+/* RED/GREEN A — the graph-quality guarantee behind the low-RAM retention cap.
+ *
+ * The fused cross-file LSP step re-parses each file's source to resolve calls
+ * whose receiver type lives in ANOTHER file (the per-file pass cannot). When
+ * the retention cap drops a file's source, that resolution MUST still happen
+ * via a bounded on-demand re-read; otherwise the cross-file CALLS edge is LOST.
+ *
+ * Fixture: a Java<->Kotlin pair with genuinely cross-language calls that only
+ * the cross-file LSP resolves — JavaCaller.call -> KotlinService.ping (Java ->
+ * Kotlin) and KotlinService.ping -> JavaService.pong (Kotlin -> Java). These
+ * carry the "lsp" strategy and do NOT exist without the cross-file source,
+ * unlike same-file or import-local Python calls which the per-file pass already
+ * resolves (so counting lsp edges on those cannot detect the fallback).
+ *
+ * Three scenarios asserted GREEN with the re-read fallback in place:
+ *   1. CONTROL   — default retention: both cross-file edges present. Proves the
+ *                  fixture genuinely produces them (non-vacuity guard).
+ *   2. NO-RETAIN — retain_sources=false: nothing retained -> edges survive only
+ *                  via the re-read fallback.
+ *   3. OVER-CAP  — per-file cap = 1 byte: every file dropped by the SIZE cap ->
+ *                  edges survive only via the re-read fallback.
+ * On main (no fallback) scenarios 2 and 3 LOSE both edges = RED; scenario 1
+ * stays present = the non-vacuity control. */
+TEST(parallel_cross_file_reread_preserves_unretained_edges) {
+    char tmpdir[256];
+    snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_par_xf_reread_XXXXXX");
+    if (!cbm_mkdtemp(tmpdir)) {
+        FAIL("mkdtemp failed");
+    }
+
+    char jpath[512];
+    snprintf(jpath, sizeof(jpath), "%s/src/main/java/com/example/Example.java", tmpdir);
+    char jdir[512];
+    snprintf(jdir, sizeof(jdir), "%s/src/main/java/com/example", tmpdir);
+    cbm_mkdir_p(jdir, 0755);
+    FILE *jf = fopen(jpath, "w");
+    if (!jf) {
+        FAIL("fopen Example.java failed");
+    }
+    fprintf(jf, "package com.example;\n"
+                "\n"
+                "class JavaCaller {\n"
+                "    String call(KotlinService kotlinService) {\n"
+                "        return kotlinService.ping(new JavaService());\n"
+                "    }\n"
+                "}\n"
+                "\n"
+                "class JavaService {\n"
+                "    String pong() {\n"
+                "        return \"pong\";\n"
+                "    }\n"
+                "}\n");
+    fclose(jf);
+
+    char kpath[512];
+    snprintf(kpath, sizeof(kpath), "%s/src/main/kotlin/com/example/KotlinService.kt", tmpdir);
+    char kdir[512];
+    snprintf(kdir, sizeof(kdir), "%s/src/main/kotlin/com/example", tmpdir);
+    cbm_mkdir_p(kdir, 0755);
+    FILE *kf = fopen(kpath, "w");
+    if (!kf) {
+        FAIL("fopen KotlinService.kt failed");
+    }
+    fprintf(kf, "package com.example\n"
+                "\n"
+                "class KotlinService {\n"
+                "    fun ping(javaService: JavaService): String {\n"
+                "        return javaService.pong()\n"
+                "    }\n"
+                "}\n");
+    fclose(kf);
+
+    cbm_file_info_t files[2] = {0};
+    files[0].path = jpath;
+    files[0].rel_path = (char *)"src/main/java/com/example/Example.java";
+    files[0].language = CBM_LANG_JAVA;
+    files[1].path = kpath;
+    files[1].rel_path = (char *)"src/main/kotlin/com/example/KotlinService.kt";
+    files[1].language = CBM_LANG_KOTLIN;
+
+    /* CONTROL (retained) + two drop scenarios that reach the cross-file edge
+     * only via the on-demand re-read: NO-RETAIN disables retention entirely;
+     * OVER-CAP sets a 1-byte per-file cap so every file is dropped by size. */
+    const cbm_parallel_extract_opts_t no_retain = {
+        .retain_sources = false,
+        .retain_sources_set = true,
+    };
+    const cbm_parallel_extract_opts_t over_cap = {
+        .retain_sources = true,
+        .retain_sources_set = true,
+        .retain_per_file_max_bytes = 1, /* 1 byte → every file dropped by the size cap */
+    };
+    const cbm_parallel_extract_opts_t *scenarios[3] = {NULL, &no_retain, &over_cap};
+
+    for (int s = 0; s < 3; s++) {
+        cbm_gbuf_t *gbuf = run_parallel_with_extract_opts("com", tmpdir, files, 2, 2, scenarios[s]);
+        ASSERT_NOT_NULL(gbuf);
+
+        const cbm_gbuf_edge_t *java_to_kotlin =
+            find_calls_edge_by_tails(gbuf, "JavaCaller.call", "KotlinService.ping");
+        const cbm_gbuf_edge_t *kotlin_to_java =
+            find_calls_edge_by_tails(gbuf, "KotlinService.ping", "JavaService.pong");
+
+        /* Both cross-file (Java↔Kotlin) CALLS edges must be present in EVERY
+         * scenario. In the drop scenarios (s=1,2) the caller's source is NOT
+         * retained, so these edges exist ONLY because resolve_worker re-reads
+         * the source on demand. Without that fallback (main) they are LOST. */
+        ASSERT_NOT_NULL(java_to_kotlin);
+        ASSERT_NOT_NULL(kotlin_to_java);
+        /* And they must come from the source-dependent cross-file LSP, not a
+         * source-free suffix heuristic — proving the re-read actually ran. */
+        ASSERT_NOT_NULL(java_to_kotlin->properties_json);
+        ASSERT_NOT_NULL(strstr(java_to_kotlin->properties_json, "\"strategy\":\"lsp"));
+        ASSERT_NOT_NULL(kotlin_to_java->properties_json);
+        ASSERT_NOT_NULL(strstr(kotlin_to_java->properties_json, "\"strategy\":\"lsp"));
+
+        cbm_gbuf_free(gbuf);
+    }
+
+    th_rmtree(tmpdir);
+    PASS();
+}
+
 /* issue #294: gRPC service-name extraction must (a) preserve the canonical
  * proto service name (FooServiceClient → FooService, not Foo) and (b) only
  * match real stub/client types — ordinary receiver vars must NOT produce
@@ -941,6 +1076,7 @@ SUITE(parallel) {
     RUN_TEST(parallel_node_count);
     RUN_TEST(parallel_python_lsp_override_emits_lsp_strategy_edges);
     RUN_TEST(parallel_python_lsp_override_cross_file_emits_lsp_strategy_edges);
+    RUN_TEST(parallel_cross_file_reread_preserves_unretained_edges);
     RUN_TEST(parallel_java_kotlin_lsp_override_cross_file_emits_lsp_strategy_edges);
     RUN_TEST(parallel_lsp_tail_match_fallbacks_gated_to_jvm);
     RUN_TEST(parallel_calls_parity);