perf(index): futile-nap latch + supervised-worker fast exit

Two wall-clock fixes from a linux-kernel (8.5M nodes) profiling run:

1. Extraction back-pressure futility latch: when a full collect+nap cycle ends
   still over the RSS budget, the resident floor (accumulated graph + retained
   sources) -- not in-flight transients -- holds the memory, so napping again on
   every subsequent pull reclaims nothing and only idles workers (measured: one
   full cycle per pull, ~390 s at 79% avg CPU across 63k pulls). Latch bp_futile
   after the first futile cycle, proceed with the designed soft overshoot, and
   re-arm via the cheap per-pull probe once RSS drains under budget. WARN
   mem.backpressure.futile exactly once per latch event (0->1 transition).

2. Supervised-worker fast exit: the index worker piecemeal-freed a multi-GB
   graph (and store) right before process death (measured: minutes of free()
   in the tail). In worker role, skip cbm_pipeline_free and _Exit after the
   response is written and flushed; the OS reclaims memory wholesale. In-process
   paths (tests, kill switch, degrade) free normally, keeping ASan/LSan
   meaningful.

Reproduce-first: pipeline_backpressure_futile_nap_disengages (64-file fixture >
MIN_FILES_FOR_PARALLEL so the parallel gate actually runs, CBM_MEM_BUDGET_MB=1,
deterministic nap-cycle counter; engagement guard cycles>=1 prevents a vacuous
pass). RED on the old gate: 64 cycles (one per pull) > bound cores+2. GREEN with
the latch. Suites: pipeline 216, mcp 140, subprocess 17 -- all pass.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/main.c

@@ -479,6 +479,16 @@ static int run_cli(int argc, char **argv) {
         } else {
             exit_code = cli_print_mcp_result(result);
         }
+        if (cbm_index_worker_active()) {
+            /* Supervised worker: the response is delivered (file + stdout).
+             * Skip the multi-GB teardown (server/store frees) — the process
+             * dies now and the OS reclaims everything wholesale; piecemeal
+             * free() of a kernel-scale graph costs minutes. _Exit skips
+             * atexit/LSan by design for this prod worker path. */
+            cbm_log_info("index.worker.fast_exit", "action", "_Exit");
+            fflush(NULL);
+            _Exit(exit_code);
+        }
         free(result);
     }
 

src/mcp/mcp.c

@@ -3866,8 +3866,17 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
 
     char *json = yy_doc_to_str(doc);
     yyjson_mut_doc_free(doc);
-    /* Free the pipeline only after the response doc copied the excluded list. */
-    cbm_pipeline_free(p);
+    /* Free the pipeline only after the response doc copied the excluded list.
+     * Supervised worker: skip the deep free — the process exits right after
+     * handing over the response (main.c fast-exits), and piecemeal-freeing a
+     * multi-GB graph before process death costs minutes on kernel-scale repos;
+     * the OS reclaims it wholesale at exit. In-process paths (tests, kill
+     * switch, degrade) still free normally. */
+    if (cbm_index_worker_active()) {
+        cbm_log_info("index.worker.fast_exit", "skip", "pipeline_free");
+    } else {
+        cbm_pipeline_free(p);
+    }
     free(project_name);
     free(repo_path);
 

src/pipeline/pass_parallel.c

@@ -98,6 +98,20 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
 #include <string.h>
 #include <time.h>
 
+/* Back-pressure nap-cycle counter (test observability): each execution of the
+ * over-budget collect+nap gate counts one cycle. Lets tests assert the gate does
+ * not re-pay the full nap tax on every file pull when napping cannot reclaim
+ * memory (the resident floor, not in-flight transients, holds the budget). */
+static _Atomic long g_bp_nap_cycles = 0;
+
+long cbm_pp_bp_nap_cycles(void) {
+    return atomic_load_explicit(&g_bp_nap_cycles, memory_order_relaxed);
+}
+
+void cbm_pp_bp_nap_cycles_reset(void) {
+    atomic_store_explicit(&g_bp_nap_cycles, 0, memory_order_relaxed);
+}
+
 /* Parse a positive MB-valued retention env knob (CBM_RETAIN_*_MB) into bytes.
  * Follows the limits.c strtol convention: unset / unparseable / non-positive
  * → return 0 so the caller keeps its derived default. */
@@ -687,6 +701,13 @@ typedef struct {
      * path lock). Merged into the pipeline in the sequential merge loop. */
     pp_err_list_t *err_lists;
     _Atomic int oversized_warned; /* throttle for the index.file_oversized WARN */
+
+    /* Back-pressure futility latch: set when a full collect+nap cycle ended
+     * still over budget — the resident floor (graph + retained sources), not
+     * in-flight transients, holds the memory, so napping cannot reclaim it.
+     * While set, pulls skip the nap (the designed soft overshoot); the cheap
+     * over-budget probe re-arms the gate once RSS drains under budget. */
+    _Atomic int bp_futile;
 } extract_ctx_t;
 
 /* Cap on the number of index.file_oversized WARN lines (the full list still goes
@@ -763,14 +784,37 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
          * near the budget instead of letting all workers parse their biggest
          * files at once. Self-disabling when the budget is unset (tests) or RSS
          * is under budget; bounded spins avoid deadlock when the resident graph
-         * is itself near budget (then proceed with a soft overshoot). */
-        if (cbm_mem_budget() > 0 && cbm_mem_over_budget()) {
-            cbm_mem_collect();
-            for (int bp = 0; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
-                             !atomic_load_explicit(ec->cancelled, memory_order_relaxed);
-                 bp++) {
-                struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
-                cbm_nanosleep(&nap, NULL);
+         * is itself near budget (then proceed with a soft overshoot).
+         *
+         * Futility latch: when a FULL nap cycle ends still over budget, the
+         * resident floor — not transients — holds the memory; napping again on
+         * the next pull cannot reclaim it and only idles workers (linux kernel:
+         * one full cycle per pull ≈ 390 s at 79% avg CPU). Latch bp_futile and
+         * proceed with the soft overshoot; the over-budget probe below re-arms
+         * the gate as soon as RSS drains under budget. */
+        if (cbm_mem_budget() > 0) {
+            bool over = cbm_mem_over_budget();
+            bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0;
+            if (over && !futile) {
+                cbm_mem_collect();
+                atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed);
+                int bp = 0;
+                for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
+                       !atomic_load_explicit(ec->cancelled, memory_order_relaxed);
+                     bp++) {
+                    struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
+                    cbm_nanosleep(&nap, NULL);
+                }
+                if (bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget()) {
+                    /* Log only the 0→1 transition: all workers race into the
+                     * gate before anyone latches, so a plain store would WARN
+                     * once per worker (12 lines per latch event). */
+                    if (atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) {
+                        cbm_log_warn("mem.backpressure.futile", "action", "soft_overshoot");
+                    }
+                }
+            } else if (!over && futile) {
+                atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
             }
         }
 
@@ -1075,6 +1119,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
     atomic_init(&ec.retained_bytes, 0);
     atomic_init(&ec.retain_cap_warned, 0);
     atomic_init(&ec.oversized_warned, 0);
+    atomic_init(&ec.bp_futile, 0);
 
     /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
     CBM_PROF_START(t_dispatch);

src/pipeline/pipeline_internal.h

@@ -614,4 +614,11 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges);
 bool extract_grpc_service_method(const char *callee, char *service, size_t srv_sz, char *method,
                                  size_t meth_sz);
 
+/* Extraction back-pressure observability (pass_parallel.c): nap-cycle counter
+ * for the over-budget collect+nap gate. Test hook — asserts the gate stops
+ * re-paying the nap tax once a full cycle failed to reclaim under budget
+ * (futile: the resident floor, not transients, holds the memory). */
+long cbm_pp_bp_nap_cycles(void);
+void cbm_pp_bp_nap_cycles_reset(void);
+
 #endif /* CBM_PIPELINE_INTERNAL_H */

tests/test_pipeline.c

@@ -8,6 +8,7 @@
 #include "foundation/platform.h" // cbm_normalize_path_sep (drive-canonicalization regression)
 #include "test_framework.h"
 #include "test_helpers.h"
+#include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test)
 #include "pipeline/pipeline.h"
 #include "pipeline/pipeline_internal.h"
 #include "store/store.h"
@@ -6541,6 +6542,79 @@ TEST(pipeline_committed_counts_match_persisted) {
     PASS();
 }
 
+/* Reproduce-first (perf, linux-kernel finding): the extraction back-pressure
+ * gate must stop re-paying the full collect+nap tax on every file pull once a
+ * full nap cycle has failed to reclaim under budget. With CBM_MEM_BUDGET_MB=1
+ * the test process RSS is permanently over budget and NOTHING napping can
+ * change that (the resident floor IS the process) — napping is provably futile.
+ * OLD behavior: one full 40-spin nap cycle per pulled file (kernel index: ~63k
+ * pulls × ~120 ms ÷ 12 workers ≈ 390 s of idle workers at 79% avg CPU).
+ * FIXED: the first futile cycle flips a shared flag; later pulls proceed with
+ * the designed soft overshoot. Cycle count then can't exceed one cycle per
+ * worker (workers already inside the gate when the flag flips) plus re-probes.
+ * RED on the unfixed gate: cycles == file count (64) > cores+2.
+ * The counter (cbm_pp_bp_nap_cycles) makes this deterministic — no timing.
+ *
+ * The gate lives ONLY in the parallel extract path, so the fixture MUST exceed
+ * MIN_FILES_FOR_PARALLEL (50) — else the run routes sequential, the gate never
+ * fires, and the test would pass vacuously (cycles==0). The engagement assert
+ * below (cycles >= 1) is a hard guard against that regressing silently. */
+TEST(pipeline_backpressure_futile_nap_disengages) {
+    /* 64 tiny files: > MIN_FILES_FOR_PARALLEL (50) so the parallel path (and its
+     * back-pressure gate) actually runs; old-code cycles (~64) >> the bound. */
+    snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/cbm_test_XXXXXX");
+    if (!cbm_mkdtemp(g_tmpdir)) {
+        FAIL("failed to create temp dir");
+    }
+    for (int i = 0; i < 64; i++) {
+        char path[512];
+        snprintf(path, sizeof(path), "%s/f%02d.go", g_tmpdir, i);
+        FILE *f = fopen(path, "w");
+        if (!f) {
+            FAIL("failed to create fixture file");
+        }
+        fprintf(f, "package main\n\nfunc F%02d() int {\n\treturn %d\n}\n", i, i);
+        fclose(f);
+    }
+
+    /* 1 MB budget: over-budget on every pull, unreclaimable by napping. */
+    cbm_setenv("CBM_MEM_BUDGET_MB", "1", 1);
+    cbm_mem_init(0);
+    ASSERT_TRUE(cbm_mem_budget() > 0);
+    ASSERT_TRUE(cbm_mem_over_budget());
+
+    cbm_pp_bp_nap_cycles_reset();
+    cbm_pipeline_t *p = cbm_pipeline_new(g_tmpdir, NULL, CBM_MODE_FULL);
+    ASSERT_NOT_NULL(p);
+    int rc = cbm_pipeline_run(p);
+    long cycles = cbm_pp_bp_nap_cycles();
+
+    /* Restore the tests' default budget-off state BEFORE asserting. */
+    cbm_unsetenv("CBM_MEM_BUDGET_MB");
+    cbm_mem_init(0);
+    cbm_pipeline_free(p);
+    teardown_test_repo();
+
+    ASSERT_EQ(rc, 0);
+    /* Engagement guard (anti-vacuous): the gate must have actually run — the
+     * parallel path taken and the 1 MB budget exceeded on every pull. cycles==0
+     * means the fixture routed sequential (or the gate was compiled out) and
+     * this test proved NOTHING; fail loudly rather than pass vacuously. */
+    if (cycles < 1) {
+        FAIL("back-pressure gate never engaged (cycles==0) — fixture routed sequential?");
+    }
+    /* Futile napping must disengage: at most one in-flight cycle per worker
+     * plus a small margin, never one per file (64). */
+    long bound = (long)cbm_system_info().total_cores + 2;
+    if (cycles > bound) {
+        char msg[128];
+        snprintf(msg, sizeof(msg), "nap cycles %ld > bound %ld (gate re-paid per pull)", cycles,
+                 bound);
+        FAIL(msg);
+    }
+    PASS();
+}
+
 SUITE(pipeline) {
     /* Index lock */
     RUN_TEST(pipeline_lock_try_acquire);
@@ -6554,6 +6628,8 @@ SUITE(pipeline) {
     RUN_TEST(pipeline_cancel);
     RUN_TEST(pipeline_cancel_null);
     RUN_TEST(pipeline_run_null);
+    /* Extraction back-pressure */
+    RUN_TEST(pipeline_backpressure_futile_nap_disengages);
     /* File persistence */
     RUN_TEST(store_file_persistence);
     RUN_TEST(store_bulk_persistence);