fix(incremental): drop Process nodes no SPAWNS reaches any more

A Process node exists only as the target of a SPAWNS edge (a call or a
shell command starting a program) and belongs to no file. When the last
file starting the program was deleted or edited, an incremental re-index
removed the edge but kept the node, which a full index never makes: the
two diverged (gardener: deleting hack/tools.mk left install-promtool.sh
and install-protoc.sh behind).

Both incremental routes now drop such nodes: the full-load route on its
in-memory graph before the post-passes (new
cbm_gbuf_delete_unreferenced), the closure-repair route on the staging
database right after the patch, the first point where that route sees
the whole graph (its in-memory buffer holds no edges).

A full index is unchanged on 17 repositories (all graphs identical).
Deleting a makefile included by another, a script CI jobs run, or a
makefile a recipe makes into now leaves the graph a fresh index builds,
apart from SEMANTICALLY_RELATED edges, which differ as before.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/graph_buffer/graph_buffer.c

@@ -1498,6 +1498,64 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
     return deleted_count;
 }
 
+int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *edge_type) {
+    if (!gb || !label || !edge_type) {
+        return CBM_NOT_FOUND;
+    }
+    node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label);
+    if (!arr || arr->count == 0) {
+        return 0;
+    }
+    /* The victims first: the per-node unlinking below edits `arr`. */
+    cbm_gbuf_node_t **victims =
+        cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)arr->count * sizeof(*victims));
+    if (!victims) {
+        return CBM_NOT_FOUND;
+    }
+    int n = 0;
+    for (int i = 0; i < arr->count; i++) {
+        const cbm_gbuf_edge_t **edges = NULL;
+        int count = 0;
+        const char *qn = arr->items[i]->qualified_name;
+        cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL;
+        if (node &&
+            cbm_gbuf_find_edges_by_target_type(gb, node->id, edge_type, &edges, &count) == 0 &&
+            count == 0) {
+            victims[n++] = node;
+        }
+    }
+    if (n == 0) {
+        cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims);
+        return 0;
+    }
+    CBMHashTable *deleted_set = cbm_ht_create((uint32_t)n);
+    for (int i = 0; i < n; i++) {
+        cbm_gbuf_node_t *node = victims[i];
+        if (!node->qualified_name) {
+            continue;
+        }
+        char id_buf[CBM_SZ_32];
+        make_id_key(id_buf, sizeof(id_buf), node->id);
+        cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
+                   intptr_to_ptr(SKIP_ONE));
+        remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, node->label), node->id);
+        remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_name, node->name), node->id);
+        gb_drop_variant_spans(gb, node->qualified_name);
+        cbm_ht_delete(gb->node_by_qn, node->qualified_name);
+        if (node->id >= 0 && node->id < gb->by_id_cap) {
+            gb->by_id[node->id] = NULL;
+        }
+        /* As delete_by_file: a NULL QN fails the dump's liveness check. */
+        cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->qualified_name);
+        node->qualified_name = NULL;
+    }
+    cascade_delete_edges(gb, deleted_set);
+    cbm_ht_foreach(deleted_set, free_index_key, NULL);
+    cbm_ht_free(deleted_set);
+    cbm_free(CBM_MEM_CLASS_GBUF_INDEX, victims);
+    return n;
+}
+
 int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) {
     if (!gb || !db_path || !project) {
         return CBM_NOT_FOUND;

src/graph_buffer/graph_buffer.h

@@ -110,6 +110,12 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label);
  * Used by incremental indexing to remove stale nodes before re-extraction. */
 int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path);
 
+/* Delete every `label` node that no `edge_type` edge targets (with any edges
+ * it still has); returns how many. For nodes that exist only as such a
+ * target and carry no file (Process): incremental indexing drops them when
+ * their last referrer is gone, as a full index never makes them. */
+int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *edge_type);
+
 /* Bulk-load all nodes and edges for a project from an existing SQLite DB
  * into this graph buffer. Returns 0 on success. */
 int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project);

src/pipeline/pipeline_delta.c

@@ -247,6 +247,28 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *
     return 0;
 }
 
+int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
+                                const char *edge_type) {
+    sqlite3 *db = cbm_store_get_db(store);
+    if (!db) {
+        return CBM_NOT_FOUND;
+    }
+    sqlite3_stmt *stmt = NULL;
+    if (sqlite3_prepare_v2(db,
+                           "DELETE FROM nodes WHERE project = ?1 AND label = ?2 AND NOT EXISTS"
+                           " (SELECT 1 FROM edges e WHERE e.project = ?1"
+                           " AND e.target_id = nodes.id AND e.type = ?3)",
+                           CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
+        return CBM_NOT_FOUND;
+    }
+    sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT);
+    sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT);
+    sqlite3_bind_text(stmt, 3, edge_type, CBM_NOT_FOUND, SQLITE_TRANSIENT);
+    int rc = sqlite3_step(stmt);
+    sqlite3_finalize(stmt);
+    return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND;
+}
+
 /* Every project node becomes a proxy. The first cut filtered by label and
  * the fail-closed patch immediately found the counterexamples (synthetic
  * Decorator nodes on django; Macro is six of the kernel's 8.5M nodes) — any

src/pipeline/pipeline_incremental.c

@@ -3085,6 +3085,12 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
         goto out;
     }
     cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
+    /* The proxy buffer holds no edges, so only the patched staging graph can
+     * tell a Process node no SPAWNS reaches any more (its last spawner was
+     * purged). */
+    if (cbm_delta_drop_unreferenced(staging, project, "Process", "SPAWNS") < 0) {
+        goto out;
+    }
 
     /* Importance, in SQL, over the now-complete staging graph. This is the
      * FIRST point on this route where the whole project is queryable: the
@@ -3690,6 +3696,11 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
         phase_rc = cbm_pipeline_check_cancel(&ctx);
     }
     if (phase_rc == 0) {
+        /* A Process node exists only as a SPAWNS target (pass_spawns.c,
+         * pass_shell.c) and carries no file: when the last file starting the
+         * program is gone, nothing else removes it -- a full index never
+         * makes it. This buffer holds the whole graph. */
+        (void)cbm_gbuf_delete_unreferenced(existing, "Process", "SPAWNS");
         phase_rc = run_postpasses(&ctx, changed_files, ci, project, true);
     }
 

src/pipeline/pipeline_internal.h

@@ -1290,6 +1290,10 @@ int cbm_delta_snapshot_inbound(cbm_store_t *store, const char *project, const ch
 void cbm_delta_free_snapshot(cbm_delta_saved_edge_t *items, int count);
 int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *paths,
                     int path_count);
+/* Delete the project's `label` nodes no `edge_type` edge targets (a Process
+ * whose last spawner was purged); returns how many, or -1 on failure. */
+int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label,
+                                const char *edge_type);
 /* Pre-seed proxy nodes with their REAL database ids and move the gbuf id
  * watermark above MAX(id); returns that max id, or -1 on failure. */
 int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf);

tests/test_pipeline.c

@@ -3457,6 +3457,57 @@ TEST(incremental_shell_deploy_edit_keeps_its_edges) {
     PASS();
 }
 
+/* Deleting the only file that starts a program leaves no Process node
+ * behind -- a full index never makes one without a SPAWNS edge. One script
+ * (pass_shell.c) and one Python call (pass_spawns.c) go; the program a kept
+ * script starts stays. `legacy`: through the full-load route, else the
+ * closure-repair route. Returns the Process count after the deletion, or -1;
+ * *route: the route taken. */
+static int incremental_spawner_delete(bool legacy, cbm_incremental_route_t *route) {
+    char tmp[256];
+    snprintf(tmp, sizeof(tmp), "/tmp/cbm_spawngc_XXXXXX");
+    if (!cbm_mkdtemp(tmp)) {
+        return -1;
+    }
+    write_temp_file(tmp, "hack/a.sh", "#!/bin/sh\n./bin/tool-a --x\n");
+    write_temp_file(tmp, "hack/b.sh", "#!/bin/sh\n./bin/tool-b\n");
+    write_temp_file(tmp, "app.py",
+                    "import subprocess\n\n\ndef run():\n    subprocess.run([\"tool-c\"])\n");
+    char db_path[512];
+    snprintf(db_path, sizeof(db_path), "%s/spawngc.db", tmp);
+    static const char processes[] = "SELECT COUNT(*) FROM nodes WHERE label='Process'";
+    int full = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, processes) : -1;
+    char path[512];
+    snprintf(path, sizeof(path), "%s/hack/a.sh", tmp);
+    (void)remove(path);
+    snprintf(path, sizeof(path), "%s/app.py", tmp);
+    (void)remove(path);
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+    if (legacy) {
+        cbm_pipeline_incremental_test_force_legacy_partial_once();
+    }
+#else
+    (void)legacy;
+#endif
+    int after = k8s_inc_index(tmp, db_path) == 0 ? k8s_inc_count(db_path, processes) : -1;
+    *route = cbm_pipeline_incremental_test_last_route();
+    th_rmtree(tmp);
+    return full == 3 ? after : -1;
+}
+
+TEST(incremental_deleted_spawner_drops_its_process) {
+    cbm_incremental_route_t route = CBM_INCREMENTAL_ROUTE_NONE;
+    int left = incremental_spawner_delete(false, &route);
+    ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
+    ASSERT_EQ(left, 1);
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+    left = incremental_spawner_delete(true, &route);
+    ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL);
+    ASSERT_EQ(left, 1);
+#endif
+    PASS();
+}
+
 /* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage,
  * while the shared name registry must NOT leak those relations into other
  * languages' textual resolution: a Python call or identifier sharing the
@@ -19082,6 +19133,7 @@ SUITE(pipeline_semantic_manifest_repro) {
     RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419);
     RUN_TEST(incremental_k8s_edits_reach_other_files);
     RUN_TEST(incremental_shell_deploy_edit_keeps_its_edges);
+    RUN_TEST(incremental_deleted_spawner_drops_its_process);
     RUN_TEST(pipeline_sql_lineage_and_relation_isolation);
     RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage);
     RUN_TEST(pipeline_dbt_jinja_lineage);