From 3da5fe800a8f81311a42ad2f355129b6811bc680 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 25 Jun 2026 01:19:09 -0400 Subject: [PATCH] perf(incremental): batch file-hash upserts in one transaction persist_hashes (pipeline_incremental.c:452) upserted one file-hash row per file in autocommit -- a 10k-file reindex issued 10k separate COMMITs (each a WAL fsync). Wrap both upsert loops (current discovery + mode-skipped) in cbm_store_begin/commit so N files -> 1 COMMIT. Falls back to per-row autocommit if BEGIN fails (a caller already holds a transaction): 'batched' is gated on BEGIN's return and COMMIT runs only when BEGIN succeeded, so the existing partial-failure policy is unchanged. Fork-origin perf finding #6 (git blame: Andrew Hundt). Compiles clean (make cbm). Other perf findings are upstream-origin (cypher sort/DISTINCT, query_graph JSON, include_connected, properties re-parse -- DeusData #593/#601/#238) or fork-origin but higher-risk (deferred). Signed-off-by: Andrew Hundt --- src/pipeline/pipeline_incremental.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index c6a2024b5..b89c280f2 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -455,6 +455,12 @@ static void persist_hashes(cbm_store_t *store, const char *project, cbm_file_inf int current_failed = 0; int ms_failed = 0; + /* Batch all hash upserts in one transaction: N files -> 1 COMMIT under + * WAL instead of N autocommit fsyncs (a 10k-file reindex did 10k separate + * commits). If BEGIN fails (e.g. store busy), fall back to per-row + * autocommit. The partial-failure policy below is unchanged. */ + bool batched = (cbm_store_begin(store) == CBM_STORE_OK); + /* Current discovery: re-stat to capture any mtime/size that changed * during the run, and write fresh hash rows for visited files. */ for (int i = 0; i < file_count; i++) { @@ -498,6 +504,10 @@ static void persist_hashes(cbm_store_t *store, const char *project, cbm_file_inf } } + if (batched) { + (void)cbm_store_commit(store); + } + if (current_failed > 0 || ms_failed > 0) { cbm_log_warn("incremental.persist_summary", "current_failed", itoa_buf_incr(current_failed), "mode_skipped_failed", itoa_buf_incr(ms_failed));