trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-11-07 08:43:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix/issue-2175

Browse Source
This commit is contained in:
DavidHLP
2026-09-19 20:31:28 +08:00
committed by GitHub
parent 3b7f255921 59a05eb1bf
commit e176ae552b
70 changed files with 5761 additions and 820 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.cbm
+13 -1
View File
@@ -248,6 +248,7 @@ FOUNDATION_SRCS = \
src/foundation/compat_fs.c \
src/foundation/compat_regex.c \
src/foundation/mem.c \
src/foundation/mem_core.c \
src/foundation/diagnostics.c \
src/foundation/profile.c \
src/foundation/dump_verify.c \
@@ -275,6 +276,8 @@ EXTRACTION_SRCS = \
$(CBM_DIR)/extract_k8s.c \
$(CBM_DIR)/extract_dbt.c \
$(CBM_DIR)/helpers.c \
$(CBM_DIR)/result_compact.c \
$(CBM_DIR)/result_spill.c \
$(CBM_DIR)/lang_specs.c \
$(CBM_DIR)/macro_table.c \
$(CBM_DIR)/iris_export_xml.c \
@@ -1188,8 +1191,17 @@ lint-no-suppress:
lint: lint-tidy lint-cppcheck lint-format lint-no-suppress
@echo "=== All linters passed ==="
# Memory-core linter: memory is allocated through src/foundation/mem_core.h,
# not through raw malloc/calloc/realloc/free/strdup. A checked-in baseline
# (scripts/memory-core-baseline.txt) records each file's raw-site count and the
# gate goes red the moment any file grows. Files only ever go down; lower the
# baseline line in the same change that migrates the file.
lint-memory-core:
@echo "=== memory-core linter ==="
@python3 scripts/lint-memory-core.py
# CI linters (no clang-tidy — platform-dependent, enforced locally via pre-commit)
lint-ci: lint-cppcheck lint-format lint-no-suppress
lint-ci: lint-cppcheck lint-format lint-no-suppress lint-memory-core
@echo "=== CI linters passed ==="
# ── Local memory-diagnostic lanes (not PR-CI gates by decision: the diag
README.md
+2 -2
View File
@@ -3,7 +3,7 @@
[![GitHub Release](https://img.shields.io/github/v/release/DeusData/codebase-memory-mcp?style=flat&color=blue)](https://github.com/DeusData/codebase-memory-mcp/releases/latest)
[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
[![CI](https://img.shields.io/github/actions/workflow/status/DeusData/codebase-memory-mcp/dry-run.yml?label=CI)](https://github.com/DeusData/codebase-memory-mcp/actions/workflows/dry-run.yml)
[![Tests](https://img.shields.io/badge/tests-6768_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp)
[![Tests](https://img.shields.io/badge/tests-8050_passing-brightgreen)](https://github.com/DeusData/codebase-memory-mcp)
[![Languages](https://img.shields.io/badge/languages-162-orange)](https://github.com/DeusData/codebase-memory-mcp)
[![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp)
[![Agents](https://img.shields.io/badge/agent_surfaces-45-purple)](https://github.com/DeusData/codebase-memory-mcp)
@@ -425,7 +425,7 @@ scripts/build.sh # without the UI (development only)
Every platform ships **one self-contained executable**: the graph UI and the agent integration templates are linked into the binary, so an extracted archive is immediately complete.
Run the test suite (6,768 tests across 120 suites):
Run the test suite (8,060 tests across 141 suites):
```bash
scripts/test.sh # full: clean sanitizer build + all suites + guards
flake.nix
+1 -1
View File
@@ -63,7 +63,7 @@
packages = forAllSystems (pkgs: rec {
default = pkgs.stdenv.mkDerivation {
pname = "codebase-memory-mcp";
version = "0.10.8";
version = "0.11.0";
src = ./.;
internal/cbm/arena.h
+7 -41
View File
@@ -1,41 +1,7 @@
#ifndef CBM_ARENA_H
#define CBM_ARENA_H
#include <stddef.h>
// CBMArena is a simple bump allocator that allocates from fixed-size blocks.
// All memory is freed at once via cbm_arena_destroy(). Individual frees are not
// supported — this is by design for per-file extraction where all data has the
// same lifetime.
#define CBM_ARENA_MAX_BLOCKS 256
#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial
typedef struct {
char *blocks[CBM_ARENA_MAX_BLOCKS];
size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats)
int nblocks;
size_t block_size;
size_t used; // bytes used in current block
size_t total_alloc; // cumulative bytes allocated (for stats)
} CBMArena;
// Initialize an arena with the default block size.
void cbm_arena_init(CBMArena *a);
// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion.
// All returned pointers are 8-byte aligned.
void *cbm_arena_alloc(CBMArena *a, size_t n);
// Duplicate a string into arena memory. Returns arena-owned copy.
char *cbm_arena_strdup(CBMArena *a, const char *s);
// Duplicate a string of known length into arena memory. NUL-terminates.
char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len);
// sprintf into arena memory. Returns arena-owned string.
char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
// Free all blocks. Arena is invalid after this call.
void cbm_arena_destroy(CBMArena *a);
#endif // CBM_ARENA_H
/* The arena lives in src/foundation/arena.h. This file used to be a stale
* copy with the same include guard and a shorter API; whichever header a
* translation unit included first won, silently. One definition now. */
#ifndef CBM_INTERNAL_ARENA_SHIM_H
#define CBM_INTERNAL_ARENA_SHIM_H
#include "../../src/foundation/arena.h" /* relative: the lsp_all unit has no -Isrc */
#endif /* CBM_INTERNAL_ARENA_SHIM_H */
internal/cbm/cbm.c
+187 -25
View File
@@ -1,7 +1,9 @@
/* Full declaration set for the same CBMArena, and it must precede cbm.h:
* internal/cbm/arena.h declares a subset and the two share the CBM_ARENA_H
* guard, so whichever is included first is the one this file sees. */
#include "foundation/arena.h" // cbm_arena_init_sized
#include "foundation/arena.h" // cbm_arena_init_sized
#include "foundation/mem_core.h" // class accounting for the bound allocators
#include "foundation/log.h" // cbm_log_warn -- extract.lsp.skipped
#include "cbm.h"
#include "arena.h" // CBMArena, cbm_arena_init/alloc/strdup/destroy
#include "helpers.h"
@@ -222,6 +224,13 @@ static const char *cbm_string_read(void *payload, uint32_t byte, TSPoint point,
* edge). A generous WALL ceiling stays as a backstop so a genuinely
* stuck/spinning parse still terminates in bounded time. */
#define CBM_PARSE_WALL_CEILING_FACTOR 12ULL /* ~60 s ceiling for the 5 s CPU budget */
/* A parse that used more than 1/N of the per-file budget disqualifies the file
* from the unbudgeted LSP walks (see cbm_extract_file_ex). */
#define CBM_LSP_BUDGET_SHARE_DIV 2ULL
/* The unified walk may spend this many parse budgets of thread CPU time: wide
* enough for a 7,873-definition reference file (~10 s), tight enough to stop the
* generated JIT tests (65-350 s). */
#define CBM_WALK_BUDGET_FACTOR 6ULL
typedef struct {
uint64_t cpu_deadline_ns; // trip once this thread's CPU time passes it
@@ -291,6 +300,20 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang)
* to mimalloc would mismatch ASan/CRT frees — there these binds compile to
* no-ops and the build stays unchanged. */
/* SQLite on a dedicated mimalloc heap per thread: ON only in the index worker,
* whose default heap holds the graph (SQLite churn on that heap paid a page
* walk per allocation: 132 s vs 9.7 s on the kernel's coverage publish). OFF
* everywhere else: the daemon runs a thread per connection, and a heap
* created per such thread pins every SQLite block the shared connection
* keeps (page cache, statement cache) to pages nobody's heap owns any more --
* the Linux soak grew 180 KB per query, 11 -> 144 MB in ten minutes, where
* the default thread heap had been flat (2026-09-14). */
static _Atomic int g_sqlite_dedicated_heap;
void cbm_sqlite_dedicated_heap(bool on) {
atomic_store_explicit(&g_sqlite_dedicated_heap, on ? 1 : 0, memory_order_relaxed);
}
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
#include <assert.h>
@@ -303,16 +326,49 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang)
* hook here the biggest per-request allocations in the process — SQLite's page
* cache and its query working set — are invisible to the attribution profile
* (#581). */
/* SQLite allocates from a mimalloc heap of its own, one per thread. Its page
* queues then hold SQLite blocks only. Sharing the thread's default heap with
* the graph -- 40M+ blocks, millions of pages once the graph moved onto the
* core -- made every statement-journal chunk of the coverage publish step pay
* a walk over the graph's pages: 132 s on the kernel where v0.10.8, whose
* graph lived outside mimalloc, took 9.7 s (sampled 2026-09-14). mi_free
* works across heaps, so xFree and cross-thread frees are unchanged; a
* thread's heap is released with the thread. */
static _Thread_local mi_heap_t *tl_sqlite_heap;
/* NULL = the calling thread's default heap (mi_malloc); see the switch above. */
static mi_heap_t *sqlite_heap(void) {
if (!atomic_load_explicit(&g_sqlite_dedicated_heap, memory_order_relaxed)) {
return NULL;
}
if (!tl_sqlite_heap) {
tl_sqlite_heap = mi_heap_new();
}
return tl_sqlite_heap;
}
static void *cbm_sqlite_malloc(int n) {
void *block = mi_malloc((size_t)n);
mi_heap_t *heap = sqlite_heap();
void *block = heap ? mi_heap_malloc(heap, (size_t)n) : mi_malloc((size_t)n);
if (block) {
cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(block));
}
return block;
}
static void cbm_sqlite_free(void *p) {
if (p) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, mi_usable_size(p));
}
mi_free(p);
}
static void *cbm_sqlite_realloc(void *p, int n) {
if (p) {}
void *grown = mi_realloc(p, (size_t)n);
size_t old_size = p ? mi_usable_size(p) : 0;
mi_heap_t *heap = sqlite_heap();
void *grown = heap ? mi_heap_realloc(heap, p, (size_t)n) : mi_realloc(p, (size_t)n);
if (grown) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_STORE, old_size);
cbm_mem_class_add_external(CBM_MEM_CLASS_STORE, mi_usable_size(grown));
}
return grown;
}
static int cbm_sqlite_size(void *p) {
@@ -325,18 +381,31 @@ static int cbm_sqlite_roundup(int n) {
* through these, and they too skip the interposer. */
static void *cbm_ts_malloc(size_t n) {
void *block = mi_malloc(n);
if (block) {
cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block));
}
return block;
}
static void *cbm_ts_calloc(size_t count, size_t size) {
void *block = mi_calloc(count, size);
if (block) {
cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(block));
}
return block;
}
static void *cbm_ts_realloc(void *p, size_t n) {
if (p) {}
size_t old_size = p ? mi_usable_size(p) : 0;
void *grown = mi_realloc(p, n);
if (grown) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, old_size);
cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(grown));
}
return grown;
}
static void cbm_ts_free(void *p) {
if (p) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, mi_usable_size(p));
}
mi_free(p);
}
@@ -1575,13 +1644,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
const CBMReturnTypeTable *return_type_table,
CBMArena *scratch) {
// Allocate result on heap (arena inside for all string data)
enum { SINGLE = 1 };
CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
CBMFileResult *result = cbm_result_alloc();
if (!result) {
return NULL;
}
cbm_arena_init(&result->arena);
cbm_work_arena_take(&result->arena);
CBMArena *a = &result->arena;
/* Crash-quarantine hard guard (Stage 3c): a file the supervisor pinned as a
@@ -1642,6 +1710,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
ts_parser_reset(parser);
uint64_t t0 = now_ns();
uint64_t cpu_start_ns = cbm_thread_cpu_time_ns();
// Build string input + timeout options for parse_with_options
CBMStringInput str_input = {source, (uint32_t)source_len};
@@ -1654,8 +1723,9 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
TSParseOptions opts = {0};
CBMParseBudget budget = {0}; // cppcheck-suppress unreadVariable
uint64_t budget_ns = 0;
if (timeout_micros > 0) {
uint64_t budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC;
budget_ns = (uint64_t)timeout_micros * USEC_TO_NSEC;
// Descheduling burns wall time but not CPU: gate on this thread's CPU
// time so a starved-but-parseable file is not abandoned, with a generous
// wall ceiling as a backstop against a genuinely spinning/stuck parse.
@@ -1688,6 +1758,33 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
TSNode root = ts_tree_root_node(tree);
/* Parse-budget share. A file whose parse alone consumed more than
* 1/CBM_LSP_BUDGET_SHARE_DIV of its budget is too large for the per-file
* LSP walk that follows: the walk is superlinear in expression size and
* has no budget of its own (C#, a 23 MB single-expression JIT test: 354 s
* in the walk, then a crash in the cross-file resolve on the same tree,
* 2026-09-14 -- the parse used to time out at 5 s and hide both). The
* unified extractor's defs stay; the LSP refinement here and the
* cross-file resolve (cbm_pxc_dispatch_file) skip the file, logged. The
* budget is the same for every parser, so the rule is too. */
bool lsp_skipped = timeout_micros > 0 && (t1 - t0) * CBM_LSP_BUDGET_SHARE_DIV > budget_ns;
#ifdef CBM_ENABLE_TEST_SEAMS
{
const char *skip_on = getenv("CBM_TEST_LSP_SKIP_ON");
if (skip_on && skip_on[0] && rel_path && strstr(rel_path, skip_on)) {
lsp_skipped = true; /* the test names the file; no real timing involved */
}
}
#endif
if (lsp_skipped) {
char parse_ms[CBM_SZ_32];
snprintf(parse_ms, sizeof(parse_ms), "%llu",
(unsigned long long)((t1 - t0) / CBM_NSEC_PER_MSEC));
cbm_log_warn("extract.lsp.skipped", "reason", "parse_budget", "parse_ms", parse_ms, "path",
rel_path ? rel_path : "");
result->lsp_skipped = true;
}
// Compute module QN. Java/Go derive the module from the CONTAINING
// DIRECTORY (package semantics) rather than baking the filename stem in,
// so def QNs, the LSP caller_qn, and the textual calls-enclosing QN all
@@ -1710,6 +1807,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
.root = root,
.macro_table = macro_table,
.return_type_table = return_type_table,
.walk_deadline_cpu_ns =
timeout_micros > 0 ? cbm_thread_cpu_time_ns() + budget_ns * CBM_WALK_BUDGET_FACTOR : 0,
};
// Run extractors: defs + imports use separate walks (unique recursion patterns),
@@ -1717,6 +1816,21 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
cbm_extract_definitions(&ctx);
cbm_extract_imports(&ctx);
cbm_extract_unified(&ctx);
if (ctx.walk_budget_exhausted) {
result->walk_truncated = true;
result->lsp_skipped = true;
cbm_log_warn("extract.walk.truncated", "reason", "cpu_budget", "path",
rel_path ? rel_path : "");
}
/* A file that spent the budget on parse plus walk is too heavy for the
* unbudgeted LSP walks as well (the C# JIT test files: 65-73 s each in
* the per-file walk after a parse under the share rule). */
if (!result->lsp_skipped && timeout_micros > 0 &&
cbm_thread_cpu_time_ns() - cpu_start_ns > budget_ns) {
result->lsp_skipped = true;
cbm_log_warn("extract.lsp.skipped", "reason", "file_budget", "path",
rel_path ? rel_path : "");
}
// Channel detection (Socket.IO / EventEmitter) — JS/TS only.
cbm_extract_channels(&ctx);
@@ -1736,7 +1850,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
// LSP type-aware call/usage resolution (per-file). Runs in every mode;
// refines the tree-sitter + textual-resolution graph with type info.
uint64_t lsp_start = now_ns();
{
if (!result->lsp_skipped) {
if (language == CBM_LANG_GO) {
cbm_run_go_lsp(a, result, source, source_len, root);
}
@@ -1775,15 +1889,15 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
if (language == CBM_LANG_CSHARP) {
cbm_run_cs_lsp(a, result, source, source_len, root);
}
}
if (language == CBM_LANG_JAVA) {
cbm_run_java_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_KOTLIN) {
cbm_run_kotlin_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_RUST) {
cbm_run_rust_lsp(a, result, source, source_len, root);
if (language == CBM_LANG_JAVA) {
cbm_run_java_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_KOTLIN) {
cbm_run_kotlin_lsp(a, result, source, source_len, root);
}
if (language == CBM_LANG_RUST) {
cbm_run_rust_lsp(a, result, source, source_len, root);
}
}
atomic_fetch_add(&total_lsp_ns, now_ns() - lsp_start);
@@ -2156,6 +2270,40 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
return result;
}
/* ── Per-worker working arena (see cbm.h) ───────────────────────────── */
static CBM_TLS CBMArena tl_work_arena;
static CBM_TLS bool tl_work_arena_live = false;
void cbm_work_arena_take(CBMArena *into) {
if (tl_work_arena_live) {
*into = tl_work_arena;
tl_work_arena_live = false;
cbm_arena_rewind(into);
return;
}
cbm_arena_init(into);
}
void cbm_work_arena_release(void) {
if (tl_work_arena_live) {
cbm_arena_destroy(&tl_work_arena);
tl_work_arena_live = false;
}
}
void cbm_work_arena_give(CBMArena *from) {
if (!from || from->nblocks == 0) {
return;
}
if (tl_work_arena_live || cbm_arena_capacity(from) > (size_t)CBM_WORK_ARENA_KEEP_BYTES) {
cbm_arena_destroy(from);
return;
}
tl_work_arena = *from;
tl_work_arena_live = true;
memset(from, 0, sizeof(*from));
}
/* Public entry. Owns the traversal scratch arena for the whole of one file's
* extraction: created here, handed to the body as ctx->scratch, destroyed on
* the way out. The body has seven early returns, so bracketing it in a wrapper
@@ -2176,6 +2324,25 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
return result;
}
CBMFileResult *cbm_result_alloc(void) {
/* The one raw allocation of a result: cbm_free_result releases it with
* the matching free. Extraction and the spill loader both come here. */
enum { SINGLE = 1 };
return (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
}
void cbm_result_release_owned(CBMFileResult *result) {
if (!result) {
return;
}
for (int i = 0; i < result->owned_result_count; i++) {
cbm_free_result(result->owned_results[i]);
}
free(result->owned_results);
result->owned_results = NULL;
result->owned_result_count = 0;
}
void cbm_free_result(CBMFileResult *result) {
if (!result) {
return;
@@ -2184,12 +2351,7 @@ void cbm_free_result(CBMFileResult *result) {
ts_tree_delete(result->cached_tree);
result->cached_tree = NULL;
}
for (int i = 0; i < result->owned_result_count; i++) {
cbm_free_result(result->owned_results[i]);
}
free(result->owned_results);
result->owned_results = NULL;
result->owned_result_count = 0;
cbm_result_release_owned(result);
cbm_arena_destroy(&result->arena);
free(result);
}
internal/cbm/cbm.h
+93 -32
View File
@@ -248,31 +248,36 @@ typedef enum {
} CBMSourceOrigin;
typedef struct {
const char *callee_name; // raw callee text ("pkg.Func", "foo")
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL
const char *second_arg_name; // second argument identifier (handler ref) or NULL
CBMCallArg args[CBM_MAX_CALL_ARGS]; // first N arguments with expressions
int arg_count; // number of captured arguments
int loop_depth; // enclosing loop nesting at the call site
int branch_depth; // enclosing branch nesting at the call site
int start_line; // 1-based source line of the call (for def range-match)
uint32_t site_start_byte; // exact AST occurrence span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_method; // method/member call with an UNRESOLVED receiver. Perl:
// arrow/method call ($obj->m). TS/JS/TSX: member call
// x.foo() whose receiver is not this/super. Python:
// x.foo() where x is not self/cls/super() and is not
// rooted in an imported name. Read by the weak-member
// guard and by the pxc synthetic-carrier dedup key in
// pass_lsp_cross.c. Default false.
bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit
// C++ operator). Never fall back to textual resolution.
bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a
// parameter of an enclosing function, so it cannot be the
// module-level foo. Python only today. Read by the
// weak-local-binding guard. Default false.
const char *callee_name; // raw callee text ("pkg.Func", "foo")
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *first_string_arg; // first string literal argument (URL, topic, key) or NULL
const char *second_arg_name; // second argument identifier (handler ref) or NULL
/* First arg_count captured arguments, arena-allocated on first capture;
* NULL when arg_count == 0. Was an inline args[CBM_MAX_CALL_ARGS] (256 of
* the record's 320 bytes) -- the Go corpus census (2026-09-13) put 825k
* calls at 251 MB with most of that empty slots. Readers index it exactly
* as before; only `sizeof` changed. */
CBMCallArg *args;
int arg_count; // number of captured arguments (<= CBM_MAX_CALL_ARGS)
int loop_depth; // enclosing loop nesting at the call site
int branch_depth; // enclosing branch nesting at the call site
int start_line; // 1-based source line of the call (for def range-match)
uint32_t site_start_byte; // exact AST occurrence span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_method; // method/member call with an UNRESOLVED receiver. Perl:
// arrow/method call ($obj->m). TS/JS/TSX: member call
// x.foo() whose receiver is not this/super. Python:
// x.foo() where x is not self/cls/super() and is not
// rooted in an imported name. Read by the weak-member
// guard and by the pxc synthetic-carrier dedup key in
// pass_lsp_cross.c. Default false.
bool requires_lsp_resolution; // synthetic semantic candidate (for example an implicit
// C++ operator). Never fall back to textual resolution.
bool callee_is_locally_bound; // bare call foo() whose callee identifier is bound as a
// parameter of an enclosing function, so it cannot be the
// module-level foo. Python only today. Read by the
// weak-local-binding guard. Default false.
} CBMCall;
typedef struct {
@@ -286,16 +291,18 @@ typedef enum {
} CBMUsageKind;
typedef struct {
const char *ref_name; // referenced identifier
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
const char *ref_name; // referenced identifier
const char *enclosing_func_qn; // QN of enclosing function (or module QN)
/* Fixed-width fields grouped so the record packs to 40 bytes (was 48; the
* Go corpus holds 4.68M of these). Field meanings unchanged. */
uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity
uint32_t site_start_byte; // exact reference-token span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMUsageKind kind; // ordinary USAGE or explicit callable reference
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool may_be_call_reference; // syntactic candidate; exact LSP proof may upgrade its edge
bool semantic_reference_blocked; // lexical evidence blocks only unproven textual fallback
bool semantic_reference_local_shadow; // blocker belongs to a non-module lexical scope
uint32_t lexical_scope_id; // extraction-local scope instance; never graph identity
uint32_t site_start_byte; // exact reference-token span; end > start when present
uint32_t site_end_byte; // exclusive byte offset in the source file
CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer
bool is_member_access; // token is the member half of a selector/attribute
// (Go x.f — field_identifier). The extractor strips
// the receiver, so this is the only surviving record
@@ -541,7 +548,16 @@ typedef struct CBMFileResult {
int error_region_count;
bool is_test_file;
int imports_count;
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
TSTree *cached_tree; // retained parse tree (caller frees via cbm_free_tree)
/* The parse alone used more than its share of the per-file budget: the
* per-file LSP walk and the cross-file resolve skip this file (its
* unified-extractor defs stay). Set by cbm_extract_file_ex, honoured by
* cbm_pxc_dispatch_file -- one site for every language. */
bool lsp_skipped;
/* The unified walk stopped at its CPU budget: defs/calls/usages found up
* to that point are kept, the rest of the file is not walked. Implies
* lsp_skipped. */
bool walk_truncated;
CBMLanguage cached_lang; // language of cached tree (for parser selection)
// Retained source bytes — copied into `arena` by the parallel
@@ -632,6 +648,14 @@ typedef struct {
* class-body variable def records which class declares it (parent_class)
* without changing its module-level qualified name. NULL elsewhere. */
const char *var_parent_class;
/* Per-file walk budget (thread CPU time, ns; 0 = unbounded). The unified
* cursor walk checks it every 1024 nodes and stops when it is spent, so
* no single file can hold a worker for minutes: a 23 MB single-expression
* C# test file cost 346 s in usage stamping alone (tree-sitter's
* ts_node_parent descends from the root, quadratic on a deep tree;
* 2026-09-14). What was extracted before the stop is kept. */
uint64_t walk_deadline_cpu_ns;
bool walk_budget_exhausted;
} CBMExtractCtx;
// --- Public API ---
@@ -645,6 +669,12 @@ typedef struct {
// also calls it so non-main entry points (pipeline passes) still get the binds.
// In the test build (no CBM_BIND_TS_ALLOCATOR) this is a no-op.
void cbm_alloc_init(void);
/* SQLite allocates from a dedicated mimalloc heap per thread while on; the
* index worker turns it on (its default heap holds the graph). Off elsewhere:
* a thread-per-connection daemon would pin connection-lifetime blocks to
* dead threads. The switch exists in every build; it changes nothing where
* the allocator binds are compiled out. */
void cbm_sqlite_dedicated_heap(bool on);
// Initialize the library. Call once at startup. Returns 0 on success.
int cbm_init(void);
@@ -675,6 +705,31 @@ void cbm_index_mark_done(const char *rel_path);
// Extract all data from one file. Caller must call cbm_free_result().
// source must remain valid for the duration of the call.
// timeout_micros: per-file parse timeout in microseconds (0 = no timeout).
/* Compact a finished result: copy everything reachable from it -- every
* record array at exact count, every string once (interned by content within
* the file), the retained source -- into one exact-size arena, and destroy the
* working arena the extractors wrote into. Measured on the Go corpus
* (2026-09-13): 14.8 GB written per index, 3.4 GB reachable; the rest was
* node-text copies and abandoned array generations no one could free because
* the result owned the arena. Call once, after the last per-file write and
* before the result is cached for later passes. Later appends into the arena
* still work (growth restarts at the default block). A composite's owned
* per-unit results are released: after the deep copy nothing points at them.
* On allocation failure the result is left exactly as it was. */
void cbm_result_compact(CBMFileResult *result);
/* The working arena the extractors write into, per worker thread. Extraction
* takes it (rewound, pages still mapped) instead of allocating a fresh arena
* per file; compaction returns it instead of destroying it. Reusing the same
* addresses directly is what stops the purge/re-commit churn that kept a
* kernel worker at 15 GB resident with 4-5 GB charged. An arena that grew past
* CBM_WORK_ARENA_KEEP_BYTES (one giant file) is destroyed, not kept. */
enum { CBM_WORK_ARENA_KEEP_BYTES = 16 * 1024 * 1024 };
void cbm_work_arena_take(CBMArena *into);
void cbm_work_arena_give(CBMArena *from);
/* Drop this thread's kept working arena (end of an extraction pass). */
void cbm_work_arena_release(void);
CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path, int64_t timeout_micros,
const char **extra_defines, // NULL-terminated, or NULL
@@ -696,6 +751,12 @@ CBMFileResult *cbm_extract_file_ex(
// Free all memory associated with a result.
void cbm_free_result(CBMFileResult *result);
/* Allocate an empty result; cbm_free_result releases it. */
CBMFileResult *cbm_result_alloc(void);
/* Release a composite result's per-unit results (the owner of that array). */
void cbm_result_release_owned(CBMFileResult *result);
// Free only the cached tree from a result (caller retained it for reuse).
void cbm_free_tree(CBMFileResult *result);
internal/cbm/extract_calls.c
+13
View File
@@ -2059,6 +2059,12 @@ static void extract_call_args(CBMExtractCtx *ctx, TSNode args, CBMCall *call) {
for (uint32_t ai = 0; ai < argc && call->arg_count < CBM_MAX_CALL_ARGS; ai++) {
TSNode arg_node = ts_node_named_child(args, ai);
const char *ak = ts_node_type(arg_node);
if (!call->args) {
call->args = cbm_arena_calloc(ctx->arena, CBM_MAX_CALL_ARGS * sizeof(CBMCallArg));
if (!call->args) {
return;
}
}
CBMCallArg *ca = &call->args[call->arg_count];
memset(ca, 0, sizeof(*ca));
@@ -3768,6 +3774,13 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML
if (strcmp(ack, "method_arg") != 0) {
continue;
}
if (!call.args) {
call.args = cbm_arena_calloc(ctx->arena,
CBM_MAX_CALL_ARGS * sizeof(CBMCallArg));
if (!call.args) {
break;
}
}
CBMCallArg *ca = &call.args[call.arg_count];
memset(ca, 0, sizeof(*ca));
ca->index = call.arg_count;
internal/cbm/extract_unified.c
+26
View File
@@ -5,6 +5,8 @@
#include "lang_specs.h" // CBMLangSpec, cbm_lang_spec, CBM_LANG_*
#include "tree_sitter/api.h" // TSNode, TSTreeCursor, ts_tree_cursor_*, ts_node_*
#include "foundation/constants.h"
#include "foundation/compat.h" // cbm_thread_cpu_time_ns
#include <stdlib.h>
enum { MAX_INFRA_BINDINGS = 8 };
@@ -2596,9 +2598,33 @@ void cbm_extract_unified(CBMExtractCtx *ctx) {
state.branch_depth = 0;
uint32_t depth = 0;
uint32_t visited = 0;
#ifdef CBM_ENABLE_TEST_SEAMS
/* CBM_TEST_WALK_BUDGET_NODES=<n>: the budget is "spent" after n nodes,
* no real timing involved. */
uint32_t seam_budget_nodes = 0;
{
const char *seam = getenv("CBM_TEST_WALK_BUDGET_NODES");
if (seam && seam[0]) {
seam_budget_nodes = (uint32_t)strtoul(seam, NULL, 10);
}
}
#endif
for (;;) {
TSNode node = ts_tree_cursor_current_node(&cursor);
visited++;
if (ctx->walk_deadline_cpu_ns != 0 && (visited & 1023u) == 0 &&
cbm_thread_cpu_time_ns() > ctx->walk_deadline_cpu_ns) {
ctx->walk_budget_exhausted = true;
break;
}
#ifdef CBM_ENABLE_TEST_SEAMS
if (seam_budget_nodes != 0 && visited > seam_budget_nodes) {
ctx->walk_budget_exhausted = true;
break;
}
#endif
bool trivia = is_unified_trivia_node(node);
if (!trivia) {
/* Trivia consumes no semantic state. Scope expiry may be deferred
internal/cbm/lsp/c_lsp.c
+44 -44
View File
@@ -63,6 +63,7 @@ void c_lsp_init(CLSPContext *ctx, CBMArena *arena, const char *source, int sourc
ctx->source = source;
ctx->source_len = source_len;
ctx->registry = registry;
ctx->registry_head = (CBMTypeRegistry *)registry;
ctx->module_qn = module_qn;
ctx->module_qn_len = module_qn ? strlen(module_qn) : 0;
ctx->enclosing_func_qn = module_qn;
@@ -2819,10 +2820,10 @@ static const CBMRegisteredFunc *c_lookup_member_depth(CLSPContext *ctx, const ch
size_t slen = strlen(shortn);
const char *best_qn = NULL;
CBMTypeShortIter it;
cbm_registry_types_by_short_name(ctx->registry, shortn, &it);
cbm_registry_types_by_short_name_chain(ctx->registry, shortn, &it);
int i;
while ((i = cbm_type_short_iter_next(&it)) >= 0) {
const char *q = ctx->registry->types[i].qualified_name;
const char *q = it.reg->types[i].qualified_name;
if (!q) {
continue;
}
@@ -3670,12 +3671,12 @@ static const CBMRegisteredFunc *c_lookup_free_operator(CLSPContext *ctx, const c
const char *right_ns = extract_namespace_from_qn(ctx->arena, right_qn);
const CBMRegisteredFunc *match = NULL;
CBMFreeFuncIter it;
cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it);
for (int index = cbm_free_func_iter_next(&it); index >= 0;
index = cbm_free_func_iter_next(&it)) {
if (index >= ctx->registry->func_count)
if (index >= it.reg->func_count)
continue;
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index];
const CBMRegisteredFunc *candidate = &it.reg->funcs[index];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) {
continue;
@@ -3730,13 +3731,13 @@ static const CBMRegisteredFunc *c_lookup_free_unary_operator(CLSPContext *ctx,
const CBMRegisteredFunc *match = NULL;
const CBMType *actuals[1] = {operand_type};
CBMFreeFuncIter it;
cbm_registry_free_funcs_by_short_name(ctx->registry, operator_name, &it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, operator_name, &it);
for (int index = cbm_free_func_iter_next(&it); index >= 0;
index = cbm_free_func_iter_next(&it)) {
if (index >= ctx->registry->func_count) {
if (index >= it.reg->func_count) {
continue;
}
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[index];
const CBMRegisteredFunc *candidate = &it.reg->funcs[index];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, operator_name) != 0 || !candidate->qualified_name) {
continue;
@@ -4882,18 +4883,16 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
// indexing bitcoin). Cross-phase template deduction then relies on the
// positional fallback, which is graceful degradation.
if (ctx->in_template && ctx->template_param_count > 0 && !ctx->registry_shared) {
// Find the registered function and set type_param_names
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) {
CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri];
if (strcmp(rf->qualified_name, func_qn) == 0 && !rf->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
for (int ti = 0; ti < ctx->template_param_count; ti++)
tpn[ti] = ctx->template_param_names[ti];
tpn[ctx->template_param_count] = NULL;
rf->type_param_names = tpn;
break;
}
// Set type_param_names on the registered function -- on the head's
// own copy of it (copy-on-write), never on a shared base entry.
CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn);
if (rf && !rf->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
for (int ti = 0; ti < ctx->template_param_count; ti++)
tpn[ti] = ctx->template_param_names[ti];
tpn[ctx->template_param_count] = NULL;
rf->type_param_names = tpn;
}
}
@@ -4929,12 +4928,9 @@ static void c_process_function(CLSPContext *ctx, TSNode func_node) {
}
// Set min_params on the registered function (for default-arg overload matching)
if (total_params > 0 && defaulted_params > 0) {
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->func_count; ri++) {
CBMRegisteredFunc *rf = &((CBMTypeRegistry *)ctx->registry)->funcs[ri];
if (strcmp(rf->qualified_name, func_qn) == 0 && rf->min_params < 0) {
rf->min_params = total_params - defaulted_params;
break;
}
CBMRegisteredFunc *rf = cbm_registry_func_for_update(ctx->registry_head, func_qn);
if (rf && rf->min_params < 0) {
rf->min_params = total_params - defaulted_params;
}
}
@@ -5059,7 +5055,7 @@ static void c_process_body_child(CLSPContext *ctx, TSNode child) {
tpn[ctx->template_param_count] = NULL;
rf.type_param_names = tpn;
}
cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf);
cbm_registry_add_func(ctx->registry_head, rf);
}
}
}
@@ -5161,14 +5157,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
// Store template param names on the registered type (for substitution)
if (ctx->in_template && ctx->template_param_names && ctx->template_param_count > 0) {
CBMRegisteredType *rt = NULL;
for (int ri = 0; ri < ((CBMTypeRegistry *)ctx->registry)->type_count; ri++) {
if (strcmp(((CBMTypeRegistry *)ctx->registry)->types[ri].qualified_name,
class_qn) == 0) {
rt = &((CBMTypeRegistry *)ctx->registry)->types[ri];
break;
}
}
CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn);
if (rt && !rt->type_param_names) {
const char **tpn = (const char **)cbm_arena_alloc(
ctx->arena, (ctx->template_param_count + 1) * sizeof(const char *));
@@ -5341,14 +5330,25 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
should_upgrade = true;
}
if (should_upgrade) {
// Update existing entry's signature return type
const CBMType **new_rets = (const CBMType **)cbm_arena_alloc(
ctx->arena, 2 * sizeof(const CBMType *));
new_rets[0] = actual_ret;
new_rets[1] = NULL;
CBMRegisteredFunc *mut = (CBMRegisteredFunc *)existing;
mut->signature = cbm_type_func_replace_returns(
ctx->arena, existing->signature, new_rets);
/* Refine the return type on the HEAD copy only
* (copy-on-write). `existing` may live in the
* sealed shared base; writing a scratch-arena
* signature into it was read by other workers
* after this file's arena died (ASan heap-use-
* after-free in c_adl_resolve on dotnet/runtime,
* 2026-09-14). NULL = the head is sealed too:
* skip the refinement. */
CBMRegisteredFunc *mut = cbm_registry_func_for_update(
ctx->registry_head, existing->qualified_name);
if (mut) {
const CBMType **new_rets =
(const CBMType **)cbm_arena_alloc(
ctx->arena, 2 * sizeof(const CBMType *));
new_rets[0] = actual_ret;
new_rets[1] = NULL;
mut->signature = cbm_type_func_replace_returns(
ctx->arena, mut->signature, new_rets);
}
}
break;
}
@@ -5364,7 +5364,7 @@ static void c_process_class(CLSPContext *ctx, TSNode class_node) {
rf.receiver_type = ctx->enclosing_class_qn;
rf.signature = cbm_type_func(ctx->arena, NULL, NULL, rets);
rf.min_params = -1;
cbm_registry_add_func((CBMTypeRegistry *)ctx->registry, rf);
cbm_registry_add_func(ctx->registry_head, rf);
break;
} else if (strcmp(dk, "reference_declarator") == 0) {
actual_ret = cbm_type_reference(ctx->arena, actual_ret);
internal/cbm/lsp/c_lsp.h
+8 -3
View File
@@ -13,6 +13,11 @@ typedef struct {
const char *source;
int source_len;
const CBMTypeRegistry *registry;
/* The writable head of the registry chain: the per-file overlay the
* dispatcher hands in, or the per-file registry on the non-cross path.
* Every refinement (lazy add, min_params, template params) goes here
* via cbm_registry_*_for_update; nothing behind ->fallback is written. */
CBMTypeRegistry *registry_head;
CBMScope *current_scope;
// Include map: header_path -> namespace QN prefix
@@ -94,9 +99,9 @@ typedef struct {
// READ-ONLY across resolve workers — never mutate it
// (and never store per-worker arena pointers into it)
bool debug;
int eval_depth; // recursion depth for c_eval_expr_type (crash guard)
int eval_steps; // total expression eval calls for current file (hang guard)
int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting)
int eval_depth; // recursion depth for c_eval_expr_type (crash guard)
int eval_steps; // total expression eval calls for current file (hang guard)
int walk_depth; // c_resolve_calls_in_node self-recursion (AST nesting)
int control_flow_depth; // if/loop/switch/catch nesting; assignments merge fail-closed
} CLSPContext;
internal/cbm/lsp/cs_lsp.c
+16 -8
View File
@@ -492,10 +492,10 @@ const char *cs_resolve_type_name(CSLSPContext *ctx, const char *raw) {
CBMTypeShortIter ts_it;
int i;
if (ctx->registry) {
cbm_registry_types_by_short_name(ctx->registry, the_short, &ts_it);
cbm_registry_types_by_short_name_chain(ctx->registry, the_short, &ts_it);
}
while (ctx->registry && (i = cbm_type_short_iter_next(&ts_it)) >= 0) {
const CBMRegisteredType *cand = &ctx->registry->types[i];
const CBMRegisteredType *cand = &ts_it.reg->types[i];
if (!cand->short_name || strcmp(cand->short_name, the_short) != 0)
continue;
int score = 0;
@@ -643,12 +643,12 @@ static const CBMRegisteredFunc *cs_lookup_extension(CSLSPContext *ctx, const cha
* registration-order tie-break exactly (bucket chains are not in funcs[]
* order). */
CBMFreeFuncIter ext_it;
cbm_registry_free_funcs_by_short_name(ctx->registry, method_name, &ext_it);
cbm_registry_free_funcs_by_short_name_chain(ctx->registry, method_name, &ext_it);
const CBMRegisteredFunc *best = NULL;
int best_idx = -1;
int i;
while ((i = cbm_free_func_iter_next(&ext_it)) >= 0) {
const CBMRegisteredFunc *cand = &ctx->registry->funcs[i];
const CBMRegisteredFunc *cand = &ext_it.reg->funcs[i];
if (!cand->short_name || strcmp(cand->short_name, method_name) != 0)
continue;
if (cand->receiver_type)
@@ -1522,8 +1522,12 @@ static const char *cs_resolve_callable_name(CSLSPContext *ctx, const char *name)
* generated fixtures can register receiverless callables. Only accept a
* unique short-name match; ambiguity is intentionally not guessed. */
const CBMRegisteredFunc *only = NULL;
for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) {
const CBMRegisteredFunc *candidate = &ctx->registry->funcs[i];
CBMFreeFuncIter all_funcs;
if (ctx->registry) {
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
}
for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const CBMRegisteredFunc *candidate = &all_funcs.reg->funcs[i];
if (candidate->receiver_type || !candidate->short_name ||
strcmp(candidate->short_name, name) != 0) {
continue;
@@ -2142,8 +2146,12 @@ static void cs_resolve_invocation(CSLSPContext *ctx, TSNode call) {
/* Last resort: any free function with this short name in registry. */
const CBMRegisteredFunc *best = NULL;
int best_score = -1;
for (int i = 0; ctx->registry && i < ctx->registry->func_count; i++) {
const CBMRegisteredFunc *cand = &ctx->registry->funcs[i];
CBMFreeFuncIter all_funcs;
if (ctx->registry) {
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
}
for (int i = -1; ctx->registry && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const CBMRegisteredFunc *cand = &all_funcs.reg->funcs[i];
if (cand->receiver_type)
continue;
if (!cand->short_name || strcmp(cand->short_name, bare) != 0)
internal/cbm/lsp/go_lsp.c
+4 -2
View File
@@ -1551,8 +1551,10 @@ static void resolve_calls_in_node_inner(GoLSPContext* ctx, TSNode node) {
int impl_count = 0;
// Skip stdlib types when interface is from a project package
bool iface_is_project = iface_qn && strchr(iface_qn, '/') != NULL;
for (int ti = 0; ti < ctx->registry->type_count && impl_count < 2; ti++) {
const CBMRegisteredType* cand = &ctx->registry->types[ti];
CBMTypeShortIter all_types;
cbm_registry_all_types_chain(ctx->registry, &all_types);
for (int ti = -1; impl_count < 2 && (ti = cbm_type_short_iter_next(&all_types)) >= 0;) {
const CBMRegisteredType* cand = &all_types.reg->types[ti];
if (cand->is_interface) continue;
if (!cand->qualified_name) continue;
if (cand->alias_of) continue;
internal/cbm/lsp/py_lsp.c
+61 -81
View File
@@ -238,6 +238,7 @@ void py_lsp_init(PyLSPContext *ctx, CBMArena *arena, const char *source, int sou
ctx->source = source;
ctx->source_len = source_len;
ctx->registry = registry;
ctx->registry_head = (CBMTypeRegistry *)registry;
ctx->module_qn = module_qn;
ctx->resolved_calls = out;
ctx->current_scope = py_scope_push_checked(ctx);
@@ -394,7 +395,7 @@ static void py_import_syntax_match_add(PyImportSyntaxMatch *match, PyDirectImpor
* registry then retains responsibility for proving that the exact target is
* an undecorated, unambiguous callable. */
static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode statement,
TSNode imported_name, const char *incoming_qn) {
TSNode imported_name, const char *incoming_qn) {
if (!ctx || !incoming_qn || ts_node_is_null(statement) || ts_node_is_null(imported_name))
return NULL;
TSNode module = py_from_import_module_node(statement);
@@ -407,10 +408,9 @@ static const char *py_canonical_from_import_qn(PyLSPContext *ctx, TSNode stateme
while (module_suffix && *module_suffix == '.')
module_suffix++;
const char *full_suffix =
module_suffix && module_suffix[0]
? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member)
: member;
const char *full_suffix = module_suffix && module_suffix[0]
? cbm_arena_sprintf(ctx->arena, "%s.%s", module_suffix, member)
: member;
if (!full_suffix)
return NULL;
if (py_qn_has_boundary_suffix(incoming_qn, full_suffix))
@@ -445,17 +445,16 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char
TSNode alias = ts_node_child_by_field_name(item, "alias", 5);
if (py_import_node_text_equals(ctx, alias, local)) {
const char *canonical = py_canonical_from_import_qn(ctx, stmt, name, qn);
py_import_syntax_match_add(
match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
py_import_syntax_match_add(match,
canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
}
} else if ((strcmp(kind, "identifier") == 0 ||
strcmp(kind, "dotted_name") == 0) &&
} else if ((strcmp(kind, "identifier") == 0 || strcmp(kind, "dotted_name") == 0) &&
py_import_node_text_equals(ctx, item, local)) {
const char *canonical = py_canonical_from_import_qn(ctx, stmt, item, qn);
py_import_syntax_match_add(
match, canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
py_import_syntax_match_add(match,
canonical ? PY_FROM_IMPORT : PY_IMPORT_UNCLASSIFIED,
canonical ? canonical : qn);
}
}
return;
@@ -489,8 +488,7 @@ static void py_import_match_statement(PyLSPContext *ctx, TSNode stmt, const char
}
}
static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match,
const char **qn_io) {
static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match, const char **qn_io) {
if (!match || !qn_io)
return PY_DIRECT_IMPORT_UNKNOWN;
if (match->count == 0)
@@ -503,8 +501,7 @@ static PyDirectImportKind py_import_match_result(PyImportSyntaxMatch *match,
}
static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNode stmt,
const char *local,
const char **qn_io) {
const char *local, const char **qn_io) {
const char *qn = qn_io ? *qn_io : NULL;
if (!ctx || !local || !qn || !qn_io || ts_node_is_null(stmt))
return PY_DIRECT_IMPORT_UNKNOWN;
@@ -513,8 +510,8 @@ static PyDirectImportKind py_import_kind_from_statement(PyLSPContext *ctx, TSNod
return py_import_match_result(&match, qn_io);
}
static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root,
const char *local, const char **qn_io) {
static PyDirectImportKind py_import_kind_from_ast(PyLSPContext *ctx, TSNode root, const char *local,
const char **qn_io) {
const char *qn = qn_io ? *qn_io : NULL;
if (!ctx || !local || !qn || !qn_io || ts_node_is_null(root))
return PY_DIRECT_IMPORT_UNKNOWN;
@@ -589,36 +586,34 @@ static void py_bind_import_index(PyLSPContext *ctx, int index, bool synthetic_fa
if (!local || !qn || strcmp(local, "*") == 0)
return;
PyDirectImportKind direct_kind = ctx->import_kinds
? (PyDirectImportKind)ctx->import_kinds[index]
: PY_DIRECT_IMPORT_UNKNOWN;
PyDirectImportKind direct_kind =
ctx->import_kinds ? (PyDirectImportKind)ctx->import_kinds[index] : PY_DIRECT_IMPORT_UNKNOWN;
bool from_style = direct_kind == PY_FROM_IMPORT ||
(direct_kind == PY_DIRECT_IMPORT_UNKNOWN &&
import_is_from_style(local, qn));
(direct_kind == PY_DIRECT_IMPORT_UNKNOWN && import_is_from_style(local, qn));
const CBMType *t;
if (direct_kind == PY_DIRECT_IMPORT_UNALIASED) {
// Python binds only the root of an unaliased dotted import.
// Python binds only the root of an unaliased dotted import.
t = cbm_type_module(ctx->arena, local);
} else if (direct_kind == PY_DIRECT_IMPORT_ALIASED) {
t = cbm_type_module(ctx->arena, qn);
} else if (from_style) {
// `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute
// resolution checks the registry to upgrade to MODULE / class
// / function as appropriate.
// `from X import Y` — bind Y to NAMED(X.Y). Phase 6 attribute
// resolution checks the registry to upgrade to MODULE / class
// / function as appropriate.
t = cbm_type_named(ctx->arena, qn);
} else if (strchr(qn, '.') != NULL) {
// Dotted path whose tail does NOT match the local name: an
// ALIASED binding — `from X import Y as Z` (Z names function/
// class X.Y) or `import a.b as z` (z names module a.b). The
// CBMImport shape cannot distinguish the two, but NAMED(qn)
// covers both: phase 6 upgrades it to the registered function/
// class for the from-import and to MODULE for the module alias.
// Binding MODULE here made `g()` calls on `from m import f as g`
// resolve as calls on a module — lsp=MISS, and the whole CALLS
// edge was lost (#988).
// Dotted path whose tail does NOT match the local name: an
// ALIASED binding — `from X import Y as Z` (Z names function/
// class X.Y) or `import a.b as z` (z names module a.b). The
// CBMImport shape cannot distinguish the two, but NAMED(qn)
// covers both: phase 6 upgrades it to the registered function/
// class for the from-import and to MODULE for the module alias.
// Binding MODULE here made `g()` calls on `from m import f as g`
// resolve as calls on a module — lsp=MISS, and the whole CALLS
// edge was lost (#988).
t = cbm_type_named(ctx->arena, qn);
} else {
// `import X` / `import X as Y` (single segment) — MODULE(X).
// `import X` / `import X as Y` (single segment) — MODULE(X).
t = cbm_type_module(ctx->arena, qn);
}
const CBMRegisteredFunc *imported =
@@ -841,8 +836,7 @@ static void py_emit_resolved_call(PyLSPContext *ctx, const char *callee_qn, cons
}
static void py_emit_resolved_reference(PyLSPContext *ctx, const char *callee_qn,
const char *source_name, TSNode site,
const char *strategy) {
const char *source_name, TSNode site, const char *strategy) {
if (!ctx || !ctx->resolved_calls || !callee_qn || !ctx->enclosing_func_qn ||
ts_node_is_null(site)) {
return;
@@ -1027,8 +1021,7 @@ static void py_resolve_value_references_at(PyLSPContext *ctx, TSNode call) {
if (candidate && cbm_scope_contains(ctx->current_scope, source_name)) {
const CBMType *binding =
cbm_type_resolve_alias(cbm_scope_lookup(ctx->current_scope, source_name));
if (binding && binding->kind == CBM_TYPE_NAMED &&
binding->data.named.qualified_name &&
if (binding && binding->kind == CBM_TYPE_NAMED && binding->data.named.qualified_name &&
strcmp(binding->data.named.qualified_name, candidate) == 0) {
py_emit_unresolved_reference(ctx, candidate, arg);
}
@@ -1195,16 +1188,10 @@ static void py_register_instance_field(PyLSPContext *ctx, const char *class_qn,
return;
}
// Find the type entry. cbm_registry_lookup_type returns a const pointer;
// we need a mutable pointer into the registry's array.
CBMRegisteredType *rt = NULL;
for (int i = 0; i < ctx->registry->type_count; i++) {
const char *qn = ctx->registry->types[i].qualified_name;
if (qn && strcmp(qn, class_qn) == 0) {
rt = &ctx->registry->types[i];
break;
}
}
// The writable entry for the class: the head's own, or a copy of the
// base entry made in the head (copy-on-write). The shared base is never
// written -- it used to be, from every parallel resolve worker.
CBMRegisteredType *rt = cbm_registry_type_for_update(ctx->registry_head, class_qn);
if (!rt)
return;
@@ -1570,11 +1557,12 @@ static const CBMType *py_eval_expr_type_uncached(PyLSPContext *ctx, TSNode node)
const char *prefix = cbm_arena_sprintf(ctx->arena, "%s.", qn);
size_t prefix_len = strlen(prefix);
bool is_submodule = false;
for (int i = 0; i < ctx->registry->func_count; i++) {
const char *fqn = ctx->registry->funcs[i].qualified_name;
CBMFreeFuncIter all_funcs;
cbm_registry_all_funcs_chain(ctx->registry, &all_funcs);
for (int i = -1; !is_submodule && (i = cbm_free_func_iter_next(&all_funcs)) >= 0;) {
const char *fqn = all_funcs.reg->funcs[i].qualified_name;
if (fqn && strncmp(fqn, prefix, prefix_len) == 0) {
is_submodule = true;
break;
}
}
if (is_submodule)
@@ -2600,8 +2588,7 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) {
const char *local = ctx->import_local_names[i];
const char *import_qn = ctx->import_module_qns[i];
if (local && import_qn && strcmp(local, fname) == 0 &&
strcmp(import_qn, qn) == 0 &&
py_import_index_is_from_binding(ctx, i)) {
strcmp(import_qn, qn) == 0 && py_import_index_is_from_binding(ctx, i)) {
imported_alias = true;
break;
}
@@ -2758,8 +2745,8 @@ static void py_emit_call_for(PyLSPContext *ctx, TSNode call_node) {
// Skip if mod is already rooted under the project to avoid
// "<root>.<root>.mod".
if (!(strncmp(mod, ctx->module_qn, root_len) == 0 && mod[root_len] == '.')) {
char *qual_mod = (char *)cbm_arena_alloc(ctx->arena, root_len + 1 +
strlen(mod) + 1);
char *qual_mod =
(char *)cbm_arena_alloc(ctx->arena, root_len + 1 + strlen(mod) + 1);
if (qual_mod) {
memcpy(qual_mod, ctx->module_qn, root_len);
qual_mod[root_len] = '.';
@@ -4047,10 +4034,8 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char
// For methods, bind `self`/`cls` AFTER param walk so the receiver type
// wins over the unannotated `self` / `cls` parameter declaration.
if (ctx->enclosing_class_qn) {
py_scope_bind(ctx, "self",
cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "cls",
cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "self", cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
py_scope_bind(ctx, "cls", cbm_type_named(ctx->arena, ctx->enclosing_class_qn));
}
TSNode body = ts_node_child_by_field_name(func_node, "body", 4);
@@ -4197,12 +4182,14 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, TSNode root) {
if (!ctx || !ctx->registry || !ctx->module_qn)
return;
size_t prefix_len = strlen(ctx->module_qn);
for (int i = 0; i < ctx->registry->type_count; i++) {
const CBMRegisteredType *type = &ctx->registry->types[i];
CBMTypeShortIter all_types;
cbm_registry_all_types_chain(ctx->registry, &all_types);
for (int i = -1; (i = cbm_type_short_iter_next(&all_types)) >= 0;) {
const CBMRegisteredType *type = &all_types.reg->types[i];
const char *qn = type->qualified_name;
const char *name = type->short_name;
if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 ||
qn[prefix_len] != '.' || py_root_defines_class_named(ctx, root, name)) {
if (!qn || !name || strncmp(qn, ctx->module_qn, prefix_len) != 0 || qn[prefix_len] != '.' ||
py_root_defines_class_named(ctx, root, name)) {
continue;
}
py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn));
@@ -4502,8 +4489,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
py_invalidate_possible_bindings(ctx, definition, depth + 1);
return;
}
if (strcmp(kind, "import_statement") == 0 ||
strcmp(kind, "import_from_statement") == 0) {
if (strcmp(kind, "import_statement") == 0 || strcmp(kind, "import_from_statement") == 0) {
py_invalidate_import_bindings(ctx, node);
return;
}
@@ -4511,8 +4497,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
TSNode right = ts_node_child_by_field_name(node, "right", 5);
if (ts_node_is_null(right))
return;
py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4),
depth + 1);
py_invalidate_binding_target(ctx, ts_node_child_by_field_name(node, "left", 4), depth + 1);
py_invalidate_possible_bindings(ctx, right, depth + 1);
return;
}
@@ -4523,8 +4508,7 @@ static void py_invalidate_possible_bindings(PyLSPContext *ctx, TSNode node, int
py_invalidate_possible_bindings(ctx, right, depth + 1);
return;
}
if (strcmp(kind, "named_expression") == 0 ||
strcmp(kind, "assignment_expression") == 0) {
if (strcmp(kind, "named_expression") == 0 || strcmp(kind, "assignment_expression") == 0) {
TSNode name = ts_node_child_by_field_name(node, "name", 4);
if (ts_node_is_null(name))
name = ts_node_child_by_field_name(node, "left", 4);
@@ -4619,8 +4603,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l
continue;
}
const char *candidate_qn = ctx->import_module_qns[i];
PyDirectImportKind kind =
py_import_kind_from_statement(ctx, stmt, local, &candidate_qn);
PyDirectImportKind kind = py_import_kind_from_statement(ctx, stmt, local, &candidate_qn);
if (!py_replayable_import_kind(kind))
continue;
if (!chosen_qn) {
@@ -4642,8 +4625,7 @@ static void py_replay_import_local(PyLSPContext *ctx, TSNode stmt, const char *l
py_bind_import_index(ctx, chosen, false);
}
static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt,
unsigned char *consumed) {
static void py_replay_import_statement(PyLSPContext *ctx, TSNode stmt, unsigned char *consumed) {
if (py_import_statement_is_wildcard(stmt)) {
py_invalidate_module_bindings_for_wildcard(ctx);
return;
@@ -4686,8 +4668,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) {
py_bind_external_module_classes(ctx, root);
unsigned char *consumed_imports = NULL;
if (ctx->import_count > 0) {
consumed_imports =
(unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count);
consumed_imports = (unsigned char *)cbm_arena_alloc(ctx->arena, (size_t)ctx->import_count);
if (consumed_imports)
memset(consumed_imports, 0, (size_t)ctx->import_count);
}
@@ -4700,8 +4681,7 @@ void py_lsp_process_file(PyLSPContext *ctx, TSNode root) {
for (uint32_t i = 0; i < nc; i++) {
TSNode c = ts_node_named_child(root, i);
const char *ck = ts_node_type(c);
if (strcmp(ck, "import_statement") == 0 ||
strcmp(ck, "import_from_statement") == 0) {
if (strcmp(ck, "import_statement") == 0 || strcmp(ck, "import_from_statement") == 0) {
py_replay_import_statement(ctx, c, consumed_imports);
} else if (strcmp(ck, "function_definition") == 0) {
py_bind_module_function(ctx, c);
internal/cbm/lsp/py_lsp.h
+4
View File
@@ -44,6 +44,10 @@ typedef struct {
const char *source;
int source_len;
const CBMTypeRegistry *registry;
/* Writable head of the registry chain (the per-file overlay, or the
* per-file registry on the non-cross path); refinements go here via
* cbm_registry_*_for_update, never into anything behind ->fallback. */
CBMTypeRegistry *registry_head;
CBMScope *current_scope;
// Import map: local_name -> module_qn (arena-allocated, NULL-terminated).
internal/cbm/lsp/type_registry.c
+231 -8
View File
@@ -301,6 +301,9 @@ static void build_ffunc_short_index(CBMTypeRegistry *reg, CBMArena *idx_arena) {
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeShortIter *out) {
out->reg = reg;
out->chain = false;
out->key = short_name;
out->shadow = NULL;
out->hash = fnv1a(short_name);
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
if (reg->type_short_buckets && reg->type_short_bucket_count > 0) {
@@ -319,7 +322,7 @@ void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *sh
}
}
int cbm_type_short_iter_next(CBMTypeShortIter *it) {
static int type_short_next_local(CBMTypeShortIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->type_short_entries[it->chain_idx];
@@ -335,9 +338,79 @@ int cbm_type_short_iter_next(CBMTypeShortIter *it) {
return -1;
}
static const CBMRegisteredType *lookup_type_self(const CBMTypeRegistry *reg,
const char *qualified_name);
static const CBMRegisteredFunc *lookup_func_self(const CBMTypeRegistry *reg,
const char *qualified_name);
/* A base entry is shadowed when the head holds an entry with the same QN. */
static bool type_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) {
if (!shadow || shadow == reg || !reg->types[i].qualified_name) {
return false;
}
return lookup_type_self(shadow, reg->types[i].qualified_name) != NULL;
}
static bool func_shadowed(const CBMTypeRegistry *shadow, const CBMTypeRegistry *reg, int i) {
if (!shadow || shadow == reg || !reg->funcs[i].qualified_name) {
return false;
}
return lookup_func_self(shadow, reg->funcs[i].qualified_name) != NULL;
}
static void type_short_open_linear(const CBMTypeRegistry *reg, CBMTypeShortIter *out) {
out->reg = reg;
out->hash = 0;
out->chain_idx = -1;
out->tail_i = 0;
out->tail_end = reg->type_count;
}
int cbm_type_short_iter_next(CBMTypeShortIter *it) {
for (;;) {
int p = type_short_next_local(it);
if (p >= 0) {
if (it->chain && type_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
if (key) {
cbm_registry_types_by_short_name(next, key, it);
} else {
type_short_open_linear(next, it);
}
it->chain = true;
it->key = key;
it->shadow = shadow;
}
}
void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name,
CBMTypeShortIter *out) {
cbm_registry_types_by_short_name(head, short_name, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out) {
type_short_open_linear(head, out);
out->chain = true;
out->key = NULL;
out->shadow = head;
}
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
CBMTypeEmbedIter *out) {
out->reg = reg;
out->chain = false;
out->key = bare;
out->shadow = NULL;
out->hash = fnv1a(bare);
out->prev_type = -1;
if (reg->type_qn_buckets && reg->type_qn_bucket_count > 0) {
@@ -358,7 +431,7 @@ void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char
}
}
int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
static int type_embed_next_local(CBMTypeEmbedIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->type_embed_entries[it->chain_idx];
@@ -377,9 +450,40 @@ int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
return -1;
}
int cbm_type_embed_iter_next(CBMTypeEmbedIter *it) {
for (;;) {
int p = type_embed_next_local(it);
if (p >= 0) {
if (it->chain && type_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
cbm_registry_types_by_embedded_bare(next, key, it);
it->chain = true;
it->shadow = shadow;
}
}
void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare,
CBMTypeEmbedIter *out) {
cbm_registry_types_by_embedded_bare(head, bare, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMFreeFuncIter *out) {
out->reg = reg;
out->chain = false;
out->key = short_name;
out->shadow = NULL;
out->hash = fnv1a(short_name);
if (reg->func_qn_buckets && reg->func_qn_bucket_count > 0) {
if (reg->ffunc_short_buckets && reg->ffunc_short_bucket_count > 0) {
@@ -397,7 +501,7 @@ void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const cha
}
}
int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
static int free_func_next_local(CBMFreeFuncIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->ffunc_short_entries[it->chain_idx];
@@ -413,9 +517,57 @@ int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
return -1;
}
static void free_func_open_linear(const CBMTypeRegistry *reg, CBMFreeFuncIter *out) {
out->reg = reg;
out->hash = 0;
out->chain_idx = -1;
out->tail_i = 0;
out->tail_end = reg->func_count;
}
int cbm_free_func_iter_next(CBMFreeFuncIter *it) {
for (;;) {
int p = free_func_next_local(it);
if (p >= 0) {
if (it->chain && func_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const char *key = it->key;
const CBMTypeRegistry *shadow = it->shadow;
if (key) {
cbm_registry_free_funcs_by_short_name(next, key, it);
} else {
free_func_open_linear(next, it);
}
it->chain = true;
it->key = key;
it->shadow = shadow;
}
}
void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head,
const char *short_name, CBMFreeFuncIter *out) {
cbm_registry_free_funcs_by_short_name(head, short_name, out);
out->chain = true;
out->shadow = head;
}
void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out) {
free_func_open_linear(head, out);
out->chain = true;
out->key = NULL;
out->shadow = head;
}
void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
const char *method_name, CBMMethodIter *out) {
memset(out, 0, sizeof(*out));
memset(out, 0, sizeof(*out)); /* chain=false, shadow=NULL */
out->reg = reg;
out->receiver_qn = receiver_qn;
out->method_name = method_name;
@@ -435,10 +587,7 @@ void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
}
}
int cbm_method_iter_next(CBMMethodIter *it) {
if (!it || !it->reg || !it->receiver_qn || !it->method_name) {
return -1;
}
static int method_next_local(CBMMethodIter *it) {
const CBMTypeRegistry *reg = it->reg;
while (it->chain_idx >= 0) {
const CBMRegistryHashEntry *e = &reg->method_entries[it->chain_idx];
@@ -465,6 +614,80 @@ int cbm_method_iter_next(CBMMethodIter *it) {
return -1;
}
int cbm_method_iter_next(CBMMethodIter *it) {
if (!it || !it->reg || !it->receiver_qn || !it->method_name) {
return -1;
}
for (;;) {
int p = method_next_local(it);
if (p >= 0) {
if (it->chain && func_shadowed(it->shadow, it->reg, p)) {
continue;
}
return p;
}
if (!it->chain || !it->reg->fallback) {
return -1;
}
const CBMTypeRegistry *next = it->reg->fallback;
const CBMTypeRegistry *shadow = it->shadow;
cbm_registry_methods(next, it->receiver_qn, it->method_name, it);
it->chain = true;
it->shadow = shadow;
}
}
void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn,
const char *method_name, CBMMethodIter *out) {
cbm_registry_methods(head, receiver_qn, method_name, out);
out->chain = true;
out->shadow = head;
}
CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name) {
if (!head || !qualified_name || head->read_only) {
return NULL;
}
const CBMRegisteredFunc *own = lookup_func_self(head, qualified_name);
if (own) {
return (CBMRegisteredFunc *)own;
}
for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) {
const CBMRegisteredFunc *base = lookup_func_self(r, qualified_name);
if (base) {
int before = head->func_count;
cbm_registry_add_func(head, *base);
if (head->func_count == before + 1) {
return &head->funcs[before];
}
return NULL;
}
}
return NULL;
}
CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name) {
if (!head || !qualified_name || head->read_only) {
return NULL;
}
const CBMRegisteredType *own = lookup_type_self(head, qualified_name);
if (own) {
return (CBMRegisteredType *)own;
}
for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) {
const CBMRegisteredType *base = lookup_type_self(r, qualified_name);
if (base) {
int before = head->type_count;
cbm_registry_add_type(head, *base);
if (head->type_count == before + 1) {
return &head->types[before];
}
return NULL;
}
}
return NULL;
}
void cbm_registry_finalize_into(CBMTypeRegistry *reg, CBMArena *idx_arena) {
if (!reg || !idx_arena)
return;
internal/cbm/lsp/type_registry.h
+46
View File
@@ -242,6 +242,16 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
/* Chain mode (see cbm_registry_*_chain): when this registry is exhausted
* the iterator re-opens on reg->fallback; `reg` then names the registry
* the yielded index belongs to, so callers deref it->reg->types[i], never
* the registry they started from. `key` re-opens the same query on the
* next link; NULL = linear scan. `shadow` is the head: fallback entries
* whose qualified_name the head also holds are skipped (an overlay copy
* hides its base original). */
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMTypeShortIter;
void cbm_registry_types_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
CBMTypeShortIter *out);
@@ -263,6 +273,9 @@ typedef struct {
int tail_i; // next tail/linear type index
int tail_end; // reg->type_count snapshot
int prev_type; // last yielded type index (adjacent-dedup); -1 = none
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMTypeEmbedIter;
void cbm_registry_types_by_embedded_bare(const CBMTypeRegistry *reg, const char *bare,
CBMTypeEmbedIter *out);
@@ -277,6 +290,9 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
bool chain;
const char *key;
const CBMTypeRegistry *shadow;
} CBMFreeFuncIter;
void cbm_registry_free_funcs_by_short_name(const CBMTypeRegistry *reg, const char *short_name,
@@ -296,11 +312,41 @@ typedef struct {
int chain_idx;
int tail_i;
int tail_end;
bool chain;
const CBMTypeRegistry *shadow;
} CBMMethodIter;
void cbm_registry_methods(const CBMTypeRegistry *reg, const char *receiver_qn,
const char *method_name, CBMMethodIter *out);
int cbm_method_iter_next(CBMMethodIter *it);
/* ── Chain-aware iteration and copy-on-write (per-file overlay contract) ──
*
* A resolve walk is handed a per-file OVERLAY registry chained (`fallback`) to
* the immutable shared base. Lookups already chain; these make the index
* iterators and the whole-registry scans chain too, and give a walk one way
* to refine an entry: copy it into the head first. Nothing behind `fallback`
* is ever written. Every yielded index belongs to it->reg at that moment. */
void cbm_registry_types_by_short_name_chain(const CBMTypeRegistry *head, const char *short_name,
CBMTypeShortIter *out);
void cbm_registry_types_by_embedded_bare_chain(const CBMTypeRegistry *head, const char *bare,
CBMTypeEmbedIter *out);
void cbm_registry_free_funcs_by_short_name_chain(const CBMTypeRegistry *head,
const char *short_name, CBMFreeFuncIter *out);
void cbm_registry_methods_chain(const CBMTypeRegistry *head, const char *receiver_qn,
const char *method_name, CBMMethodIter *out);
/* Linear scans over every func / type in the chain (head first, shadowed
* base entries skipped). Same iterator types, same it->reg contract. */
void cbm_registry_all_funcs_chain(const CBMTypeRegistry *head, CBMFreeFuncIter *out);
void cbm_registry_all_types_chain(const CBMTypeRegistry *head, CBMTypeShortIter *out);
/* The writable entry for qualified_name: the head's own entry if it has one,
* else a copy of the first fallback entry added to the head (copy-on-write),
* else NULL. NULL also when the head is sealed (read_only) -- a walk must never
* mutate a shared registry, so the caller skips the refinement, exactly as the
* sealed-registry no-op in cbm_registry_add_func does today. */
CBMRegisteredFunc *cbm_registry_func_for_update(CBMTypeRegistry *head, const char *qualified_name);
CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const char *qualified_name);
// --- TS-specific helpers (return NULL for types without these signatures) ---
// If the type has a call signature (e.g., `interface F { (x:number): string }`), return
internal/cbm/result_compact.c
+460
View File
@@ -0,0 +1,460 @@
/*
* result_compact.c — copy a finished CBMFileResult into one exact-size arena.
*
* Why this exists (Go corpus census, 2026-09-13): extraction writes every
* temporary into the result arena — cbm_node_text copies at 496 call sites,
* per-node QN sprintf, enclosing-QN strings — and GROW_ARRAY leaves each
* previous generation of every record array dead behind it. 14.8 GB written,
* 3.4 GB reachable, and all of it retained until after resolve because the
* result owned the arena. Compaction walks what is reachable, measures it,
* copies it into a single block of exactly that size (strings interned by
* content within the file), and destroys the working arena.
*
* Three passes over one traversal:
* COUNT — number of string references, to size the intern table
* MEASURE — bytes every allocation will take (aligned like the arena does)
* COPY — the same allocations, for real, into the fresh arena
* MEASURE and COPY issue identical allocation sequences, so the block fits
* exactly; any failure leaves the result untouched.
*/
#include "cbm.h"
#include "foundation/arena.h"
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "result_spill.h" /* cbm_result_relocate */
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
enum { CR_ALIGN = 7, CR_MIN_TABLE = 64, CR_TABLE_LOAD = 2 };
typedef enum { CR_COUNT = 0, CR_MEASURE, CR_COPY, CR_RELOCATE } cr_phase_t;
typedef struct {
const char *src; /* NULL = empty slot */
char *dst; /* copy in the new arena; NULL until COPY reaches it */
uint64_t hash;
size_t len; /* strlen(src) */
} cr_slot_t;
typedef struct {
cr_phase_t phase;
size_t refs; /* COUNT: string references seen */
size_t bytes; /* MEASURE: total arena bytes */
CBMArena *dst; /* COPY: the fresh arena */
bool failed;
cr_slot_t *slots;
size_t cap; /* power of two */
/* RELOCATE: pointers in [old_base, old_base + old_len) move by delta. */
const char *old_base;
size_t old_len;
ptrdiff_t delta;
} cr_ctx_t;
static bool cr_in_old_block(const cr_ctx_t *c, const void *p) {
const char *cp = (const char *)p;
return cp >= c->old_base && cp < c->old_base + c->old_len;
}
static size_t cr_aligned(size_t n) {
return (n + CR_ALIGN) & ~(size_t)CR_ALIGN;
}
static uint64_t cr_hash(const char *s, size_t len) {
uint64_t h = 1469598103934665603ULL;
for (size_t i = 0; i < len; i++) {
h ^= (unsigned char)s[i];
h *= 1099511628211ULL;
}
return h;
}
/* Find or insert the slot for s. Never fails: the table is sized from COUNT
* at half load, and the same references are presented again in COPY. */
static cr_slot_t *cr_slot(cr_ctx_t *c, const char *s) {
size_t len = strlen(s);
uint64_t h = cr_hash(s, len);
size_t mask = c->cap - SKIP_ONE;
size_t i = (size_t)h & mask;
for (;;) {
cr_slot_t *slot = &c->slots[i];
if (!slot->src) {
slot->src = s;
slot->hash = h;
slot->len = len;
slot->dst = NULL;
return slot;
}
if (slot->hash == h && slot->len == len &&
(slot->src == s || memcmp(slot->src, s, len) == 0)) {
return slot;
}
i = (i + SKIP_ONE) & mask;
}
}
/* A raw allocation of `bytes` in the new arena: MEASURE books it, COPY makes
* it. Zero bytes is no allocation (the arena returns NULL for it too). */
static void *cr_alloc(cr_ctx_t *c, size_t bytes) {
if (bytes == 0) {
return NULL;
}
if (c->phase == CR_MEASURE) {
c->bytes += cr_aligned(bytes);
return NULL;
}
if (c->phase == CR_COPY) {
void *p = cbm_arena_alloc(c->dst, bytes);
if (!p) {
c->failed = true;
}
return p;
}
return NULL;
}
/* A string field: interned by content. COPY rewrites the field. */
static void cr_str(cr_ctx_t *c, const char **field) {
const char *s = *field;
if (!s) {
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, s)) {
*field = s + c->delta;
}
return;
}
if (c->phase == CR_COUNT) {
c->refs++;
return;
}
cr_slot_t *slot = cr_slot(c, s);
if (c->phase == CR_MEASURE) {
if (!slot->dst) {
slot->dst = (char *)s; /* mark as booked; reset before COPY */
c->bytes += cr_aligned(slot->len + SKIP_ONE);
}
return;
}
if (!slot->dst) {
char *copy = (char *)cr_alloc(c, slot->len + SKIP_ONE);
if (!copy) {
return;
}
memcpy(copy, slot->src, slot->len + SKIP_ONE);
slot->dst = copy;
}
*field = slot->dst;
}
/* A blob (fingerprint, retained source): copied verbatim. */
static void cr_blob(cr_ctx_t *c, const void **field, size_t bytes) {
if (!*field || bytes == 0) {
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, *field)) {
*field = (const char *)*field + c->delta;
}
return;
}
if (c->phase == CR_COPY) {
void *copy = cr_alloc(c, bytes);
if (!copy) {
return;
}
memcpy(copy, *field, bytes);
*field = copy;
} else {
(void)cr_alloc(c, bytes);
}
}
/* A record array: copied at exact count, then the caller walks its fields. */
static void cr_array(cr_ctx_t *c, void **items, int count, size_t elem) {
if (!*items || count <= 0) {
if (c->phase == CR_COPY) {
*items = NULL;
}
return;
}
if (c->phase == CR_RELOCATE) {
if (cr_in_old_block(c, *items)) {
*items = (char *)*items + c->delta;
}
return;
}
cr_blob(c, (const void **)items, (size_t)count * elem);
}
/* A NULL-terminated list of strings: the pointer array plus each string. */
static void cr_list(cr_ctx_t *c, const char ***field) {
const char **list = *field;
if (!list) {
return;
}
int n = 0;
if (c->phase == CR_RELOCATE) {
/* The array may live in a block we can no longer read at its old
* address: relocate the pointer first, then count and walk it. */
cr_blob(c, (const void **)field, sizeof(char *));
list = *field;
while (list[n]) {
n++;
}
for (int i = 0; i < n; i++) {
cr_str(c, &list[i]);
}
return;
}
while (list[n]) {
n++;
}
cr_blob(c, (const void **)field, (size_t)(n + SKIP_ONE) * sizeof(char *));
const char **walk = *field; /* the copy in COPY, the original otherwise */
for (int i = 0; i < n && walk; i++) {
cr_str(c, &walk[i]);
}
}
/* A counted list of strings (signature_param_types). */
static void cr_counted_list(cr_ctx_t *c, const char ***field, int count) {
if (!*field || count <= 0) {
return;
}
cr_blob(c, (const void **)field, (size_t)count * sizeof(char *));
/* (RELOCATE: the pointer moved; the walk below now reads the new array.) */
const char **walk = *field;
for (int i = 0; i < count && walk; i++) {
cr_str(c, &walk[i]);
}
}
static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) {
cr_str(c, &d->name);
cr_str(c, &d->qualified_name);
cr_str(c, &d->label);
cr_str(c, &d->file_path);
cr_str(c, &d->signature);
cr_str(c, &d->return_type);
cr_str(c, &d->receiver);
cr_str(c, &d->docstring);
cr_str(c, &d->parent_class);
cr_list(c, &d->decorators);
cr_list(c, &d->base_classes);
cr_list(c, &d->param_names);
cr_list(c, &d->param_types);
cr_counted_list(c, &d->signature_param_types, d->signature_param_count);
cr_list(c, &d->return_types);
cr_str(c, &d->route_path);
cr_str(c, &d->route_method);
cr_blob(c, (const void **)&d->fingerprint,
d->fingerprint_k > 0 ? (size_t)d->fingerprint_k * sizeof(uint32_t) : 0);
cr_str(c, &d->structural_profile);
cr_str(c, &d->body_tokens);
cr_str(c, &d->impl_trait);
}
static void cr_walk_call(cr_ctx_t *c, CBMCall *call) {
cr_str(c, &call->callee_name);
cr_str(c, &call->enclosing_func_qn);
cr_str(c, &call->first_string_arg);
cr_str(c, &call->second_arg_name);
int argc = call->arg_count;
if (argc > CBM_MAX_CALL_ARGS) {
argc = CBM_MAX_CALL_ARGS;
}
if (call->args && argc > 0) {
cr_array(c, (void **)&call->args, argc, sizeof(CBMCallArg));
for (int i = 0; i < argc && call->args; i++) {
cr_str(c, &call->args[i].expr);
cr_str(c, &call->args[i].value);
cr_str(c, &call->args[i].keyword);
}
} else if (c->phase == CR_COPY) {
call->args = NULL;
call->arg_count = 0;
}
}
static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {
cr_array(c, (void **)&r->defs.items, r->defs.count, sizeof(CBMDefinition));
for (int i = 0; i < r->defs.count && r->defs.items; i++) {
cr_walk_def(c, &r->defs.items[i]);
}
cr_array(c, (void **)&r->calls.items, r->calls.count, sizeof(CBMCall));
for (int i = 0; i < r->calls.count && r->calls.items; i++) {
cr_walk_call(c, &r->calls.items[i]);
}
cr_array(c, (void **)&r->imports.items, r->imports.count, sizeof(CBMImport));
for (int i = 0; i < r->imports.count && r->imports.items; i++) {
cr_str(c, &r->imports.items[i].local_name);
cr_str(c, &r->imports.items[i].module_path);
}
cr_array(c, (void **)&r->usages.items, r->usages.count, sizeof(CBMUsage));
for (int i = 0; i < r->usages.count && r->usages.items; i++) {
cr_str(c, &r->usages.items[i].ref_name);
cr_str(c, &r->usages.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->throws.items, r->throws.count, sizeof(CBMThrow));
for (int i = 0; i < r->throws.count && r->throws.items; i++) {
cr_str(c, &r->throws.items[i].exception_name);
cr_str(c, &r->throws.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->rw.items, r->rw.count, sizeof(CBMReadWrite));
for (int i = 0; i < r->rw.count && r->rw.items; i++) {
cr_str(c, &r->rw.items[i].var_name);
cr_str(c, &r->rw.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->type_refs.items, r->type_refs.count, sizeof(CBMTypeRef));
for (int i = 0; i < r->type_refs.count && r->type_refs.items; i++) {
cr_str(c, &r->type_refs.items[i].type_name);
cr_str(c, &r->type_refs.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->env_accesses.items, r->env_accesses.count, sizeof(CBMEnvAccess));
for (int i = 0; i < r->env_accesses.count && r->env_accesses.items; i++) {
cr_str(c, &r->env_accesses.items[i].env_key);
cr_str(c, &r->env_accesses.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->type_assigns.items, r->type_assigns.count, sizeof(CBMTypeAssign));
for (int i = 0; i < r->type_assigns.count && r->type_assigns.items; i++) {
cr_str(c, &r->type_assigns.items[i].var_name);
cr_str(c, &r->type_assigns.items[i].type_name);
cr_str(c, &r->type_assigns.items[i].enclosing_func_qn);
}
cr_array(c, (void **)&r->impl_traits.items, r->impl_traits.count, sizeof(CBMImplTrait));
for (int i = 0; i < r->impl_traits.count && r->impl_traits.items; i++) {
cr_str(c, &r->impl_traits.items[i].trait_name);
cr_str(c, &r->impl_traits.items[i].struct_name);
cr_str(c, &r->impl_traits.items[i].struct_qn);
}
cr_array(c, (void **)&r->resolved_calls.items, r->resolved_calls.count,
sizeof(CBMResolvedCall));
for (int i = 0; i < r->resolved_calls.count && r->resolved_calls.items; i++) {
cr_str(c, &r->resolved_calls.items[i].caller_qn);
cr_str(c, &r->resolved_calls.items[i].callee_qn);
cr_str(c, &r->resolved_calls.items[i].strategy);
cr_str(c, &r->resolved_calls.items[i].reason);
}
cr_array(c, (void **)&r->string_refs.items, r->string_refs.count, sizeof(CBMStringRef));
for (int i = 0; i < r->string_refs.count && r->string_refs.items; i++) {
cr_str(c, &r->string_refs.items[i].value);
cr_str(c, &r->string_refs.items[i].enclosing_func_qn);
cr_str(c, &r->string_refs.items[i].key_path);
}
cr_array(c, (void **)&r->infra_bindings.items, r->infra_bindings.count,
sizeof(CBMInfraBinding));
for (int i = 0; i < r->infra_bindings.count && r->infra_bindings.items; i++) {
cr_str(c, &r->infra_bindings.items[i].source_name);
cr_str(c, &r->infra_bindings.items[i].target_url);
cr_str(c, &r->infra_bindings.items[i].broker);
}
cr_array(c, (void **)&r->channels.items, r->channels.count, sizeof(CBMChannel));
for (int i = 0; i < r->channels.count && r->channels.items; i++) {
cr_str(c, &r->channels.items[i].channel_name);
cr_str(c, &r->channels.items[i].transport);
cr_str(c, &r->channels.items[i].enclosing_func_qn);
}
cr_str(c, &r->module_qn);
cr_str(c, &r->namespace_name);
cr_list(c, &r->exports);
cr_list(c, &r->constants);
cr_list(c, &r->global_vars);
cr_list(c, &r->macros);
cr_str(c, &r->error_msg);
cr_str(c, &r->error_ranges);
cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0);
}
void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base) {
if (!result || !old_base || !new_base || len == 0 || old_base == new_base) {
return;
}
cr_ctx_t c;
memset(&c, 0, sizeof(c));
c.phase = CR_RELOCATE;
c.old_base = old_base;
c.old_len = len;
c.delta = new_base - old_base;
cr_walk(&c, result);
}
static size_t cr_pow2_at_least(size_t n) {
size_t cap = CR_MIN_TABLE;
while (cap < n) {
cap *= PAIR_LEN;
}
return cap;
}
void cbm_result_compact(CBMFileResult *result) {
if (!result || result->arena.nblocks == 0) {
return;
}
cr_ctx_t c;
memset(&c, 0, sizeof(c));
/* Work on a copy of the header: every pointer rewrite lands here and the
* caller's result is replaced only once everything succeeded. */
CBMFileResult tmp = *result;
c.phase = CR_COUNT;
cr_walk(&c, &tmp);
c.cap = cr_pow2_at_least(c.refs * CR_TABLE_LOAD + CR_MIN_TABLE);
c.slots = (cr_slot_t *)cbm_calloc(CBM_MEM_CLASS_EXTRACT, c.cap * sizeof(cr_slot_t));
if (!c.slots) {
return;
}
c.phase = CR_MEASURE;
cr_walk(&c, &tmp);
for (size_t i = 0; i < c.cap; i++) {
c.slots[i].dst = NULL; /* MEASURE used dst as a booked marker */
}
CBMArena fresh;
cbm_arena_init_exact(&fresh, c.bytes);
if (fresh.nblocks == 0) {
cbm_free(CBM_MEM_CLASS_EXTRACT, c.slots);
return;
}
c.phase = CR_COPY;
c.dst = &fresh;
cr_walk(&c, &tmp);
cbm_free(CBM_MEM_CLASS_EXTRACT, c.slots);
if (c.failed) {
cbm_arena_destroy(&fresh);
return;
}
/* Exact-count arrays: nothing may append into the dead headroom. */
tmp.defs.cap = tmp.defs.count;
tmp.calls.cap = tmp.calls.count;
tmp.imports.cap = tmp.imports.count;
tmp.usages.cap = tmp.usages.count;
tmp.throws.cap = tmp.throws.count;
tmp.rw.cap = tmp.rw.count;
tmp.type_refs.cap = tmp.type_refs.count;
tmp.env_accesses.cap = tmp.env_accesses.count;
tmp.type_assigns.cap = tmp.type_assigns.count;
tmp.impl_traits.cap = tmp.impl_traits.count;
tmp.resolved_calls.cap = tmp.resolved_calls.count;
tmp.string_refs.cap = tmp.string_refs.count;
tmp.infra_bindings.cap = tmp.infra_bindings.count;
tmp.channels.cap = tmp.channels.count;
/* A composite kept its per-unit results only so shallow-copied strings
* stayed valid; every string is now a copy of its own. */
cbm_result_release_owned(result);
tmp.owned_results = NULL;
tmp.owned_result_count = 0;
cbm_work_arena_give(&result->arena); /* kept for this thread's next file */
tmp.arena = fresh;
*result = tmp;
}
internal/cbm/result_spill.c
+278
View File
@@ -0,0 +1,278 @@
/*
* result_spill.c — see result_spill.h.
*
* Layout on disk, per parked result:
* spill_rec_hdr_t (magic, block length, block base address at park time,
* the CBMFileResult header as it was in memory)
* block bytes (the single compacted arena block)
*
* The header's pointers are meaningless on disk; the loader rebuilds the
* arena at a new address and shifts every pointer by the delta
* (cbm_result_relocate, implemented on the compaction traversal so it sees
* exactly the fields compaction copied).
*/
#include "result_spill.h"
#include "foundation/arena.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/constants.h"
#include "foundation/log.h"
#include "foundation/mem_core.h"
/* The park writes a result as an opaque image: the record header (a struct
* copy) and the compacted arena block. Both carry padding bytes no code ever
* wrote -- inside structs, between objects -- and MemorySanitizer tracks
* that mark through the compaction's memcpy, so it refuses the fwrite of an
* image that is read back whole and never interpreted byte by byte (CI MSan
* lane on #2202: offset 4087, then 6714, of a 6,952-byte block). Under MSan
* the image is declared defined right before the write; every other build
* compiles this to nothing. */
#include "foundation/sanitized.h" /* __has_feature exists everywhere, cppcheck included */
#if __has_feature(memory_sanitizer)
#include <sanitizer/msan_interface.h>
#define SPILL_IMAGE_DEFINED(p, n) __msan_unpoison((p), (n))
#else
#define SPILL_IMAGE_DEFINED(p, n) ((void)0)
#endif
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#include <process.h>
#define SPILL_PID() ((long)_getpid())
#define SPILL_SEEK(fp, off) _fseeki64((fp), (long long)(off), SEEK_SET)
#else
#include <unistd.h>
#define SPILL_PID() ((long)getpid())
#define SPILL_SEEK(fp, off) fseeko((fp), (off_t)(off), SEEK_SET)
#endif
static const uint64_t SPILL_MAGIC = 0x5350494C4C524553ULL; /* "SPILLRES" */
typedef struct {
uint64_t magic;
uint64_t block_len;
uint64_t old_base; /* (uintptr_t) of the block when parked */
CBMFileResult header;
} spill_rec_hdr_t;
typedef struct {
FILE *fp;
cbm_mutex_t mu; /* reads seek; writes append -- one lock per file */
char path[CBM_SZ_1K];
uint64_t end; /* bytes written so far (append offset) */
} spill_file_t;
typedef struct {
int writer; /* -1 = empty */
uint64_t offset; /* record start in that writer's file */
uint64_t rec_len; /* header + block */
} spill_slot_t;
struct cbm_result_spill {
spill_file_t *files;
int writers;
spill_slot_t *slots;
int slot_count;
_Atomic int64_t parked;
_Atomic int64_t bytes;
_Atomic int64_t loads;
};
cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots) {
if (!dir || !dir[0] || writers <= 0 || slots <= 0) {
return NULL;
}
char spill_dir[CBM_SZ_1K];
if (snprintf(spill_dir, sizeof(spill_dir), "%s/spill", dir) >= (int)sizeof(spill_dir)) {
return NULL;
}
if (!cbm_mkdir_p(spill_dir, 0700)) {
cbm_log_warn("mem.spill.open_failed", "dir", spill_dir, "reason", "mkdir");
return NULL;
}
cbm_result_spill_t *sp = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*sp));
if (!sp) {
return NULL;
}
sp->files = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)writers * sizeof(spill_file_t));
sp->slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)slots * sizeof(spill_slot_t));
if (!sp->files || !sp->slots) {
cbm_result_spill_close(sp);
return NULL;
}
sp->writers = writers;
sp->slot_count = slots;
for (int i = 0; i < slots; i++) {
sp->slots[i].writer = -1;
}
for (int w = 0; w < writers; w++) {
spill_file_t *f = &sp->files[w];
snprintf(f->path, sizeof(f->path), "%s/spill-%ld-%d.bin", spill_dir, SPILL_PID(), w);
f->fp = cbm_fopen(f->path, "w+b");
if (!f->fp) {
cbm_log_warn("mem.spill.open_failed", "path", f->path, "reason", "fopen");
cbm_result_spill_close(sp);
return NULL;
}
cbm_mutex_init(&f->mu);
}
cbm_log_info("mem.spill.open", "dir", spill_dir, "writers", writers > 0 ? "yes" : "no");
return sp;
}
bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot) {
return sp && slot >= 0 && slot < sp->slot_count && sp->slots[slot].writer >= 0;
}
bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result) {
if (!sp || !result || writer < 0 || writer >= sp->writers || slot < 0 ||
slot >= sp->slot_count || sp->slots[slot].writer >= 0) {
return false;
}
/* Only a compacted result is one block with every pointer inside it; a
* result that owns sub-results (embedded languages) has more arenas than
* that and stays in memory. A retained parse tree is a re-parse cache,
* not data: it is dropped with the in-memory result below. */
if (result->arena.nblocks != 1 || result->owned_result_count != 0) {
return false;
}
spill_file_t *f = &sp->files[writer];
spill_rec_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.magic = SPILL_MAGIC;
hdr.block_len = result->arena.used; /* bytes actually written into the block */
hdr.old_base = (uint64_t)(uintptr_t)result->arena.blocks[0];
hdr.header = *result;
hdr.header.cached_tree = NULL; /* never on disk: the loader gets no tree */
SPILL_IMAGE_DEFINED(&hdr, sizeof(hdr));
SPILL_IMAGE_DEFINED(result->arena.blocks[0], hdr.block_len);
cbm_mutex_lock(&f->mu);
uint64_t offset = f->end;
bool ok = SPILL_SEEK(f->fp, offset) == 0 && fwrite(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
(hdr.block_len == 0 || fwrite(result->arena.blocks[0], hdr.block_len, 1, f->fp) == 1);
if (ok) {
f->end = offset + sizeof(hdr) + hdr.block_len;
}
cbm_mutex_unlock(&f->mu);
if (!ok) {
cbm_log_warn("mem.spill.write_failed", "path", f->path);
return false;
}
sp->slots[slot].writer = writer;
sp->slots[slot].offset = offset;
sp->slots[slot].rec_len = sizeof(hdr) + hdr.block_len;
atomic_fetch_add_explicit(&sp->parked, 1, memory_order_relaxed);
atomic_fetch_add_explicit(&sp->bytes, (int64_t)(sizeof(hdr) + hdr.block_len),
memory_order_relaxed);
cbm_free_result(result);
return true;
}
CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot) {
if (!cbm_result_spill_has(sp, slot)) {
return NULL;
}
const spill_slot_t *s = &sp->slots[slot];
spill_file_t *f = &sp->files[s->writer];
spill_rec_hdr_t hdr;
cbm_mutex_lock(&f->mu);
bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
hdr.magic == SPILL_MAGIC;
CBMFileResult *r = NULL;
if (ok) {
r = cbm_result_alloc();
if (r) {
*r = hdr.header;
r->cached_tree = NULL;
memset(&r->arena, 0, sizeof(r->arena));
cbm_arena_init_exact(&r->arena, hdr.block_len ? (size_t)hdr.block_len : 1);
if (r->arena.nblocks == 1 &&
(hdr.block_len == 0 ||
fread(r->arena.blocks[0], (size_t)hdr.block_len, 1, f->fp) == 1)) {
r->arena.used = (size_t)hdr.block_len;
r->arena.total_alloc = (size_t)hdr.block_len;
} else {
ok = false;
}
} else {
ok = false;
}
}
cbm_mutex_unlock(&f->mu);
if (!ok) {
cbm_log_warn("mem.spill.read_failed", "path", f->path);
if (r) {
cbm_free_result(r);
}
return NULL;
}
cbm_result_relocate(r, (const char *)(uintptr_t)hdr.old_base, (size_t)hdr.block_len,
r->arena.blocks[0]);
atomic_fetch_add_explicit((_Atomic int64_t *)&sp->loads, 1, memory_order_relaxed);
return r;
}
bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out) {
if (!out || !cbm_result_spill_has(sp, slot)) {
return false;
}
const spill_slot_t *s = &sp->slots[slot];
spill_file_t *f = &sp->files[s->writer];
spill_rec_hdr_t hdr;
cbm_mutex_lock(&f->mu);
bool ok = SPILL_SEEK(f->fp, s->offset) == 0 && fread(&hdr, sizeof(hdr), 1, f->fp) == 1 &&
hdr.magic == SPILL_MAGIC;
cbm_mutex_unlock(&f->mu);
if (ok) {
*out = hdr.header;
}
return ok;
}
void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls) {
CBMFileResult hdr;
bool ok = cbm_result_spill_peek_header(sp, slot, &hdr);
if (defs) {
*defs = ok ? hdr.defs.count : 0;
}
if (impls) {
*impls = ok ? hdr.impl_traits.count : 0;
}
}
void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes,
int64_t *loads) {
if (parked) {
*parked = sp ? atomic_load_explicit(&sp->parked, memory_order_relaxed) : 0;
}
if (bytes) {
*bytes = sp ? atomic_load_explicit(&sp->bytes, memory_order_relaxed) : 0;
}
if (loads) {
*loads = sp ? atomic_load_explicit(&sp->loads, memory_order_relaxed) : 0;
}
}
void cbm_result_spill_close(cbm_result_spill_t *sp) {
if (!sp) {
return;
}
if (sp->files) {
for (int w = 0; w < sp->writers; w++) {
spill_file_t *f = &sp->files[w];
if (f->fp) {
(void)fclose(f->fp);
(void)cbm_unlink(f->path);
cbm_mutex_destroy(&f->mu);
}
}
cbm_free(CBM_MEM_CLASS_OTHER, sp->files);
}
cbm_free(CBM_MEM_CLASS_OTHER, sp->slots);
cbm_free(CBM_MEM_CLASS_OTHER, sp);
}
internal/cbm/result_spill.h
+65
View File
@@ -0,0 +1,65 @@
/*
* result_spill.h — per-file extraction results parked on disk when the
* memory budget is hit.
*
* A compacted CBMFileResult is one exact-size arena block plus a header whose
* pointers all point into that block (cbm_result_compact guarantees it). That
* makes a result a relocatable blob: written as header + block, read back at
* any address by shifting every pointer by the base delta. The store is one
* append-only file per writer thread under the cache directory; loads use
* pread and are safe from any thread. Nothing here decides WHEN to spill --
* the pipeline does, on the budget signal (see cbm_pipeline_ctx_t.spill).
*/
#ifndef CBM_RESULT_SPILL_H
#define CBM_RESULT_SPILL_H
#include "cbm.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
typedef struct cbm_result_spill cbm_result_spill_t;
/* One store for one pipeline run: `writers` append-only files under
* <dir>/spill-<pid>-<w>.bin, `slots` result slots (one per input file).
* NULL on failure (no directory, no space): the caller keeps results in
* memory as before. */
cbm_result_spill_t *cbm_result_spill_open(const char *dir, int writers, int slots);
/* Park a compacted result in slot `slot` via writer `writer`, then free the
* in-memory result (a retained parse tree goes with it; a loaded result has
* none). Returns false (and leaves the result untouched) when the result is
* not a single-block compaction, owns sub-results, or the write fails. */
bool cbm_result_spill_park(cbm_result_spill_t *sp, int writer, int slot, CBMFileResult *result);
/* True when slot `slot` holds a parked result. */
bool cbm_result_spill_has(const cbm_result_spill_t *sp, int slot);
/* Load slot `slot` back into memory: a fresh CBMFileResult the caller owns
* (cbm_free_result). NULL when the slot is empty or the read fails. Safe from
* several threads for different slots. */
CBMFileResult *cbm_result_spill_load(const cbm_result_spill_t *sp, int slot);
/* Read only the parked header of slot `slot` into *out: every count in it
* is valid, every pointer meaningless. False when the slot is empty or the
* read fails. Lets a consumer skip the load when there is nothing to read. */
bool cbm_result_spill_peek_header(const cbm_result_spill_t *sp, int slot, CBMFileResult *out);
/* Read only the parked header of slot `slot`: how many defs and impl
* relations it holds (the collector sizes its array before loading). */
void cbm_result_spill_peek_counts(const cbm_result_spill_t *sp, int slot, int *defs, int *impls);
/* Counters for the log: results parked, bytes on disk, loads served. */
void cbm_result_spill_stats(const cbm_result_spill_t *sp, int64_t *parked, int64_t *bytes,
int64_t *loads);
/* Close and delete the files. */
void cbm_result_spill_close(cbm_result_spill_t *sp);
/* Relocation primitive used by the loader: every pointer inside `result`
* that points into [old_base, old_base + len) is shifted to the block now at
* `new_base`. Exposed for tests. */
void cbm_result_relocate(CBMFileResult *result, const char *old_base, size_t len, char *new_base);
#endif /* CBM_RESULT_SPILL_H */
pkg/chocolatey/codebase-memory-mcp.nuspec
+2 -2
View File
@@ -2,7 +2,7 @@
<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
<metadata>
<id>codebase-memory-mcp</id>
<version>0.10.8</version>
<version>0.11.0</version>
<title>codebase-memory-mcp</title>
<authors>DeusData</authors>
<projectUrl>https://github.com/DeusData/codebase-memory-mcp</projectUrl>
@@ -28,7 +28,7 @@ After installation, run:
to configure your coding agents automatically.
</description>
<releaseNotes>https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.10.8</releaseNotes>
<releaseNotes>https://github.com/DeusData/codebase-memory-mcp/releases/tag/v0.11.0</releaseNotes>
</metadata>
<files>
<file src="tools\**" target="tools" />
pkg/chocolatey/tools/chocolateyInstall.ps1
+2 -2
View File
@@ -1,9 +1,9 @@
$ErrorActionPreference = 'Stop'
$packageName = 'codebase-memory-mcp'
$version = '0.10.8'
$version = '0.11.0'
$url64 = "https://github.com/DeusData/codebase-memory-mcp/releases/download/v${version}/codebase-memory-mcp-windows-amd64.zip"
$checksum64 = 'b43ad982994c4d829670749e08d3b622a74bb20041fc0a7d02bef6113f81c34d'
$checksum64 = '6eb6beaf261b19e419766e78baf93cbc3cf1c6338cff8fb7c0234859f96d1685'
$installDir = Join-Path $env:ChocolateyBinRoot $packageName
Install-ChocolateyZipPackage `
pkg/go/cmd/codebase-memory-mcp/main.go
+1 -1
View File
@@ -36,7 +36,7 @@ import (
const (
repo = "DeusData/codebase-memory-mcp"
version = "0.10.8"
version = "0.11.0"
windowsBinaryName = "codebase-memory-mcp.exe"
maxRedirects = 5
pkg/npm/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"description": "Fast code intelligence for AI coding agents — installs a verified native runtime set",
"mcpName": "io.github.DeusData/codebase-memory-mcp",
"license": "MIT",
pkg/pypi/pyproject.toml
+1 -1
View File
@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
[project]
name = "codebase-memory-mcp"
version = "0.10.8"
version = "0.11.0"
description = "Fast code intelligence for AI coding agents — installs a verified native runtime set"
readme = "README.md"
license = { text = "MIT" }
pkg/pypi/src/codebase_memory_mcp/_cli.py
+1 -1
View File
@@ -338,7 +338,7 @@ def _version() -> str:
from importlib.metadata import version
return version("codebase-memory-mcp")
except Exception:
return "0.10.8"
return "0.11.0"
def _os_name() -> str:
scripts/lint-memory-core.py
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
"""
lint-memory-core.py — the memory-core linter.
Memory in this project is allocated through ONE core (src/foundation/mem_core.h),
not through scattered raw malloc/calloc/realloc/free/strdup. This linter is the
gate that keeps that true.
WHY A RATCHET, NOT A BAN. An audit on 2026-09-13 found ~800 raw allocation
sites in src/ and none of them can be migrated in one change. So the gate works
like every other honest debt gate: a checked-in baseline records how many raw
sites each file has TODAY, and the build goes red the moment any file has MORE
than its baseline, or a file not in the baseline grows one. Files can only ever
go down. When a file is migrated, its baseline line is lowered (or removed) in
the same change -- and --strict turns "below baseline" into a failure too, so
the baseline cannot silently rot behind the code.
WHAT COUNTS AS RAW. A call to malloc/calloc/realloc/free/strdup/strndup that is
not part of a longer identifier. cbm_alloc, cbm_free, cbm_calloc, mi_malloc,
cbm_arena_alloc and heap_strdup are all NOT matches: the character before the
name is an identifier character. Comments and string literals are stripped
first, so prose that mentions malloc( does not trip the gate -- the security
audit already bit us once on exactly that with fork(.
EXEMPT. The core itself and the allocator plumbing it sits on:
src/foundation/mem_core.c the route
(arena.c allocates its blocks THROUGH the core -- class arena -- and is scanned)
src/foundation/slab_alloc.c same
src/foundation/mem.c policy/measurement, probes with malloc
src/foundation/mem_override_*.c the --wrap / override shims
src/foundation/compat*.c libc replacement surface
internal/**/vendored/** not ours
vendored/** not ours
SCOPE. Everything under src/ and internal/ -- cli, mcp, daemon, store,
pipeline, the extraction engine -- so the count is for the whole project, not
one subsystem, and a leak shows up wherever it is.
Usage:
lint-memory-core.py check against the baseline (CI)
lint-memory-core.py --strict also fail when a file is BELOW baseline
lint-memory-core.py --write-baseline regenerate scripts/memory-core-baseline.txt
lint-memory-core.py --list print every raw site (file:line: call)
Exit 0 = clean. Exit 1 = a file grew. Exit 2 = usage/IO error.
"""
from __future__ import annotations
import argparse
import pathlib
import re
import sys
ROOT = pathlib.Path(__file__).resolve().parent.parent
BASELINE = ROOT / "scripts" / "memory-core-baseline.txt"
SCAN_ROOTS = ("src", "internal")
RAW = re.compile(r"(?<![A-Za-z0-9_])(malloc|calloc|realloc|free|strdup|strndup)\s*\(")
EXEMPT_EXACT = {
"src/foundation/mem_core.c",
"src/foundation/mem.c",
}
EXEMPT_PREFIX = (
"src/foundation/mem_override_",
"src/foundation/compat",
"vendored/",
)
def is_vendored(rel: str) -> bool:
"""Any vendored/ segment anywhere under internal/ is not ours."""
return "/vendored/" in rel or rel.startswith("vendored/")
def is_exempt(rel: str) -> bool:
return rel in EXEMPT_EXACT or rel.startswith(EXEMPT_PREFIX) or is_vendored(rel)
def strip_comments_and_strings(text: str) -> str:
"""Blank out comments and string literals, preserving line structure so
reported line numbers stay right. Character-class aware enough for C:
handles escapes inside strings and does not treat // inside a string as a
comment."""
out = []
i, n = 0, len(text)
while i < n:
c = text[i]
nxt = text[i + 1] if i + 1 < n else ""
if c == "/" and nxt == "*":
j = text.find("*/", i + 2)
j = n if j < 0 else j + 2
out.append("".join(ch if ch == "\n" else " " for ch in text[i:j]))
i = j
elif c == "/" and nxt == "/":
j = text.find("\n", i)
j = n if j < 0 else j
out.append(" " * (j - i))
i = j
elif c == '"' or c == "'":
q = c
j = i + 1
while j < n and text[j] != q:
if text[j] == "\\":
j += 1
if j < n and text[j] == "\n":
break
j += 1
j = min(j + 1, n)
out.append(q + " " * max(0, j - i - 2) + (q if j - i >= 2 else ""))
i = j
else:
out.append(c)
i += 1
return "".join(out)
def scan() -> dict[str, list[tuple[int, str]]]:
hits: dict[str, list[tuple[int, str]]] = {}
for root in SCAN_ROOTS:
base = ROOT / root
if not base.exists():
continue
for path in sorted(base.rglob("*")):
if path.suffix not in (".c", ".h") or not path.is_file():
continue
rel = path.relative_to(ROOT).as_posix()
if is_exempt(rel):
continue
text = strip_comments_and_strings(path.read_text(encoding="utf-8", errors="replace"))
for lineno, line in enumerate(text.splitlines(), 1):
for m in RAW.finditer(line):
hits.setdefault(rel, []).append((lineno, m.group(1)))
return hits
def read_baseline() -> dict[str, int]:
if not BASELINE.exists():
return {}
out: dict[str, int] = {}
for raw in BASELINE.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
rel, _, count = line.rpartition("\t")
if not rel:
continue
try:
out[rel] = int(count)
except ValueError:
continue
return out
def write_baseline(hits: dict[str, list]) -> None:
lines = [
"# memory-core-baseline.txt -- raw allocator call sites per file.",
"# Generated by scripts/lint-memory-core.py --write-baseline.",
"# A file may only ever go DOWN. Lower a line in the same change that",
"# migrates the file to src/foundation/mem_core.h; never raise one.",
"",
]
for rel in sorted(hits):
lines.append(f"{rel}\t{len(hits[rel])}")
BASELINE.write_text("\n".join(lines) + "\n", encoding="utf-8")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--write-baseline", action="store_true")
ap.add_argument("--strict", action="store_true", help="fail when a file is BELOW its baseline")
ap.add_argument("--list", action="store_true", help="print every raw site")
args = ap.parse_args()
hits = scan()
total = sum(len(v) for v in hits.values())
if args.write_baseline:
write_baseline(hits)
print(f"memory-core baseline written: {len(hits)} files, {total} raw sites")
return 0
if args.list:
for rel in sorted(hits):
for lineno, call in hits[rel]:
print(f"{rel}:{lineno}: {call}(")
print(f"-- {total} raw sites in {len(hits)} files")
return 0
baseline = read_baseline()
if not baseline:
print(f"ERROR: no baseline at {BASELINE.relative_to(ROOT)}; run --write-baseline", file=sys.stderr)
return 2
grew: list[str] = []
shrank: list[str] = []
for rel, sites in sorted(hits.items()):
now = len(sites)
allowed = baseline.get(rel, 0)
if now > allowed:
# Show the LAST sites: new code is usually appended, so these are the
# likeliest culprits. The linter cannot know which sites are new
# without a diff; --list prints them all.
delta = now - allowed
tail = ", ".join(f"{ln}:{c}" for ln, c in sites[-max(delta, 1):][-6:])
grew.append(f" {rel}: grew by {delta} ({allowed} -> {now}); latest sites: {tail}"
f" [run --list for all]")
elif now < allowed:
shrank.append(f" {rel}: {now} (baseline {allowed}) -- lower the baseline")
for rel, allowed in sorted(baseline.items()):
if rel not in hits and allowed > 0:
shrank.append(f" {rel}: 0 (baseline {allowed}) -- remove the line")
if grew:
print("memory-core linter FAILED: raw allocator use grew. Allocate through")
print("src/foundation/mem_core.h (cbm_alloc/cbm_calloc/cbm_realloc/cbm_free/")
print("cbm_mem_strdup) instead of malloc/calloc/realloc/free/strdup.")
print("\n".join(grew))
if shrank:
print("memory-core ratchet: these files improved; tighten scripts/memory-core-baseline.txt:")
print("\n".join(shrank))
if grew or (args.strict and shrank):
return 1
print(f"memory-core linter passed: {total} raw sites across {len(hits)} files, none grew")
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/memory-core-baseline.txt
+90
View File
@@ -0,0 +1,90 @@
# memory-core-baseline.txt -- raw allocator call sites per file.
# Generated by scripts/lint-memory-core.py --write-baseline.
# A file may only ever go DOWN. Lower a line in the same change that
# migrates the file to src/foundation/mem_core.h; never raise one.
internal/cbm/ac.c 20
internal/cbm/arena.c 3
internal/cbm/cbm.c 13
internal/cbm/extract_defs.c 5
internal/cbm/helpers.c 3
internal/cbm/lsp/c_lsp.c 4
internal/cbm/lsp/perl_lsp.c 21
internal/cbm/lsp/py_lsp.c 2
internal/cbm/macro_table.c 1
internal/cbm/service_patterns.c 2
internal/cbm/sqlite_writer.c 92
src/cli/activation_transaction.c 80
src/cli/agent_clients.c 73
src/cli/agent_profiles.c 7
src/cli/cli.c 173
src/cli/client_adapter.c 3
src/cli/config_edit_path.c 2
src/cli/config_json_like.c 142
src/cli/config_text_edit.c 72
src/cli/config_toml_edit.c 85
src/cli/config_yaml_edit.c 117
src/cli/hook_augment.c 31
src/cypher/cypher.c 132
src/daemon/application.c 107
src/daemon/bootstrap.c 13
src/daemon/daemon.c 31
src/daemon/frontend.c 12
src/daemon/ipc.c 146
src/daemon/project_lock.c 6
src/daemon/runtime.c 45
src/daemon/service.c 6
src/daemon/version_cohort.c 9
src/discover/discover.c 31
src/discover/gitignore.c 12
src/discover/userconfig.c 15
src/foundation/diagnostics.c 7
src/foundation/lock_registry.c 18
src/foundation/platform.c 11
src/foundation/platform.h 5
src/foundation/private_file_lock.c 56
src/foundation/str_intern.c 7
src/foundation/subprocess.c 24
src/foundation/win_utf8.h 17
src/foundation/workspace.c 3
src/foundation/yaml.c 10
src/git/git_context.c 18
src/main.c 32
src/mcp/compact_out.c 39
src/mcp/index_supervisor.c 8
src/mcp/mcp.c 814
src/pipeline/artifact.c 38
src/pipeline/fqn.c 23
src/pipeline/lsp_resolve.h 4
src/pipeline/lsp_surface.c 5
src/pipeline/pass_calls.c 23
src/pipeline/pass_compile_commands.c 17
src/pipeline/pass_complexity.c 15
src/pipeline/pass_configlink.c 3
src/pipeline/pass_cross_repo.c 8
src/pipeline/pass_definitions.c 14
src/pipeline/pass_enrichment.c 15
src/pipeline/pass_ensemble_routing.c 10
src/pipeline/pass_envscan.c 2
src/pipeline/pass_githistory.c 27
src/pipeline/pass_importance.c 4
src/pipeline/pass_infrascan.c 2
src/pipeline/pass_k8s.c 13
src/pipeline/pass_lsp_cross.c 50
src/pipeline/pass_parallel.c 31
src/pipeline/pass_pkgmap.c 82
src/pipeline/pass_semantic.c 3
src/pipeline/pass_tests.c 3
src/pipeline/pass_usages.c 4
src/pipeline/path_alias.c 42
src/pipeline/pipeline.c 134
src/pipeline/pipeline_delta.c 19
src/pipeline/pipeline_incremental.c 144
src/pipeline/registry.c 19
src/pipeline/worker_pool.c 2
src/store/store.c 358
src/ui/config.c 12
src/ui/http_server.c 31
src/ui/httpd.c 21
src/ui/layout3d.c 66
src/watcher/watcher.c 33
server.json
+3 -3
View File
@@ -8,12 +8,12 @@
"source": "github"
},
"websiteUrl": "https://deusdata.github.io/codebase-memory-mcp/",
"version": "0.10.8",
"version": "0.11.0",
"packages": [
{
"registryType": "npm",
"identifier": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"runtimeHint": "npx",
"transport": {
"type": "stdio"
@@ -22,7 +22,7 @@
{
"registryType": "pypi",
"identifier": "codebase-memory-mcp",
"version": "0.10.8",
"version": "0.11.0",
"runtimeHint": "uvx",
"transport": {
"type": "stdio"
src/foundation/arena.c
+114 -8
View File
@@ -11,11 +11,16 @@
*/
#include "arena.h"
#include "foundation/constants.h"
#include "foundation/mem_core.h"
enum { ARENA_ALIGN = 7, ARENA_GROW_OK = 1 };
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#if defined(__APPLE__)
#include <pthread.h>
#include <xlocale.h>
#endif
#include <stdio.h>
void cbm_arena_init(CBMArena *a) {
@@ -28,27 +33,61 @@ void cbm_arena_init_sized(CBMArena *a, size_t block_size) {
block_size = CBM_SZ_64; /* minimum sanity */
}
a->block_size = block_size;
a->blocks[0] = (char *)malloc(block_size);
a->grow_size = block_size * PAIR_LEN;
a->blocks[0] = (char *)cbm_alloc(CBM_MEM_CLASS_ARENA, block_size);
if (a->blocks[0]) {
a->block_sizes[0] = block_size;
a->nblocks = SKIP_ONE;
}
}
void cbm_arena_init_exact(CBMArena *a, size_t bytes) {
size_t block = (bytes + ARENA_ALIGN) & ~(size_t)ARENA_ALIGN;
cbm_arena_init_sized(a, block);
a->grow_size = CBM_ARENA_DEFAULT_BLOCK_SIZE;
/* An exact block is a compacted result's image and may be written to
* disk whole (result_spill): the alignment gaps between objects and the
* padding inside structs are never written by the copy, and MemorySanitizer
* refuses an fwrite of uninitialized bytes (CI MSan lane on #2202, offset
* 4087 of a 6,952-byte block). Zeroed once here, every byte of the image
* is defined; the cost is one pass over memory the copy is about to touch. */
if (a->blocks[0]) {
memset(a->blocks[0], 0, a->block_sizes[0]);
}
}
static int arena_grow(CBMArena *a, size_t min_size) {
/* A rewound arena still owns blocks past the cursor: use the next one if
* it fits, otherwise drop it and everything after it and grow fresh. */
if (a->cur + SKIP_ONE < a->nblocks && a->blocks[a->cur + SKIP_ONE]) {
if (a->block_sizes[a->cur + SKIP_ONE] >= min_size) {
a->cur++;
a->block_size = a->block_sizes[a->cur];
a->used = 0;
return ARENA_GROW_OK;
}
for (int i = a->cur + SKIP_ONE; i < a->nblocks; i++) {
cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]);
a->blocks[i] = NULL;
a->block_sizes[i] = 0;
}
a->nblocks = a->cur + SKIP_ONE;
}
if (a->nblocks >= CBM_ARENA_MAX_BLOCKS) {
return 0;
}
size_t new_size = a->block_size * PAIR_LEN;
size_t new_size = a->grow_size;
if (new_size < min_size) {
new_size = min_size;
}
char *block = (char *)malloc(new_size);
a->grow_size = new_size * PAIR_LEN;
char *block = (char *)cbm_alloc(CBM_MEM_CLASS_ARENA, new_size);
if (!block) {
return 0;
}
a->blocks[a->nblocks] = block;
a->block_sizes[a->nblocks] = new_size;
a->cur = a->nblocks;
a->nblocks++;
a->block_size = new_size;
a->used = 0;
@@ -69,7 +108,7 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
return NULL;
}
}
char *ptr = a->blocks[a->nblocks - SKIP_ONE] + a->used;
char *ptr = a->blocks[a->cur] + a->used;
a->used += n;
a->total_alloc += n;
return ptr;
@@ -107,10 +146,53 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) {
return dst;
}
/* On macOS every vsnprintf consults the process locale under an unfair lock
* (localeconv_l inside __vfprintf). Eighteen workers formatting type names
* collapsed into that lock on the C# corpus -- the 10 MB JIT test files took
* 68 s each instead of under a second (sampled 2026-09-14) -- and the old
* two-pass form paid it twice per string. Each thread formats with a C locale
* object of its own, so no thread ever waits for another, and the common
* short string is formatted once into a stack buffer. */
#if defined(__APPLE__)
/* One locale object per thread, freed when the thread exits: the object is
* heap memory that the thread-local pointer alone kept, so every worker
* thread that ever formatted a name leaked 1,472 bytes at exit (the macOS
* LSan lane on PR #2202, 16-101 objects per test process). A pthread key
* destructor is the one hook that runs at thread exit for a TLS-held
* resource; the main thread keeps its locale until process exit. */
static _Thread_local locale_t tl_c_locale;
static pthread_key_t tl_c_locale_key;
static pthread_once_t tl_c_locale_once = PTHREAD_ONCE_INIT;
static void arena_c_locale_free(void *loc) {
if (loc) {
freelocale((locale_t)loc);
}
}
static void arena_c_locale_key_init(void) {
(void)pthread_key_create(&tl_c_locale_key, arena_c_locale_free);
}
static locale_t arena_c_locale(void) {
if (!tl_c_locale) {
tl_c_locale = newlocale(LC_ALL_MASK, "C", NULL);
if (tl_c_locale) {
pthread_once(&tl_c_locale_once, arena_c_locale_key_init);
(void)pthread_setspecific(tl_c_locale_key, tl_c_locale);
}
}
return tl_c_locale; /* NULL = the global locale, the pre-fix behaviour */
}
#define ARENA_VSNPRINTF(buf, n, fmt, ap) vsnprintf_l((buf), (n), arena_c_locale(), (fmt), (ap))
#else
#define ARENA_VSNPRINTF(buf, n, fmt, ap) vsnprintf((buf), (n), (fmt), (ap))
#endif
enum { ARENA_SPRINTF_LOCAL = 512 };
char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) {
char local[ARENA_SPRINTF_LOCAL];
va_list args;
va_start(args, fmt);
int needed = vsnprintf(NULL, 0, fmt, args);
int needed = ARENA_VSNPRINTF(local, sizeof(local), fmt, args);
va_end(args);
if (needed < 0) {
return NULL;
@@ -120,35 +202,59 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) {
if (!dst) {
return NULL;
}
if ((size_t)needed < sizeof(local)) {
memcpy(dst, local, (size_t)needed + SKIP_ONE);
return dst;
}
va_start(args, fmt);
vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args);
ARENA_VSNPRINTF(dst, (size_t)needed + SKIP_ONE, fmt, args);
va_end(args);
return dst;
}
void cbm_arena_rewind(CBMArena *a) {
if (!a || a->nblocks == 0) {
return;
}
a->cur = 0;
a->block_size = a->block_sizes[0];
a->used = 0;
a->total_alloc = 0;
}
size_t cbm_arena_capacity(const CBMArena *a) {
size_t total = 0;
for (int i = 0; a && i < a->nblocks; i++) {
total += a->block_sizes[i];
}
return total;
}
void cbm_arena_reset(CBMArena *a) {
/* Keep first block, free the rest */
for (int i = SKIP_ONE; i < a->nblocks; i++) {
free(a->blocks[i]);
cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]);
a->blocks[i] = NULL;
a->block_sizes[i] = 0;
}
if (a->nblocks > SKIP_ONE) {
a->nblocks = SKIP_ONE;
}
a->cur = 0;
a->used = 0;
a->total_alloc = 0;
/* Reset block_size to match surviving block — prevents overflow if
* block_size grew during previous allocations (e.g., CBM_SZ_128 → CBM_SZ_256). */
if (a->nblocks == SKIP_ONE) {
a->block_size = a->block_sizes[0];
a->grow_size = a->block_size * PAIR_LEN;
}
}
void cbm_arena_destroy(CBMArena *a) {
for (int i = 0; i < a->nblocks; i++) {
free(a->blocks[i]);
cbm_free(CBM_MEM_CLASS_ARENA, a->blocks[i]);
}
memset(a, 0, sizeof(*a));
}
src/foundation/arena.h
+18
View File
@@ -24,6 +24,8 @@ typedef struct {
size_t block_size; /* current block capacity */
size_t used; /* bytes used in current block */
size_t total_alloc; /* cumulative bytes allocated (for stats) */
size_t grow_size; /* size of the NEXT block added; doubles per growth */
int cur; /* index of the block allocations come from (rewind sets 0) */
} CBMArena;
/* Initialize arena with default block size. */
@@ -32,6 +34,14 @@ void cbm_arena_init(CBMArena *a);
/* Initialize arena with a custom initial block size. */
void cbm_arena_init_sized(CBMArena *a, size_t block_size);
/* Initialize an arena as ONE block of exactly `bytes` (rounded to alignment),
* with later growth restarting at the default block size rather than doubling
* the exact block. This is the compaction target: a result whose reachable
* data measures N bytes lands in one N-byte block with no tail, and a later
* append (the cross-file LSP pass adds resolved calls) costs one small block,
* not 2N. */
void cbm_arena_init_exact(CBMArena *a, size_t bytes);
/* Allocate n bytes (8-byte aligned). Returns NULL on OOM. */
void *cbm_arena_alloc(CBMArena *a, size_t n);
@@ -50,6 +60,14 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format
/* Reset arena for reuse: keeps first block, frees the rest. */
void cbm_arena_reset(CBMArena *a);
/* Rewind: keep EVERY block, start allocating from the first one again. The
* pages stay mapped and are overwritten by the next use -- no free, no
* purge, no re-commit. This is what a per-worker working arena wants between
* files: the same addresses reused directly. cbm_arena_capacity() says how
* much such an arena holds, so a caller can drop an outsized one. */
void cbm_arena_rewind(CBMArena *a);
size_t cbm_arena_capacity(const CBMArena *a);
/* Free all blocks. Arena is zeroed after this. */
void cbm_arena_destroy(CBMArena *a);
src/foundation/diagnostics.c
+15
View File
@@ -23,6 +23,7 @@
#include <mimalloc.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
@@ -538,6 +539,20 @@ static void write_diagnostics(void) {
* macOS/Windows, and peak is reconciled to never undercut current. */
current_rss = cbm_mem_rss();
peak_rss = cbm_mem_peak_rss();
/* The committed pair has the same failure: the counter is signed and
* merged per thread at thread exit, so a thread-per-connection daemon
* reads it negative once short-lived threads have freed what a long-lived
* one committed (query-leak soak, Linux 2026-09-14: heap_committed_bytes
* printed 2^64 - 121 MB from the second sample on). cbm_mem_allocator_
* committed reports 0 for a negative reading; the peak never undercuts
* the current value and never carries a wrapped one. */
current_commit = cbm_mem_allocator_committed();
if (peak_commit > (SIZE_MAX >> 1)) {
peak_commit = 0;
}
if (peak_commit < current_commit) {
peak_commit = current_commit;
}
int fds = count_open_fds();
time_t now = time(NULL);
src/foundation/dyn_array.h
+53 -31
View File
@@ -20,6 +20,7 @@
#include <stdlib.h>
#include <string.h>
#include "mem_core.h" /* same directory: units without -Isrc include this header relatively */
/* Declare a dynamic array type for a given element type. */
#define CBM_DYN_ARRAY(T) \
@@ -30,25 +31,45 @@
}
/* Push an element. Grows by 2x when full. */
#define cbm_da_push(da, item) \
do { \
if ((da)->count >= (da)->cap) { \
int _new_cap = (da)->cap ? (da)->cap * 2 : 8; \
void *_new = realloc((da)->items, (size_t)_new_cap * sizeof(*(da)->items)); \
if (!_new) \
break; \
(da)->items = _new; \
(da)->cap = _new_cap; \
} \
(da)->items[(da)->count++] = (item); \
#define cbm_da_push(da, item) \
do { \
if ((da)->count >= (da)->cap) { \
int _new_cap = (da)->cap ? (da)->cap * 2 : 8; \
void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \
(size_t)_new_cap * sizeof(*(da)->items)); \
if (!_new) \
break; \
(da)->items = _new; \
(da)->cap = _new_cap; \
} \
(da)->items[(da)->count++] = (item); \
} while (0)
/* Push with a caller-chosen first capacity. cbm_da_push starts at 8 slots,
* which is 64 bytes of headroom on every array; an index that keeps one
* small array per key (millions of keys, most holding one or two entries)
* pays that on every key. Growth stays 2x. */
#define cbm_da_push_min(da, item, min_cap) \
do { \
if ((da)->count >= (da)->cap) { \
int _new_cap = (da)->cap ? (da)->cap * 2 : (min_cap); \
void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \
(size_t)_new_cap * sizeof(*(da)->items)); \
if (!_new) \
break; \
(da)->items = _new; \
(da)->cap = _new_cap; \
} \
(da)->items[(da)->count++] = (item); \
} while (0)
/* Push an element with a pointer return (for in-place init). */
#define cbm_da_push_ptr(da) \
(((da)->count >= (da)->cap \
? ((void)((da)->cap = (da)->cap ? (da)->cap * 2 : 8), \
(void)((da)->items = realloc((da)->items, (size_t)(da)->cap * sizeof(*(da)->items)))) \
: (void)0), \
#define cbm_da_push_ptr(da) \
(((da)->count >= (da)->cap \
? ((void)((da)->cap = (da)->cap ? (da)->cap * 2 : 8), \
(void)((da)->items = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \
(size_t)(da)->cap * sizeof(*(da)->items)))) \
: (void)0), \
&(da)->items[(da)->count++])
/* Pop last element. Returns the element. Undefined if empty. */
@@ -61,24 +82,25 @@
#define cbm_da_clear(da) ((da)->count = 0)
/* Free all memory. */
#define cbm_da_free(da) \
do { \
free((da)->items); \
(da)->items = NULL; \
(da)->count = 0; \
(da)->cap = 0; \
#define cbm_da_free(da) \
do { \
cbm_free(CBM_MEM_CLASS_DYN_ARRAY, (da)->items); \
(da)->items = NULL; \
(da)->count = 0; \
(da)->cap = 0; \
} while (0)
/* Reserve capacity (grow if needed, never shrink). */
#define cbm_da_reserve(da, n) \
do { \
if ((n) > (da)->cap) { \
void *_new = realloc((da)->items, (size_t)(n) * sizeof(*(da)->items)); \
if (_new) { \
(da)->items = _new; \
(da)->cap = (n); \
} \
} \
#define cbm_da_reserve(da, n) \
do { \
if ((n) > (da)->cap) { \
void *_new = cbm_realloc(CBM_MEM_CLASS_DYN_ARRAY, (da)->items, \
(size_t)(n) * sizeof(*(da)->items)); \
if (_new) { \
(da)->items = _new; \
(da)->cap = (n); \
} \
} \
} while (0)
/* Insert at index, shifting elements right. */
src/foundation/hash_table.c
+24 -4
View File
@@ -27,11 +27,26 @@
* include below generates static inline functions named cbm_vt_init,
* cbm_vt_cleanup, cbm_vt_get, cbm_vt_insert, etc., plus the cbm_vt
* struct itself. */
/* Verstable allocates its bucket/entry blocks through the memory core; the
* table's ctx is the class those blocks are charged to (Verstable hands the
* ctx and the block size to both hooks). Without this the graph buffer's
* 8.5M node keys and 15.8M edge keys on the kernel were memory no class
* could see. */
static void *ht_alloc_in(size_t size, cbm_mem_class_t *cls) {
return cbm_alloc(*cls, size);
}
static void ht_free_in(void *ptr, size_t size, cbm_mem_class_t *cls) {
(void)size;
cbm_free(*cls, ptr);
}
#define NAME cbm_vt
#define KEY_TY const char *
#define VAL_TY void *
#define HASH_FN vt_hash_string
#define CMPR_FN vt_cmpr_string
#define CTX_TY cbm_mem_class_t
#define MALLOC_FN ht_alloc_in
#define FREE_FN ht_free_in
#include "../../internal/cbm/vendored/verstable/verstable.h"
/* The opaque CBMHashTable struct holds the Verstable instance + a
@@ -42,16 +57,20 @@ struct CBMHashTable {
};
CBMHashTable *cbm_ht_create(uint32_t initial_capacity) {
CBMHashTable *ht = (CBMHashTable *)calloc(CBM_ALLOC_ONE, sizeof(*ht));
return cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, initial_capacity);
}
CBMHashTable *cbm_ht_create_in(cbm_mem_class_t cls, uint32_t initial_capacity) {
CBMHashTable *ht = (CBMHashTable *)cbm_calloc(cls, sizeof(*ht));
if (!ht)
return NULL;
cbm_vt_init(&ht->vt);
cbm_vt_init(&ht->vt, cls);
if (initial_capacity > 0) {
/* Reserve enough buckets for the requested entries. Verstable
* computes the minimum bucket count internally. */
if (!cbm_vt_reserve(&ht->vt, (size_t)initial_capacity)) {
cbm_vt_cleanup(&ht->vt);
free(ht);
cbm_free(cls, ht);
return NULL;
}
}
@@ -61,8 +80,9 @@ CBMHashTable *cbm_ht_create(uint32_t initial_capacity) {
void cbm_ht_free(CBMHashTable *ht) {
if (!ht)
return;
cbm_mem_class_t cls = ht->vt.ctx;
cbm_vt_cleanup(&ht->vt);
free(ht);
cbm_free(cls, ht);
}
void *cbm_ht_set(CBMHashTable *ht, const char *key, void *value) {
src/foundation/hash_table.h
+6
View File
@@ -15,6 +15,7 @@
#include <stddef.h>
#include <stdint.h>
#include "mem_core.h" /* cbm_mem_class_t; same directory: the lsp_all unit has no -Isrc */
#include <stdbool.h>
/* Opaque — full definition lives in hash_table.c. */
@@ -24,6 +25,11 @@ typedef struct CBMHashTable CBMHashTable;
* buckets and avoid early growth; 0 = library default). */
CBMHashTable *cbm_ht_create(uint32_t initial_capacity);
/* Same, with the memory class the table's buckets and entries are charged
* to. cbm_ht_create charges CBM_MEM_CLASS_HASH_TABLE; an owner that wants
* its indexes attributed (the graph buffer: gbuf_index) names its class. */
CBMHashTable *cbm_ht_create_in(cbm_mem_class_t cls, uint32_t initial_capacity);
/* Free the hash table (does NOT free keys or values). */
void cbm_ht_free(CBMHashTable *ht);
src/foundation/mem.c
+97 -3
View File
@@ -32,6 +32,7 @@
#include <psapi.h>
#elif defined(__APPLE__)
#include <mach/mach.h>
#include <malloc/malloc.h> /* malloc_zone_pressure_relief */
#else
#include <unistd.h>
#endif
@@ -508,10 +509,73 @@ void cbm_mem_set_budget_for_tests(size_t bytes) {
g_budget = bytes;
}
size_t cbm_mem_allocator_committed(void) {
size_t commit = 0;
mi_process_info(NULL, NULL, NULL, NULL, NULL, &commit, NULL, NULL);
/* The statistic behind this is a signed counter merged per thread at
* thread exit; a process whose long-lived thread commits what its
* short-lived threads free reads it NEGATIVE, cast to size_t here. The
* daemon's query-leak soak reported 2^64 - 121 MB from the second sample
* on (Linux, 2026-09-14). A negative reading is no reading: report 0 so
* the charge falls back to the OS number instead of a 16 EB budget breach. */
if (commit > (SIZE_MAX >> 1)) {
return 0;
}
return commit;
}
static _Atomic size_t g_peak_charged;
size_t cbm_mem_charged(void) {
/* The OS number (phys_footprint on macOS, RSS elsewhere) is the charge
* for everything the process maps; the allocator's committed bytes are
* the floor for the memory we hold through mimalloc. macOS was measured
* under-reporting the former after MADV_FREE_REUSABLE cycles (kernel
* extraction: 4.1 GB charged, 15.6 GB committed, 13.4 GB tracked live),
* so the larger of the two is the honest reading. */
#if defined(__APPLE__)
size_t os_charge = cbm_mem_footprint();
if (os_charge == 0) {
os_charge = cbm_mem_rss(); /* footprint unavailable: fall back */
}
#else
size_t os_charge = cbm_mem_rss();
#endif
size_t committed = cbm_mem_allocator_committed();
size_t charged = committed > os_charge ? committed : os_charge;
/* High-water mark of the charge itself, at the granularity of the gate
* that reads it (every file pull, every phase mark). RSS high-water
* counts pages already purged to the OS but not yet reclaimed
* (MADV_FREE); this is the number the budget is measured against. */
size_t seen = atomic_load_explicit(&g_peak_charged, memory_order_relaxed);
while (charged > seen &&
!atomic_compare_exchange_weak_explicit(&g_peak_charged, &seen, charged,
memory_order_relaxed, memory_order_relaxed)) {}
return charged;
}
size_t cbm_mem_peak_charged(void) {
return atomic_load_explicit(&g_peak_charged, memory_order_relaxed);
}
bool cbm_mem_over_budget(void) {
size_t rss = cbm_mem_rss();
check_pressure(rss);
return rss > g_budget;
size_t charged = cbm_mem_charged();
check_pressure(charged);
return charged > g_budget;
}
/* Reclaimable memory below this share of total RAM is where paging starts to
* hurt, so it is the point at which pressing on stops being reasonable. */
enum { MEM_PRESSURE_AVAIL_DIVISOR = 8 }; /* 12.5% of total RAM */
bool cbm_mem_system_under_pressure(void) {
size_t available = cbm_system_available_ram();
if (available == 0) {
return false; /* platform cannot answer - do not abort on a guess */
}
cbm_system_info_t info = cbm_system_info();
if (info.total_ram == 0) {
return false;
}
return available < info.total_ram / MEM_PRESSURE_AVAIL_DIVISOR;
}
size_t cbm_mem_worker_budget(int num_workers) {
@@ -525,6 +589,32 @@ void cbm_mem_collect(void) {
mi_collect(true);
}
size_t cbm_mem_footprint(void) {
#if defined(__APPLE__)
task_vm_info_data_t vm = {0};
mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm, &count) == KERN_SUCCESS) {
return (size_t)vm.phys_footprint;
}
return 0;
#else
return os_rss();
#endif
}
void cbm_mem_release_to_os(void) {
/* mi_collect is the release: on Linux and Windows mimalloc owns malloc, so
* there is no libc heap to trim. glibc's malloc_trim was called here once
* and cost the static Linux release its link: the reference pulls
* libc.a(malloc.o) in beside mimalloc's malloc/free (multiple definition,
* release run 34948714902, 2026-09-15). macOS keeps the system heap for
* everything outside the core, hence the pressure-relief call there. */
mi_collect(true);
#if defined(__APPLE__)
(void)malloc_zone_pressure_relief(NULL, 0);
#endif
}
/* ── Memory map (see mem.h for how to read the triple) ─────────────── */
static const size_t MEM_MAP_BUCKET_LIMITS[CBM_MEM_MAP_BUCKETS] = {
@@ -728,6 +818,10 @@ static bool mem_phase_enabled(void) {
return on;
}
bool cbm_mem_phases_enabled(void) {
return mem_phase_enabled();
}
void cbm_mem_phase_mark(const char *label) {
if (!mem_phase_enabled()) {
return;
src/foundation/mem.h
+52
View File
@@ -56,6 +56,24 @@ size_t cbm_mem_rss(void);
/* Peak RSS in bytes. */
size_t cbm_mem_peak_rss(void);
/* High-water mark of cbm_mem_charged() as seen by its callers (the extract
* gate reads it per file pull, the phase marks per pass): the peak of the
* budget metric itself, next to the RSS peak that counts reclaimable pages. */
size_t cbm_mem_peak_charged(void);
/* The number the budget is enforced against: what the OS charges this
* process. On macOS that is phys_footprint -- resident_size keeps every page
* mimalloc has purged until the kernel reclaims it, and the kernel proof
* (2026-09-13) had the worker at 17.4 GB RSS with 5.5 GB charged after
* extraction. Elsewhere the two are the same number. Falls back to RSS when
* the footprint is unavailable. */
size_t cbm_mem_charged(void);
/* Bytes mimalloc currently has committed (its own counter). With the core
* backed by mimalloc this is the allocator-level truth: tracked live bytes
* plus fragmentation and metadata, minus nothing the OS may or may not have
* reclaimed yet. 0 when unavailable. */
size_t cbm_mem_allocator_committed(void);
/* Total budget in bytes. */
size_t cbm_mem_budget(void);
@@ -75,9 +93,39 @@ bool cbm_mem_over_budget(void);
/* Per-worker budget hint: budget / num_workers. */
size_t cbm_mem_worker_budget(int num_workers);
/* True only when the SYSTEM is genuinely short of memory, not merely when this
* process is over its advisory budget.
*
* The budget is a static fraction of TOTAL ram, so on a large host it refuses
* work the machine can plainly do: measured 2026-09-13, the linux kernel needs
* 31.75 GB of a 48 GB host against a 24 GB budget, and the previous release
* completed the very same index by overshooting to 33.56 GB. Aborting on the
* budget alone therefore turned a working index into a refusal.
*
* So the budget keeps its job as the BACKPRESSURE trigger (workers park, peers
* return transients) and this answers the different question of whether giving
* up is warranted. Returns false when availability is unknown: never abort on
* a guess. */
bool cbm_mem_system_under_pressure(void);
/* Return unused pages to the OS. Call between files to bound per-file peak. */
void cbm_mem_collect(void);
/* What the OS charges this process for memory-pressure purposes. On macOS
* that is phys_footprint, which EXCLUDES pages the allocator has already
* marked reusable (MADV_FREE_REUSABLE) -- resident_size still counts them, so
* after a large free the two can differ by gigabytes. Elsewhere equals RSS.
* 0 when unavailable. */
size_t cbm_mem_footprint(void);
/* Hand freed memory back to the OS on every allocator this process uses:
* mimalloc (mi_collect) and, on macOS, the system zones
* (malloc_zone_pressure_relief). Never glibc's malloc_trim: on Linux mimalloc
* owns malloc, and the reference alone breaks the static release link. Costs
* a few ms; call at phase boundaries after a bulk release, never per
* allocation. */
void cbm_mem_release_to_os(void);
/* ── Memory map: where does the process's memory actually live? ──────
*
* A growth diagnostic that is honest about what it cannot see. Walking the
@@ -208,6 +256,10 @@ void cbm_mem_phase_mark(const char *label);
/* Drop all accumulated phase totals (call once at the start of a measurement). */
void cbm_mem_phase_reset(void);
/* True when CBM_MEM_PHASES=1 turned phase attribution on for this process.
* Instruments that walk large structures (the result census) gate on it. */
bool cbm_mem_phases_enabled(void);
/* Write the phase table as a JSON array of {label, bytes, hits}, biggest total
* first. Returns bytes written (0 when disabled or empty). */
int cbm_mem_phase_report_json(char *out, size_t size);
src/foundation/mem_core.c
+383
View File
@@ -0,0 +1,383 @@
/*
* mem_core.c — the allocation route. See mem_core.h for why it exists.
*/
#include "foundation/mem_core.h"
#include "foundation/mem.h"
/* Ownership check for blocks handed back to the core (defined with cbm_free). */
static void check_owned(const void *block, const char *op);
#include "foundation/constants.h"
#include "foundation/log.h"
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Usable-size query, per platform.
*
* Deliberately NOT mi_usable_size: the mimalloc global override is off on
* macOS (permanently — the two-level namespace aborts on cross-boundary
* frees), so a pointer from plain malloc there is not a mimalloc block and
* mi_usable_size would be undefined behaviour on it. Each platform's own query
* is correct under whichever allocator is actually installed, including when
* that allocator IS mimalloc via the Linux/MinGW override. */
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
#include <mimalloc.h>
#define CBM_BACKING_MALLOC(n) mi_malloc(n)
#define CBM_BACKING_CALLOC(n) mi_calloc(CBM_ALLOC_ONE, n)
#define CBM_BACKING_REALLOC(p, n) mi_realloc(p, n)
#define CBM_BACKING_FREE(p) mi_free(p)
#define CBM_USABLE_SIZE(p) mi_usable_size((void *)(p))
#elif defined(__APPLE__)
#include <malloc/malloc.h> /* malloc_size */
#define CBM_USABLE_SIZE(p) malloc_size(p)
#elif defined(_WIN32)
#include <malloc.h> /* _msize */
#define CBM_USABLE_SIZE(p) _msize((void *)(p))
#elif defined(__GLIBC__) || defined(__linux__)
#include <malloc.h> /* malloc_usable_size */
#define CBM_USABLE_SIZE(p) malloc_usable_size((void *)(p))
#else
/* BSD and anything unknown: no portable query. Accounting then tracks the
* REQUESTED size, which understates by the rounding. Understating is the safe
* direction for a diagnostic (it never invents memory), and the alternative --
* a per-block header -- costs 1.6 GB at kernel scale. */
#define CBM_USABLE_SIZE_UNAVAILABLE 1
#endif
#ifndef CBM_BACKING_MALLOC
#define CBM_BACKING_MALLOC(n) malloc(n)
#define CBM_BACKING_CALLOC(n) calloc(CBM_ALLOC_ONE, n)
#define CBM_BACKING_REALLOC(p, n) realloc(p, n)
#define CBM_BACKING_FREE(p) free(p)
#endif
enum { MEM_CORE_REPORT_MIN = 64 };
typedef struct {
atomic_size_t live_bytes;
atomic_size_t live_blocks;
atomic_size_t peak_bytes;
} mem_class_stats_t;
static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT];
static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = {
"other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", "extract", "arena",
"ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array",
};
const char *cbm_mem_class_name(cbm_mem_class_t cls) {
if ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) {
return "invalid";
}
return g_class_names[cls];
}
/* Out-of-range classes are folded into OTHER rather than rejected: a
* mis-tagged allocation must still be freed correctly. Accounting accuracy is
* worth less than not corrupting the heap. */
static mem_class_stats_t *class_slot(cbm_mem_class_t cls) {
if ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) {
return &g_classes[CBM_MEM_CLASS_OTHER];
}
return &g_classes[cls];
}
/* ── Accounting: thread-local deltas, shared atomics on flush ──────────
* The hot path (every allocation and free on every worker) touches only
* thread-local memory. The shared per-class counters see one flush per
* MEM_FLUSH_BYTES / MEM_FLUSH_BLOCKS of change per thread, or an explicit
* cbm_mem_class_flush_thread() -- which every parallel-for worker calls when
* its work item ends and every reader calls for its own thread first. With
* one atomic per allocation, 18 workers on 18 cores bounced the same three
* cache lines on every block: Kotlin CPU 38 -> 121 s for a smaller graph,
* Go 177 -> 312 s (bench vs v0.10.8, 2026-09-14). A class's live figure can
* lag a running worker by at most MEM_FLUSH_BYTES; the phase marks read
* after the workers joined, so they are exact. Peaks are recorded at flush
* and are low by at most threads x MEM_FLUSH_BYTES -- a diagnostic. */
enum { MEM_FLUSH_BYTES = 256 * 1024, MEM_FLUSH_BLOCKS = 512 };
typedef struct {
long bytes; /* signed: allocations add, frees subtract */
long blocks;
} mem_delta_t;
static _Thread_local mem_delta_t tl_delta[CBM_MEM_CLASS_COUNT];
/* live += delta, never wrapping below zero: a mismatched class on free (the
* one way a caller can get this wrong) must not turn a small drift into a
* colossal bogus number that looks like a leak. Returns the new value. */
static size_t apply_signed(atomic_size_t *counter, long delta) {
if (delta >= 0) {
return atomic_fetch_add_explicit(counter, (size_t)delta, memory_order_relaxed) +
(size_t)delta;
}
size_t sub = (size_t)(-delta);
size_t seen = atomic_load_explicit(counter, memory_order_relaxed);
while (true) {
size_t want = sub > seen ? 0 : seen - sub;
if (atomic_compare_exchange_weak_explicit(counter, &seen, want, memory_order_relaxed,
memory_order_relaxed)) {
return want;
}
}
}
static void class_flush_one(cbm_mem_class_t cls) {
mem_delta_t *d = &tl_delta[cls];
if (d->bytes == 0 && d->blocks == 0) {
return;
}
long bytes = d->bytes;
long blocks = d->blocks;
d->bytes = 0;
d->blocks = 0;
mem_class_stats_t *st = &g_classes[cls];
size_t now = apply_signed(&st->live_bytes, bytes);
(void)apply_signed(&st->live_blocks, blocks);
if (bytes > 0) {
/* Peak is best-effort under concurrency: racing writers can leave it
* one flush low; that never changes a decision. */
size_t seen = atomic_load_explicit(&st->peak_bytes, memory_order_relaxed);
while (now > seen) {
if (atomic_compare_exchange_weak_explicit(&st->peak_bytes, &seen, now,
memory_order_relaxed, memory_order_relaxed)) {
break;
}
}
}
}
void cbm_mem_class_flush_thread(void) {
for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {
class_flush_one((cbm_mem_class_t)i);
}
}
static cbm_mem_class_t class_index(cbm_mem_class_t cls) {
return ((int)cls < 0 || (int)cls >= CBM_MEM_CLASS_COUNT) ? CBM_MEM_CLASS_OTHER : cls;
}
static void class_add(cbm_mem_class_t cls, size_t bytes, size_t blocks) {
cls = class_index(cls);
mem_delta_t *d = &tl_delta[cls];
d->bytes += (long)bytes;
d->blocks += (long)blocks;
if (d->bytes >= MEM_FLUSH_BYTES || d->blocks >= MEM_FLUSH_BLOCKS) {
class_flush_one(cls);
}
}
static void class_sub(cbm_mem_class_t cls, size_t bytes, size_t blocks) {
cls = class_index(cls);
mem_delta_t *d = &tl_delta[cls];
d->bytes -= (long)bytes;
d->blocks -= (long)blocks;
if (d->bytes <= -MEM_FLUSH_BYTES || d->blocks <= -MEM_FLUSH_BLOCKS) {
class_flush_one(cls);
}
}
#ifdef CBM_USABLE_SIZE_UNAVAILABLE
static size_t charge_size(const void *block, size_t requested) {
(void)block;
return requested;
}
size_t cbm_mem_usable_size(const void *block) {
(void)block;
return 0;
}
#else
static size_t charge_size(const void *block, size_t requested) {
size_t usable = CBM_USABLE_SIZE(block);
return usable ? usable : requested;
}
size_t cbm_mem_usable_size(const void *block) {
if (!block) {
return 0;
}
return CBM_USABLE_SIZE(block);
}
#endif
void *cbm_alloc(cbm_mem_class_t cls, size_t bytes) {
void *block = CBM_BACKING_MALLOC(bytes ? bytes : CBM_ALLOC_ONE);
if (!block) {
return NULL;
}
class_add(cls, charge_size(block, bytes), CBM_ALLOC_ONE);
return block;
}
void *cbm_calloc(cbm_mem_class_t cls, size_t bytes) {
void *block = CBM_BACKING_CALLOC(bytes ? bytes : CBM_ALLOC_ONE);
if (!block) {
return NULL;
}
class_add(cls, charge_size(block, bytes), CBM_ALLOC_ONE);
return block;
}
void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes) {
if (!block) {
return cbm_alloc(cls, bytes);
}
check_owned(block, "realloc");
/* Measure BEFORE: after realloc the old block is gone and its size is
* unknowable, so the decrement has to be computed first. */
size_t old = charge_size(block, 0);
void *next = CBM_BACKING_REALLOC(block, bytes ? bytes : CBM_ALLOC_ONE);
if (!next) {
return NULL; /* original intact and still charged - correct */
}
class_sub(cls, old, 0);
class_add(cls, charge_size(next, bytes), 0);
return next;
}
char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s) {
if (!s) {
return NULL;
}
size_t len = strlen(s) + CBM_ALLOC_ONE;
char *copy = (char *)cbm_alloc(cls, len);
if (!copy) {
return NULL;
}
memcpy(copy, s, len);
return copy;
}
/* A block that did not come from the backing allocator reached the core:
* a cross-allocator free (a libc strdup handed to cbm_free, which is mi_free
* in the production build) that no libc-backed test build can see. Checked
* where the backing is mimalloc and only under CBM_MEM_PHASES=1 -- the proof
* runs -- and fatal there: silent heap corruption is the alternative. */
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
static void check_owned(const void *block, const char *op) {
if (!cbm_mem_phases_enabled() || mi_is_in_heap_region(block)) {
return;
}
cbm_log_error("mem.core.foreign_block", "op", op);
abort();
}
#else
static void check_owned(const void *block, const char *op) {
(void)block;
(void)op;
}
#endif
void cbm_free(cbm_mem_class_t cls, void *block) {
if (!block) {
return;
}
check_owned(block, "free");
class_sub(cls, charge_size(block, 0), CBM_ALLOC_ONE);
CBM_BACKING_FREE(block);
}
void cbm_mem_class_add_external(cbm_mem_class_t cls, size_t bytes) {
class_add(cls, bytes, 0);
}
void cbm_mem_class_remove_external(cbm_mem_class_t cls, size_t bytes) {
class_sub(cls, bytes, 0);
}
size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls) {
cbm_mem_class_flush_thread();
return atomic_load_explicit(&class_slot(cls)->live_bytes, memory_order_relaxed);
}
size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls) {
cbm_mem_class_flush_thread();
return atomic_load_explicit(&class_slot(cls)->live_blocks, memory_order_relaxed);
}
size_t cbm_mem_class_peak_bytes(cbm_mem_class_t cls) {
cbm_mem_class_flush_thread();
return atomic_load_explicit(&class_slot(cls)->peak_bytes, memory_order_relaxed);
}
size_t cbm_mem_tracked_live_bytes(void) {
cbm_mem_class_flush_thread();
size_t total = 0;
for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {
total += atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed);
}
return total;
}
void cbm_mem_class_reset_peaks(void) {
cbm_mem_class_flush_thread();
for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {
size_t live = atomic_load_explicit(&g_classes[i].live_bytes, memory_order_relaxed);
atomic_store_explicit(&g_classes[i].peak_bytes, live, memory_order_relaxed);
}
}
int cbm_mem_class_report_json(char *out, size_t size) {
cbm_mem_class_flush_thread();
if (!out || size < MEM_CORE_REPORT_MIN) {
return 0;
}
int order[CBM_MEM_CLASS_COUNT];
int n = 0;
for (int i = 0; i < CBM_MEM_CLASS_COUNT; i++) {
if (atomic_load_explicit(&g_classes[i].peak_bytes, memory_order_relaxed) > 0) {
order[n++] = i;
}
}
if (n == 0) {
return 0;
}
/* Insertion sort: n is at most CBM_MEM_CLASS_COUNT. */
for (int i = 1; i < n; i++) {
int key = order[i];
size_t kv = atomic_load_explicit(&g_classes[key].live_bytes, memory_order_relaxed);
int j = i - 1;
while (j >= 0 &&
atomic_load_explicit(&g_classes[order[j]].live_bytes, memory_order_relaxed) < kv) {
order[j + 1] = order[j];
j--;
}
order[j + 1] = key;
}
int written = snprintf(out, size, "[");
for (int i = 0; i < n && written > 0 && (size_t)written < size; i++) {
int idx = order[i];
int add = snprintf(out + written, size - (size_t)written,
"%s{\"class\":\"%s\",\"live_bytes\":%zu,\"live_blocks\":%zu,"
"\"peak_bytes\":%zu}",
i ? "," : "", g_class_names[idx],
atomic_load_explicit(&g_classes[idx].live_bytes, memory_order_relaxed),
atomic_load_explicit(&g_classes[idx].live_blocks, memory_order_relaxed),
atomic_load_explicit(&g_classes[idx].peak_bytes, memory_order_relaxed));
if (add < 0 || (size_t)(written + add) >= size) {
return 0; /* truncated: a partial JSON array is worse than none */
}
written += add;
}
if ((size_t)written + CBM_ALLOC_ONE >= size) {
return 0;
}
out[written++] = ']';
out[written] = '\0';
return written;
}
void cbm_mem_class_log(const char *tag) {
cbm_mem_class_flush_thread();
char report[CBM_SZ_1K];
if (cbm_mem_class_report_json(report, sizeof(report)) <= 0) {
return;
}
cbm_log_info("mem.classes", "tag", tag ? tag : "-", "classes", report);
}
src/foundation/mem_core.h
+149
View File
@@ -0,0 +1,149 @@
/*
* mem_core.h — THE allocation route.
*
* Memory in this project is allocated through one core, not through ~800
* scattered malloc/calloc/strdup sites. This header is that core;
* foundation/mem.h remains policy and measurement (budget, RSS, pressure,
* phase marks) and deliberately owns no allocation.
*
* WHY THIS EXISTS (audited 2026-09-13)
*
* The budget could only ever be OBSERVED, never enforced. cbm_mem_over_budget()
* reads process RSS *after* the allocation that crossed the line already
* succeeded, and the only available response was to nap. A thermostat wired to
* a thermometer with no cooler attached.
*
* Worse, we could not even say WHERE the memory was. Two independent reasons:
*
* 1. cbm_mem_map_collect()'s live_bytes walks mi_theap_get_default() — THIS
* thread's mimalloc heap only. Walking the process-wide mi_heap_main() is
* a data race that TSan caught on macOS, so it is deliberately not done.
* An 18-worker index therefore attributes almost nothing; the rest lands
* in `residual`.
* 2. The mimalloc global override is ON for Linux/MinGW and permanently OFF
* for macOS (the two-level namespace turns this binary's free into mi_free
* while system libraries keep allocating from the system zone, and a
* pointer crossing that boundary aborts). On macOS ordinary malloc is
* served by the SYSTEM allocator: the startup audit reports
* owned_classes=0/6. Any accounting that assumes mimalloc owns the pointer
* is blind on an entire platform.
*
* So the core keeps its OWN counters. Atomic, per class, incremented at
* allocation and decremented at free. Thread-safe by construction, identical on
* every platform, and independent of which allocator actually serves malloc.
*
* NO PER-ALLOCATION HEADER. The obvious design — a {class,size} prefix — costs
* 16 bytes on every block, and the graph buffer makes ~100M of them at kernel
* scale: ~1.6 GB of pure overhead to measure a memory problem. Sizes come from
* the platform's usable-size query instead, which is exact-to-the-bucket, free,
* and correct under either allocator.
*/
#ifndef CBM_MEM_CORE_H
#define CBM_MEM_CORE_H
#include <stdbool.h>
#include <stddef.h>
/* Allocation classes.
*
* A class is a BUDGETING bucket, not a type taxonomy: split only where the
* split would change a decision. These follow the measured phase profile of a
* kernel index (2026-09-13), where the peak was 35.20 GB and the two consumers
* behaved differently — extraction transients scale with worker count, the
* semantic plateau does not. Attribution that cannot separate those two cannot
* choose between "park workers" and "stream the vectors". */
typedef enum {
CBM_MEM_CLASS_OTHER = 0, /* unclassified; the residual to drive down */
CBM_MEM_CLASS_GBUF_NODE, /* node records (~64 B each) */
CBM_MEM_CLASS_GBUF_EDGE, /* edge records (~48 B each) */
CBM_MEM_CLASS_GBUF_STRING, /* name / qualified_name / properties_json */
CBM_MEM_CLASS_GBUF_INDEX, /* the 8 lookup indexes (383 MB peak on the Go corpus) */
CBM_MEM_CLASS_EXTRACT, /* per-file working set: source text, extraction scratch */
CBM_MEM_CLASS_ARENA, /* CBMArena blocks -- every arena, whoever owns it */
CBM_MEM_CLASS_TS_TREE, /* tree-sitter: parse trees + parser state (bound allocator) */
CBM_MEM_CLASS_SEMANTIC, /* semantic pass: vectors, token pools, LSH (87 MB on Go) */
CBM_MEM_CLASS_DUMP, /* dump-time transients */
CBM_MEM_CLASS_STORE, /* SQLite (bound mem methods) + store batches and row buffers */
CBM_MEM_CLASS_HASH_TABLE, /* CBMHashTable buckets/entries not claimed by an owner class */
CBM_MEM_CLASS_DYN_ARRAY, /* CBM_DYN_ARRAY item storage (every cbm_da_* user) */
CBM_MEM_CLASS_COUNT
} cbm_mem_class_t;
/* Stable lowercase name, for logs and JSON. Never NULL, even out of range. */
const char *cbm_mem_class_name(cbm_mem_class_t cls);
/* ── The allocation route ──────────────────────────────────────────────
*
* Semantics match the C library exactly, so adoption is a mechanical rename and
* never a behaviour change:
* - cbm_alloc(cls, 0) returns a non-NULL pointer that is valid to free
* - cbm_free(cls, NULL) is a no-op
* - cbm_realloc(cls, NULL, n) behaves as cbm_alloc
* - a failed realloc leaves the original block intact and returns NULL
*
* The class passed to free/realloc MUST be the class the block was allocated
* with, or the counters drift. Pass the class through alongside the pointer,
* the same way a custom deleter would. */
void *cbm_alloc(cbm_mem_class_t cls, size_t bytes);
void *cbm_calloc(cbm_mem_class_t cls, size_t bytes);
void *cbm_realloc(cbm_mem_class_t cls, void *block, size_t bytes);
char *cbm_mem_strdup(cbm_mem_class_t cls, const char *s);
void cbm_free(cbm_mem_class_t cls, void *block);
/* ── Accounting ────────────────────────────────────────────────────────
*
* live_bytes is what the ALLOCATOR handed us (usable size), so it exceeds the
* bytes requested by the per-block rounding and is the honest number for a
* memory budget: rounding is memory the process cannot use for anything else.
*
* These counters see only memory that went through this core. Everything still
* on raw malloc is invisible here — which is the point of
* cbm_mem_tracked_live_bytes() versus the process RSS in mem.h: the gap between
* them IS the unmigrated surface, and it should shrink as adoption spreads. An
* unmeasured allocation must never read as an absent one. */
/* Push this thread's pending accounting deltas to the shared counters. Every
* reader does it for its own thread; a parallel-for worker does it when its
* work item ends, so the phase marks (read after the join) are exact. */
void cbm_mem_class_flush_thread(void);
size_t cbm_mem_class_live_bytes(cbm_mem_class_t cls);
size_t cbm_mem_class_live_blocks(cbm_mem_class_t cls);
size_t cbm_mem_tracked_live_bytes(void);
/* Peak live bytes for a class since process start (or the last reset). The
* budget question is always about the PEAK, never the value at the moment
* someone happened to look. */
size_t cbm_mem_class_peak_bytes(cbm_mem_class_t cls);
/* Drop all peaks to the current live values. For a measurement run that wants
* one phase, not the whole process history. Never resets live counters --
* those track real outstanding blocks. */
void cbm_mem_class_reset_peaks(void);
/* JSON array of {class, live_bytes, live_blocks, peak_bytes}, biggest live
* first, classes with no activity omitted. Returns bytes written, 0 if none. */
int cbm_mem_class_report_json(char *out, size_t size);
/* Log the class table at info level under `tag`. For phase boundaries in the
* index pipeline, where the interesting question is which class grew. */
void cbm_mem_class_log(const char *tag);
/* Usable size of a block obtained from this core, or 0 when the platform
* cannot answer. Exposed because the same query is what makes header-free
* accounting possible, and callers doing their own bulk accounting (arenas)
* need the identical definition to stay consistent with these counters. */
size_t cbm_mem_usable_size(const void *block);
/* ── Bulk accounting, for allocators that are not this one ─────────────
*
* The extraction engine already allocates through arenas (1301 call sites) and
* must NOT be rewritten to per-object cbm_alloc — that would undo the very
* batching that keeps its allocation count low. Instead an arena reports its
* block acquisitions here, so arena-backed memory appears in the same table as
* heap memory and the totals stay comparable.
*
* Symmetric: every add must be matched by a remove of the same size. */
void cbm_mem_class_add_external(cbm_mem_class_t cls, size_t bytes);
void cbm_mem_class_remove_external(cbm_mem_class_t cls, size_t bytes);
#endif /* CBM_MEM_CORE_H */
src/foundation/platform.h
+4
View File
@@ -116,6 +116,10 @@ typedef struct {
/* Query system information. Results are cached after first call. */
cbm_system_info_t cbm_system_info(void);
/* Physical memory the system could hand out right now, or 0 when the platform
* cannot answer. NOT cached - it changes during a run, which is the point. */
size_t cbm_system_available_ram(void);
/* Recommended worker count for parallel indexing.
* initial=true: all cores (user is waiting for initial index)
* initial=false: max(1, perf_cores-1) (leave headroom for user apps) */
src/foundation/slab_alloc.c
+14 -9
View File
@@ -39,6 +39,7 @@
#include "foundation/slab_alloc.h"
#include "foundation/compat.h"
#include "foundation/compat_thread.h"
#include "foundation/mem_core.h"
#include <stdbool.h>
#include <stddef.h>
@@ -177,7 +178,7 @@ static bool slab_map_set(uintptr_t base, slab_page_t *val) {
slab_map_unlock();
return true; /* nothing to unregister */
}
l2 = (slab_map_l2_t *)calloc(1, sizeof(*l2));
l2 = (slab_map_l2_t *)cbm_calloc(CBM_MEM_CLASS_TS_TREE, sizeof(*l2));
if (!l2) {
slab_map_unlock();
return false;
@@ -190,7 +191,7 @@ static bool slab_map_set(uintptr_t base, slab_page_t *val) {
slab_map_unlock();
return true;
}
l3 = (slab_map_l3_t *)calloc(1, sizeof(*l3));
l3 = (slab_map_l3_t *)cbm_calloc(CBM_MEM_CLASS_TS_TREE, sizeof(*l3));
if (!l3) {
slab_map_unlock();
return false;
@@ -217,6 +218,7 @@ static bool slab_grow(slab_state_t *s) {
if (cbm_aligned_alloc(&mem, SLAB_PAGE_SIZE, SLAB_PAGE_SIZE) != 0 || !mem) {
return false;
}
cbm_mem_class_add_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE);
slab_page_t *page = (slab_page_t *)mem;
page->next = s->pages;
atomic_init(&page->owner, s);
@@ -224,6 +226,7 @@ static bool slab_grow(slab_state_t *s) {
atomic_init(&page->refcount, 1u); /* owner guard */
if (!slab_map_register_page(page)) {
cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE);
cbm_aligned_free(page);
return false;
}
@@ -269,6 +272,7 @@ static void slab_reclaim_pages(slab_state_t *s, bool clear_installed) {
unsigned prev = atomic_fetch_sub_explicit(&p->refcount, 1u, memory_order_acq_rel);
if (prev == 1u) {
slab_map_unregister_page(p);
cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE);
cbm_aligned_free(p);
}
p = next;
@@ -292,7 +296,7 @@ static void *slab_malloc(size_t size) {
if (!s->freelist) {
slab_refill(s);
if (!s->freelist) {
return malloc(size); /* grow failed → heap fallback */
return cbm_alloc(CBM_MEM_CLASS_TS_TREE, size); /* grow failed → heap fallback */
}
}
slab_free_node_t *node = s->freelist;
@@ -302,8 +306,8 @@ static void *slab_malloc(size_t size) {
return node;
}
/* >64B: straight to malloc (= mimalloc in production) */
return malloc(size);
/* >64B: straight to the core (= mimalloc in production) */
return cbm_alloc(CBM_MEM_CLASS_TS_TREE, size);
}
static void *slab_calloc(size_t count, size_t size) {
@@ -338,7 +342,7 @@ static void *slab_realloc(void *ptr, size_t new_size) {
return ptr;
}
/* Promote slab → heap */
void *new_ptr = malloc(new_size);
void *new_ptr = cbm_alloc(CBM_MEM_CLASS_TS_TREE, new_size);
if (!new_ptr) {
return NULL;
}
@@ -347,8 +351,8 @@ static void *slab_realloc(void *ptr, size_t new_size) {
return new_ptr;
}
/* Case 2: heap pointer (from malloc) */
return realloc(ptr, new_size);
/* Case 2: heap pointer (from the core) */
return cbm_realloc(CBM_MEM_CLASS_TS_TREE, ptr, new_size);
}
static void slab_free(void *ptr) {
@@ -359,7 +363,7 @@ static void slab_free(void *ptr) {
slab_page_t *page = slab_map_lookup(base);
if (!page) {
/* Not a slab chunk → plain heap pointer (>64B or grow-fallback). */
free(ptr);
cbm_free(CBM_MEM_CLASS_TS_TREE, ptr);
return;
}
@@ -386,6 +390,7 @@ static void slab_free(void *ptr) {
if (prev == 1u) {
/* We returned the final chunk of a retired page → release it. */
slab_map_unregister_page(page);
cbm_mem_class_remove_external(CBM_MEM_CLASS_TS_TREE, SLAB_PAGE_SIZE);
cbm_aligned_free(page);
}
}
src/foundation/system_info.c
+62
View File
@@ -26,6 +26,7 @@ enum { DEFAULT_CORES = 1, MIN_WORKERS = 1, CBM_WORKERS_MAX = 256 };
#endif
#include <windows.h>
#elif defined(__APPLE__)
#include <mach/mach.h> /* host_statistics64 - reclaimable page accounting */
#include <sys/sysctl.h>
#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__OpenBSD__)
#include <unistd.h>
@@ -304,3 +305,64 @@ int cbm_default_worker_count(bool initial) {
int workers = info.perf_cores - SKIP_ONE;
return workers > 0 ? workers : MIN_WORKERS;
}
/* -- Available RAM --------------------------------------------------
*
* Bytes the system could hand out right now, or 0 when the platform cannot
* answer. Deliberately NOT cached: core counts and total RAM are immutable
* hardware facts, but this changes continuously and the entire point is to
* observe it DURING an index.
*
* Why this exists: the indexer decided "out of memory" by comparing its own
* RSS against a static fraction of TOTAL ram. That refuses work a machine can
* plainly do -- measured 2026-09-13, the linux kernel needs 31.75 GB on a
* 48 GB host (66 percent) against a 0.5 default budget, and the same index
* completed on the previous release by overshooting to 33.56 GB. Whether
* memory is actually scarce is a property of the SYSTEM, not of a constant. */
size_t cbm_system_available_ram(void) {
#ifdef _WIN32
MEMORYSTATUSEX status;
status.dwLength = sizeof(status);
if (GlobalMemoryStatusEx(&status)) {
return (size_t)status.ullAvailPhys;
}
return 0;
#elif defined(__APPLE__)
/* free + inactive + purgeable. Counting only free_count would report
* pressure on any machine that is merely warm, because inactive and
* purgeable pages are reclaimed on demand. */
mach_port_t host = mach_host_self();
vm_size_t page_size = 0;
if (host_page_size(host, &page_size) != KERN_SUCCESS || page_size == 0) {
return 0;
}
vm_statistics64_data_t vm_stat = {0};
mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
if (host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm_stat, &count) != KERN_SUCCESS) {
return 0;
}
uint64_t pages = (uint64_t)vm_stat.free_count + (uint64_t)vm_stat.inactive_count +
(uint64_t)vm_stat.purgeable_count;
return (size_t)(pages * (uint64_t)page_size);
#elif !defined(__NetBSD__) && !defined(__FreeBSD__) && !defined(__OpenBSD__)
/* Linux: MemAvailable is the kernel estimate and accounts for reclaimable
* slab and page cache, which MemFree does not. */
FILE *meminfo = fopen("/proc/meminfo", "re");
if (!meminfo) {
return 0;
}
char line[CBM_SZ_256];
size_t available = 0;
while (fgets(line, sizeof(line), meminfo) != NULL) {
unsigned long long kb = 0;
if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) {
available = (size_t)(kb * (unsigned long long)CBM_SZ_1K);
break;
}
}
(void)fclose(meminfo);
return available;
#else
return 0; /* BSD: unknown rather than guessed */
#endif
}
src/graph_buffer/graph_buffer.c
+355 -139
View File
@@ -36,6 +36,7 @@ enum {
#include "foundation/dyn_array.h"
#include "foundation/profile.h"
#include "foundation/mem.h"
#include "foundation/mem_core.h"
#include <sqlite3.h>
#include <stdatomic.h>
@@ -60,6 +61,18 @@ static inline void *intptr_to_ptr(intptr_t v) {
#define EDGE_KEY_BUF CBM_SZ_256
/* Per-type or per-key edge list stored in hash tables as values */
typedef struct {
uint64_t h0;
uint64_t h1;
cbm_gbuf_edge_t *edge; /* NULL = empty slot */
} edge_key_slot_t;
typedef struct {
edge_key_slot_t *slots;
size_t cap; /* power of two, 0 = not allocated */
size_t count;
} edge_key_map_t;
typedef CBM_DYN_ARRAY(const cbm_gbuf_edge_t *) edge_ptr_array_t;
/* Per-label or per-name node list */
@@ -85,6 +98,10 @@ struct cbm_gbuf {
* strings at kernel scale, plus a snprintf+strdup+hash on every one of
* the ~18 hot find_by_id call sites. */
cbm_gbuf_node_t **by_id;
/* Worker buffers (cbm_gbuf_new_worker) keep no by_id array: their ids
* come from the shared counter, so a dense array would span the whole
* global id space in every worker and double in lockstep. */
bool by_id_off;
int64_t by_id_cap;
/* Secondary node indexes */
@@ -95,7 +112,7 @@ struct cbm_gbuf {
CBM_DYN_ARRAY(cbm_gbuf_edge_t *) edges;
/* Edge dedup index: "srcID:tgtID:type" → cbm_gbuf_edge_t* */
CBMHashTable *edge_by_key;
edge_key_map_t edge_by_key; /* dedup: 128-bit key hash -> edge */
/* Edge secondary indexes: composite keys → edge_ptr_array_t */
CBMHashTable *edges_by_source_type; /* "srcID:type" → edge_ptr_array_t* */
@@ -124,7 +141,7 @@ struct cbm_gbuf {
/* ── Helpers ─────────────────────────────────────────────────────── */
static char *heap_strdup(const char *s) {
return s ? strdup(s) : strdup("{}");
return cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, s ? s : "{}");
}
/* Intern a repetitive string into the buffer's pool: identical content collapses
@@ -137,7 +154,7 @@ static const char *gb_intern(cbm_gbuf_t *gb, const char *s) {
if (found) {
return found;
}
char *copy = strdup(key);
char *copy = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, key);
if (copy) {
cbm_ht_set(gb->intern_pool, copy, copy); /* key == value == owned copy */
}
@@ -201,22 +218,142 @@ static void make_src_type_key(char *buf, size_t bufsz, int64_t src, const char *
snprintf(buf, bufsz, "%lld:%s", (long long)src, type);
}
/* ── Edge dedup index: 128-bit key hash -> edge, open addressing ──────
*
* The dedup key is "src:tgt:type:props" (make_edge_key). Keeping it as a
* string per edge cost the kernel 15.8M heap strings (~1.2 GB live, and the
* same again transiently in worker buffers during merge). Two independent
* 64-bit FNV-1a hashes of the composed key take 16 bytes; a false match needs
* both to collide, ~n^2 / 2^129 for n edges -- below any other failure mode of
* the index. Linear probing, backward-shift deletion (no tombstones), load
* kept under 1/2. Values are the edge pointers (stable heap records). */
static void edge_key_hash(const char *key, uint64_t *h0, uint64_t *h1) {
uint64_t a = 1469598103934665603ULL;
uint64_t b = 0x9E3779B97F4A7C15ULL;
for (const unsigned char *p = (const unsigned char *)key; *p; p++) {
a ^= *p;
a *= 1099511628211ULL;
b = (b ^ *p) * 0xFF51AFD7ED558CCDULL;
b ^= b >> 29;
}
*h0 = a;
*h1 = b ^ (a >> 17);
}
static void edge_key_map_free(edge_key_map_t *m) {
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots);
m->slots = NULL;
m->cap = 0;
m->count = 0;
}
static edge_key_slot_t *edge_key_map_find(const edge_key_map_t *m, uint64_t h0, uint64_t h1) {
if (!m->slots) {
return NULL;
}
size_t mask = m->cap - SKIP_ONE;
for (size_t i = (size_t)h0 & mask;; i = (i + SKIP_ONE) & mask) {
edge_key_slot_t *s = &m->slots[i];
if (!s->edge) {
return NULL;
}
if (s->h0 == h0 && s->h1 == h1) {
return s;
}
}
}
static bool edge_key_map_grow(edge_key_map_t *m) {
size_t new_cap = m->cap ? m->cap * PAIR_LEN : CBM_SZ_1K;
edge_key_slot_t *slots = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, new_cap * sizeof(*slots));
if (!slots) {
return false;
}
size_t mask = new_cap - SKIP_ONE;
for (size_t i = 0; i < m->cap; i++) {
edge_key_slot_t *s = &m->slots[i];
if (!s->edge) {
continue;
}
size_t j = (size_t)s->h0 & mask;
while (slots[j].edge) {
j = (j + SKIP_ONE) & mask;
}
slots[j] = *s;
}
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, m->slots);
m->slots = slots;
m->cap = new_cap;
return true;
}
static bool edge_key_map_set(edge_key_map_t *m, uint64_t h0, uint64_t h1, cbm_gbuf_edge_t *edge) {
if ((m->count + SKIP_ONE) * PAIR_LEN > m->cap && !edge_key_map_grow(m)) {
return false;
}
size_t mask = m->cap - SKIP_ONE;
size_t i = (size_t)h0 & mask;
while (m->slots[i].edge) {
if (m->slots[i].h0 == h0 && m->slots[i].h1 == h1) {
m->slots[i].edge = edge;
return true;
}
i = (i + SKIP_ONE) & mask;
}
m->slots[i].h0 = h0;
m->slots[i].h1 = h1;
m->slots[i].edge = edge;
m->count++;
return true;
}
static void edge_key_map_delete(edge_key_map_t *m, uint64_t h0, uint64_t h1) {
edge_key_slot_t *s = edge_key_map_find(m, h0, h1);
if (!s) {
return;
}
size_t mask = m->cap - SKIP_ONE;
size_t i = (size_t)(s - m->slots);
m->slots[i].edge = NULL;
m->count--;
/* Backward shift: pull later entries of the same probe run into the hole. */
for (size_t j = (i + SKIP_ONE) & mask; m->slots[j].edge; j = (j + SKIP_ONE) & mask) {
size_t home = (size_t)m->slots[j].h0 & mask;
/* entry at j may move to i if its home is not in (i, j] cyclically */
bool movable = (i <= j) ? (home <= i || home > j) : (home <= i && home > j);
if (movable) {
m->slots[i] = m->slots[j];
m->slots[j].edge = NULL;
i = j;
}
}
}
/* Get or create a node_ptr_array_t in a hash table */
static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node);
static node_ptr_array_t *get_or_create_node_array(CBMHashTable *ht, const char *key) {
if (!ht) {
return NULL; /* worker buffer: no secondary indexes (cbm_gbuf_new_worker) */
}
node_ptr_array_t *arr = cbm_ht_get(ht, key);
if (!arr) {
arr = calloc(CBM_ALLOC_ONE, sizeof(node_ptr_array_t));
cbm_ht_set(ht, strdup(key), arr);
arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(node_ptr_array_t));
cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr);
}
return arr;
}
/* Get or create an edge_ptr_array_t in a hash table */
static edge_ptr_array_t *get_or_create_edge_array(CBMHashTable *ht, const char *key) {
if (!ht) {
return NULL;
}
edge_ptr_array_t *arr = cbm_ht_get(ht, key);
if (!arr) {
arr = calloc(CBM_ALLOC_ONE, sizeof(edge_ptr_array_t));
cbm_ht_set(ht, strdup(key), arr);
arr = cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(edge_ptr_array_t));
cbm_ht_set(ht, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), arr);
}
return arr;
}
@@ -227,9 +364,9 @@ static void free_node_array(const char *key, void *value, void *ud) {
node_ptr_array_t *arr = value;
if (arr) {
cbm_da_free(arr);
free(arr);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr);
}
free((void *)key);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
}
/* Free an edge_ptr_array_t (callback) */
@@ -238,30 +375,40 @@ static void free_edge_array(const char *key, void *value, void *ud) {
edge_ptr_array_t *arr = value;
if (arr) {
cbm_da_free(arr);
free(arr);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, arr);
}
free((void *)key);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
}
/* Free keys only (for edge_by_key, deleted_set) */
static void free_key_only(const char *key, void *value, void *ud) {
/* Free index keys only (deleted_set id keys). */
static void free_index_key(const char *key, void *value, void *ud) {
(void)value;
(void)ud;
free((void *)key);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
}
/* Free one intern-pool entry: key == value == the single owned string copy
* that nodes/edges borrowed for label/file_path/type. */
static void free_intern_entry(const char *key, void *value, void *ud) {
(void)value;
(void)ud;
cbm_free(CBM_MEM_CLASS_GBUF_STRING, (void *)key);
}
/* Free a single node's owned strings. label and file_path are interned
* (pool-owned) — NOT freed here; the pool frees them once in cbm_gbuf_free. */
static void free_node_strings(cbm_gbuf_node_t *n) {
free(n->name);
free(n->qualified_name);
free(n->properties_json);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->name);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->properties_json);
}
/* Free a single edge's owned strings. type is interned (pool-owned) — NOT
* freed here; the pool frees it once in cbm_gbuf_free. */
/* An edge owns no strings: type AND properties_json are interned (pool-owned;
* the pool frees each once in cbm_gbuf_free). Edge properties repeat massively
* -- on the Go corpus 1.8M edges carry 343k distinct property strings and 854k
* are exactly "{}" -- and an edge's properties are never mutated in place. */
static void free_edge_strings(cbm_gbuf_edge_t *e) {
free(e->properties_json);
(void)e;
}
/* Allocate the next buffer-local or shared-atomic ID. */
@@ -303,9 +450,10 @@ static void unindex_edge(cbm_gbuf_t *gb, const cbm_gbuf_edge_t *e) {
char key[EDGE_KEY_BUF];
make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type, e->properties_json);
const char *ekey = cbm_ht_get_key(gb->edge_by_key, key);
cbm_ht_delete(gb->edge_by_key, key);
free((void *)ekey);
uint64_t h0;
uint64_t h1;
edge_key_hash(key, &h0, &h1);
edge_key_map_delete(&gb->edge_by_key, h0, h1);
make_src_type_key(key, sizeof(key), e->source_id, e->type);
remove_edge_from_ptr_array(cbm_ht_get(gb->edges_by_source_type, key), e->id);
@@ -329,7 +477,7 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) {
if (cbm_ht_get(deleted_set, src_id) || cbm_ht_get(deleted_set, tgt_id)) {
unindex_edge(gb, e);
free_edge_strings(e);
free(e);
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
} else {
gb->edges.items[write_idx++] = gb->edges.items[i];
}
@@ -341,34 +489,50 @@ static void cascade_delete_edges(cbm_gbuf_t *gb, CBMHashTable *deleted_set) {
static void register_node_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_node_t *node) {
cbm_ht_set(gb->node_by_qn, node->qualified_name, node);
if (node->id >= gb->by_id_cap) {
int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;
while (nc <= node->id) {
nc *= 2;
if (!gb->by_id_off) {
if (node->id >= gb->by_id_cap) {
int64_t nc = gb->by_id_cap > 0 ? gb->by_id_cap : CBM_SZ_1K;
while (nc <= node->id) {
nc *= 2;
}
cbm_gbuf_node_t **grown =
cbm_realloc(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id, (size_t)nc * sizeof(*grown));
if (grown) {
memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));
gb->by_id = grown;
gb->by_id_cap = nc;
}
}
cbm_gbuf_node_t **grown = realloc(gb->by_id, (size_t)nc * sizeof(*grown));
if (grown) {
memset(grown + gb->by_id_cap, 0, (size_t)(nc - gb->by_id_cap) * sizeof(*grown));
gb->by_id = grown;
gb->by_id_cap = nc;
if (node->id >= 0 && node->id < gb->by_id_cap) {
gb->by_id[node->id] = node;
}
}
if (node->id >= 0 && node->id < gb->by_id_cap) {
gb->by_id[node->id] = node;
}
node_ptr_array_t *by_label =
get_or_create_node_array(gb->nodes_by_label, node->label ? node->label : "");
cbm_da_push(by_label, (const cbm_gbuf_node_t *)node);
node_array_push(by_label, node);
node_ptr_array_t *by_name =
get_or_create_node_array(gb->nodes_by_name, node->name ? node->name : "");
cbm_da_push(by_name, (const cbm_gbuf_node_t *)node);
node_array_push(by_name, node);
}
/* Push an edge pointer into a dynamic array (wraps macro to reduce CC contribution). */
/* Per-key index arrays start at two slots: on the kernel the secondary
* indexes hold ~18M of these arrays and most keys have one or two entries,
* so the default 8-slot start was 1 GB of empty headroom. NULL = the buffer
* has no secondary indexes (worker buffers). */
enum { GB_INDEX_ARRAY_FIRST_CAP = 2 };
static void edge_array_push(edge_ptr_array_t *arr, const cbm_gbuf_edge_t *edge) {
cbm_da_push(arr, edge);
if (!arr) {
return;
}
cbm_da_push_min(arr, edge, GB_INDEX_ARRAY_FIRST_CAP);
}
static void node_array_push(node_ptr_array_t *arr, const cbm_gbuf_node_t *node) {
if (!arr) {
return;
}
cbm_da_push_min(arr, node, GB_INDEX_ARRAY_FIRST_CAP);
}
/* Index an edge by one key into a hash table bucket. */
@@ -392,6 +556,9 @@ static void register_edge_in_indexes(cbm_gbuf_t *gb, cbm_gbuf_edge_t *edge) {
/* Rebuild edge secondary indexes from scratch (after bulk deletion). */
static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) {
if (!gb->edges_by_type) {
return; /* worker buffer: nothing to rebuild */
}
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
cbm_ht_free(gb->edges_by_source_type);
cbm_ht_foreach(gb->edges_by_target_type, free_edge_array, NULL);
@@ -399,9 +566,9 @@ static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) {
cbm_ht_foreach(gb->edges_by_type, free_edge_array, NULL);
cbm_ht_free(gb->edges_by_type);
gb->edges_by_source_type = cbm_ht_create(CBM_SZ_256);
gb->edges_by_target_type = cbm_ht_create(CBM_SZ_256);
gb->edges_by_type = cbm_ht_create(CBM_SZ_32);
gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
for (int i = 0; i < gb->edges.count; i++) {
register_edge_in_indexes(gb, gb->edges.items[i]);
@@ -412,7 +579,7 @@ static void rebuild_edge_secondary_indexes(cbm_gbuf_t *gb) {
static void release_gbuf_indexes(cbm_gbuf_t *gb) {
cbm_ht_free(gb->node_by_qn);
gb->node_by_qn = NULL;
free(gb->by_id);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
gb->by_id = NULL;
gb->by_id_cap = 0;
cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL);
@@ -421,9 +588,7 @@ static void release_gbuf_indexes(cbm_gbuf_t *gb) {
cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL);
cbm_ht_free(gb->nodes_by_name);
gb->nodes_by_name = NULL;
cbm_ht_foreach(gb->edge_by_key, free_key_only, NULL);
cbm_ht_free(gb->edge_by_key);
gb->edge_by_key = NULL;
edge_key_map_free(&gb->edge_by_key);
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
cbm_ht_free(gb->edges_by_source_type);
gb->edges_by_source_type = NULL;
@@ -438,28 +603,27 @@ static void release_gbuf_indexes(cbm_gbuf_t *gb) {
/* ── Lifecycle ──────────────────────────────────────────────────── */
cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path) {
cbm_gbuf_t *gb = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_t));
cbm_gbuf_t *gb = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(cbm_gbuf_t));
if (!gb) {
return NULL;
}
gb->project = strdup(project ? project : "");
gb->root_path = strdup(root_path ? root_path : "");
gb->project = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, project ? project : "");
gb->root_path = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, root_path ? root_path : "");
gb->next_id = SKIP_ONE;
gb->shared_ids = NULL;
gb->node_by_qn = cbm_ht_create(CBM_SZ_256);
gb->node_by_qn = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->by_id = NULL;
gb->by_id_cap = 0;
gb->nodes_by_label = cbm_ht_create(CBM_SZ_32);
gb->nodes_by_name = cbm_ht_create(CBM_SZ_256);
gb->nodes_by_label = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
gb->nodes_by_name = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->edge_by_key = cbm_ht_create(CBM_SZ_512);
gb->edges_by_source_type = cbm_ht_create(CBM_SZ_256);
gb->edges_by_target_type = cbm_ht_create(CBM_SZ_256);
gb->edges_by_type = cbm_ht_create(CBM_SZ_32);
gb->edges_by_source_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->edges_by_target_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_256);
gb->edges_by_type = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_INDEX, CBM_SZ_32);
gb->intern_pool = cbm_ht_create(CBM_SZ_1K);
gb->intern_pool = cbm_ht_create_in(CBM_MEM_CLASS_GBUF_STRING, CBM_SZ_1K);
return gb;
}
@@ -473,6 +637,40 @@ cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path,
return gb;
}
cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,
_Atomic int64_t *id_source) {
cbm_gbuf_t *gb = cbm_gbuf_new_shared_ids(project, root_path, id_source);
if (!gb) {
return NULL;
}
/* A worker buffer is appended to and merged; it is never asked by label,
* name or edge type. Its secondary indexes were pure cost: on the kernel
* the index class peaked at 7.1 GB during resolve against 2.5 GB live in
* the main buffer, the difference being 18 workers' throwaway indexes.
* node_by_qn and edge_by_key stay: upsert and dedup need them. */
cbm_ht_free(gb->nodes_by_label);
cbm_ht_free(gb->nodes_by_name);
cbm_ht_free(gb->edges_by_source_type);
cbm_ht_free(gb->edges_by_target_type);
cbm_ht_free(gb->edges_by_type);
gb->nodes_by_label = NULL;
gb->nodes_by_name = NULL;
gb->edges_by_source_type = NULL;
gb->edges_by_target_type = NULL;
gb->edges_by_type = NULL;
/* Nor by id: with ids from the shared counter, a dense id -> node array
* in every worker spans the whole global id space (8.5M ids x 8 B x 18
* workers on the kernel) and all of them double in the same instant --
* a 1 GB step between two reads of the memory gate, which is where the
* 15 GB budget was missed (2026-09-14). Nothing asks a worker buffer by
* id before the merge; the main buffer answers after it. */
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
gb->by_id = NULL;
gb->by_id_cap = 0;
gb->by_id_off = true;
return gb;
}
void cbm_gbuf_free(cbm_gbuf_t *gb) {
if (!gb) {
return;
@@ -482,7 +680,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
for (int i = 0; i < gb->nodes.count; i++) {
cbm_gbuf_node_t *n = gb->nodes.items[i];
free_node_strings(n);
free(n);
cbm_free(CBM_MEM_CLASS_GBUF_NODE, n);
}
cbm_da_free(&gb->nodes);
@@ -490,7 +688,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
for (int i = 0; i < gb->edges.count; i++) {
cbm_gbuf_edge_t *e = gb->edges.items[i];
free_edge_strings(e);
free(e);
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
}
cbm_da_free(&gb->edges);
@@ -498,7 +696,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
if (gb->node_by_qn) {
cbm_ht_free(gb->node_by_qn);
}
free(gb->by_id);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, gb->by_id);
if (gb->nodes_by_label) {
cbm_ht_foreach(gb->nodes_by_label, free_node_array, NULL);
cbm_ht_free(gb->nodes_by_label);
@@ -507,10 +705,7 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
cbm_ht_foreach(gb->nodes_by_name, free_node_array, NULL);
cbm_ht_free(gb->nodes_by_name);
}
if (gb->edge_by_key) {
cbm_ht_foreach(gb->edge_by_key, free_key_only, NULL);
cbm_ht_free(gb->edge_by_key);
}
edge_key_map_free(&gb->edge_by_key);
if (gb->edges_by_source_type) {
cbm_ht_foreach(gb->edges_by_source_type, free_edge_array, NULL);
cbm_ht_free(gb->edges_by_source_type);
@@ -526,28 +721,28 @@ void cbm_gbuf_free(cbm_gbuf_t *gb) {
/* Free vector storage */
for (int i = 0; i < gb->dump_vector_count; i++) {
free((void *)gb->dump_vectors[i].vector);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_vectors[i].vector);
}
free(gb->dump_vectors);
cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors);
/* Free token vector storage */
for (int i = 0; i < gb->dump_token_vec_count; i++) {
free((void *)gb->dump_token_vecs[i].token);
free((void *)gb->dump_token_vecs[i].vector);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].token);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)gb->dump_token_vecs[i].vector);
}
free(gb->dump_token_vecs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs);
/* Free interned strings (node label/file_path, edge type) — pool owns one
* copy each (key == value), freed exactly once via free_key_only. Done after
* nodes/edges since they borrowed these pointers. */
* copy each (key == value), freed exactly once via free_intern_entry. Done
* after nodes/edges since they borrowed these pointers. */
if (gb->intern_pool) {
cbm_ht_foreach(gb->intern_pool, free_key_only, NULL);
cbm_ht_foreach(gb->intern_pool, free_intern_entry, NULL);
cbm_ht_free(gb->intern_pool);
}
free(gb->project);
free(gb->root_path);
free(gb);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->project);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, gb->root_path);
cbm_free(CBM_MEM_CLASS_OTHER, gb);
}
/* ── Vector storage ──────────────────────────────────────────────── */
@@ -560,7 +755,8 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector
if (gb->dump_vector_count >= gb->dump_vector_cap) {
int new_cap =
gb->dump_vector_cap < VEC_INIT_CAP ? VEC_INIT_CAP : gb->dump_vector_cap * VEC_GROW;
CBMDumpVector *grown = realloc(gb->dump_vectors, (size_t)new_cap * sizeof(CBMDumpVector));
CBMDumpVector *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_vectors,
(size_t)new_cap * sizeof(CBMDumpVector));
if (!grown) {
return GB_ERR;
}
@@ -568,7 +764,7 @@ int cbm_gbuf_store_vector(cbm_gbuf_t *gb, int64_t node_id, const uint8_t *vector
gb->dump_vector_cap = new_cap;
}
/* Copy vector data */
uint8_t *vec_copy = malloc((size_t)vector_len);
uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len);
if (!vec_copy) {
return GB_ERR;
}
@@ -592,15 +788,15 @@ int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t
if (gb->dump_token_vec_count >= gb->dump_token_vec_cap) {
int new_cap =
gb->dump_token_vec_cap < TV_INIT_CAP ? TV_INIT_CAP : gb->dump_token_vec_cap * TV_GROW;
CBMDumpTokenVec *grown =
realloc(gb->dump_token_vecs, (size_t)new_cap * sizeof(CBMDumpTokenVec));
CBMDumpTokenVec *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, gb->dump_token_vecs,
(size_t)new_cap * sizeof(CBMDumpTokenVec));
if (!grown) {
return GB_ERR;
}
gb->dump_token_vecs = grown;
gb->dump_token_vec_cap = new_cap;
}
uint8_t *vec_copy = malloc((size_t)vector_len);
uint8_t *vec_copy = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)vector_len);
if (!vec_copy) {
return GB_ERR;
}
@@ -610,7 +806,7 @@ int cbm_gbuf_store_token_vector(cbm_gbuf_t *gb, const char *token, const uint8_t
gb->dump_token_vecs[idx] = (CBMDumpTokenVec){
.id = idx + SKIP_ONE, /* 1-based sequential ID */
.project = gb->project,
.token = strdup(token),
.token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token),
.vector = vec_copy,
.vector_len = vector_len,
.idf = idf,
@@ -721,30 +917,30 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
cbm_ht_get(gb->nodes_by_name, existing->name ? existing->name : ""), existing->id);
}
existing->label = (char *)new_label_interned;
free(existing->name);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name);
existing->name = new_name;
existing->file_path = (char *)gb_intern(gb, file_path);
existing->start_line = start_line;
existing->end_line = end_line;
if (new_props) {
free(existing->properties_json);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json);
existing->properties_json = new_props;
}
if (label_changed) {
node_ptr_array_t *by_label = get_or_create_node_array(
gb->nodes_by_label, existing->label ? existing->label : "");
cbm_da_push(by_label, (const cbm_gbuf_node_t *)existing);
node_array_push(by_label, existing);
}
if (name_changed) {
node_ptr_array_t *by_name =
get_or_create_node_array(gb->nodes_by_name, existing->name ? existing->name : "");
cbm_da_push(by_name, (const cbm_gbuf_node_t *)existing);
node_array_push(by_name, existing);
}
return existing->id;
}
/* Heap-allocate a new node (pointer stays stable across array growth) */
cbm_gbuf_node_t *node = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_node_t));
cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t));
if (!node) {
return 0;
}
@@ -766,6 +962,19 @@ int64_t cbm_gbuf_upsert_node(cbm_gbuf_t *gb, const char *label, const char *name
return id;
}
int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json) {
if (!node) {
return GB_ERR;
}
char *copy = heap_strdup(json);
if (!copy) {
return GB_ERR;
}
cbm_free(CBM_MEM_CLASS_GBUF_STRING, node->properties_json);
node->properties_json = copy;
return 0;
}
const cbm_gbuf_node_t *cbm_gbuf_find_by_qn(const cbm_gbuf_t *gb, const char *qn) {
if (!gb || !qn) {
return NULL;
@@ -834,7 +1043,8 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) {
char id_buf[CBM_SZ_32];
make_id_key(id_buf, sizeof(id_buf), n->id);
cbm_ht_set(deleted_set, strdup(id_buf), intptr_to_ptr(SKIP_ONE));
cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
intptr_to_ptr(SKIP_ONE));
/* Remove from primary indexes */
cbm_ht_delete(gb->node_by_qn, n->qualified_name);
@@ -849,7 +1059,7 @@ int cbm_gbuf_delete_by_label(cbm_gbuf_t *gb, const char *label) {
/* Cascade-delete edges referencing deleted nodes */
cascade_delete_edges(gb, deleted_set);
cbm_ht_foreach(deleted_set, free_key_only, NULL);
cbm_ht_foreach(deleted_set, free_index_key, NULL);
cbm_ht_free(deleted_set);
return 0;
}
@@ -876,7 +1086,8 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
char id_buf[CBM_SZ_32];
make_id_key(id_buf, sizeof(id_buf), n->id);
cbm_ht_set(deleted_set, strdup(id_buf), intptr_to_ptr(SKIP_ONE));
cbm_ht_set(deleted_set, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, id_buf),
intptr_to_ptr(SKIP_ONE));
/* Remove from secondary indexes */
remove_node_from_ptr_array(cbm_ht_get(gb->nodes_by_label, n->label), n->id);
@@ -890,7 +1101,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
/* NULL out QN so dump's liveness check (cbm_ht_get by QN) fails
* even if a new node with the same QN is inserted later via merge. */
free(n->qualified_name);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, n->qualified_name);
n->qualified_name = NULL;
deleted_count++;
}
@@ -903,7 +1114,7 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path) {
/* Cascade-delete edges referencing deleted nodes */
cascade_delete_edges(gb, deleted_set);
cbm_ht_foreach(deleted_set, free_key_only, NULL);
cbm_ht_foreach(deleted_set, free_index_key, NULL);
cbm_ht_free(deleted_set);
{
char s_buf[CBM_SZ_16];
@@ -945,7 +1156,8 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje
}
sqlite3_finalize(stmt);
int64_t *old_to_new = calloc((size_t)(max_old_id + SKIP_ONE), sizeof(int64_t));
int64_t *old_to_new =
cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, (size_t)(max_old_id + SKIP_ONE) * sizeof(int64_t));
if (!old_to_new) {
cbm_store_close(store);
return CBM_NOT_FOUND;
@@ -957,7 +1169,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje
"SELECT id, label, name, qualified_name, file_path, start_line, end_line, properties "
"FROM nodes WHERE project = ? ORDER BY id",
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
free(old_to_new);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
cbm_store_close(store);
return CBM_NOT_FOUND;
}
@@ -985,7 +1197,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje
"SELECT source_id, target_id, type, properties "
"FROM edges WHERE project = ?",
CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
free(old_to_new);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
cbm_store_close(store);
return CBM_NOT_FOUND;
}
@@ -1005,7 +1217,7 @@ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *proje
}
sqlite3_finalize(stmt);
free(old_to_new);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, old_to_new);
cbm_store_close(store);
return 0;
}
@@ -1101,17 +1313,20 @@ int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_i
char key[EDGE_KEY_BUF];
make_edge_key(key, sizeof(key), source_id, target_id, type, properties_json);
cbm_gbuf_edge_t *existing = cbm_ht_get(gb->edge_by_key, key);
uint64_t h0;
uint64_t h1;
edge_key_hash(key, &h0, &h1);
edge_key_slot_t *hit = edge_key_map_find(&gb->edge_by_key, h0, h1);
cbm_gbuf_edge_t *existing = hit ? hit->edge : NULL;
if (existing) {
if (edge_props_should_replace(existing->properties_json, properties_json)) {
free(existing->properties_json);
existing->properties_json = heap_strdup(properties_json);
existing->properties_json = (char *)gb_intern(gb, properties_json);
}
return existing->id;
}
/* Heap-allocate a new edge (pointer stays stable) */
cbm_gbuf_edge_t *edge = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_edge_t));
cbm_gbuf_edge_t *edge = cbm_calloc(CBM_MEM_CLASS_GBUF_EDGE, sizeof(cbm_gbuf_edge_t));
if (!edge) {
return 0;
}
@@ -1121,13 +1336,13 @@ int64_t cbm_gbuf_insert_edge(cbm_gbuf_t *gb, int64_t source_id, int64_t target_i
edge->source_id = source_id;
edge->target_id = target_id;
edge->type = (char *)gb_intern(gb, type);
edge->properties_json = heap_strdup(properties_json);
edge->properties_json = (char *)gb_intern(gb, properties_json);
/* Store pointer in array */
cbm_da_push(&gb->edges, edge);
/* Dedup index */
cbm_ht_set(gb->edge_by_key, strdup(key), edge);
(void)edge_key_map_set(&gb->edge_by_key, h0, h1, edge);
/* Secondary indexes */
register_edge_in_indexes(gb, edge);
@@ -1212,11 +1427,12 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) {
char key[EDGE_KEY_BUF];
make_edge_key(key, sizeof(key), e->source_id, e->target_id, e->type,
e->properties_json);
const char *ekey = cbm_ht_get_key(gb->edge_by_key, key);
cbm_ht_delete(gb->edge_by_key, key);
free((void *)ekey);
uint64_t h0;
uint64_t h1;
edge_key_hash(key, &h0, &h1);
edge_key_map_delete(&gb->edge_by_key, h0, h1);
free_edge_strings(e);
free(e);
cbm_free(CBM_MEM_CLASS_GBUF_EDGE, e);
} else {
gb->edges.items[write_idx++] = gb->edges.items[i];
}
@@ -1234,8 +1450,8 @@ int cbm_gbuf_delete_edges_by_type(cbm_gbuf_t *gb, const char *type) {
/* Free remap hash table entries (key = heap string, value = heap int64_t*) */
static void free_remap_entry(const char *key, void *val, void *ud) {
(void)ud;
free((void *)key);
free(val);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, (void *)key);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, val);
}
/* Handle QN collision: update dst node fields (src wins), record remap if IDs differ.
@@ -1296,24 +1512,24 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing,
existing->id);
}
existing->label = (char *)new_label;
free(existing->name);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->name);
existing->name = heap_strdup(sn->name);
existing->file_path = (char *)gb_intern(dst, sn->file_path);
existing->start_line = sn->start_line;
existing->end_line = sn->end_line;
if (sn->properties_json) {
free(existing->properties_json);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, existing->properties_json);
existing->properties_json = heap_strdup(sn->properties_json);
}
if (label_changed) {
node_ptr_array_t *by_label = get_or_create_node_array(
dst->nodes_by_label, existing->label ? existing->label : "");
cbm_da_push(by_label, (const cbm_gbuf_node_t *)existing);
node_array_push(by_label, existing);
}
if (name_changed) {
node_ptr_array_t *by_name = get_or_create_node_array(
dst->nodes_by_name, existing->name ? existing->name : "");
cbm_da_push(by_name, (const cbm_gbuf_node_t *)existing);
node_array_push(by_name, existing);
}
}
}
@@ -1324,15 +1540,15 @@ static void merge_update_existing(cbm_gbuf_t *dst, cbm_gbuf_node_t *existing,
}
char key[CBM_SZ_32];
make_id_key(key, sizeof(key), sn->id);
int64_t *val = malloc(sizeof(int64_t));
int64_t *val = cbm_alloc(CBM_MEM_CLASS_GBUF_INDEX, sizeof(int64_t));
*val = existing->id;
cbm_ht_set(*remap, strdup(key), val);
cbm_ht_set(*remap, cbm_mem_strdup(CBM_MEM_CLASS_GBUF_INDEX, key), val);
}
}
/* Copy a non-colliding src node into dst with its original ID. */
static void merge_copy_new_node(cbm_gbuf_t *dst, const cbm_gbuf_node_t *sn) {
cbm_gbuf_node_t *node = calloc(CBM_ALLOC_ONE, sizeof(cbm_gbuf_node_t));
cbm_gbuf_node_t *node = cbm_calloc(CBM_MEM_CLASS_GBUF_NODE, sizeof(cbm_gbuf_node_t));
if (!node) {
return;
}
@@ -1445,7 +1661,7 @@ static char *extract_prop_string(const char *props, const char *key_quoted, cons
yyjson_val *v = yyjson_obj_get(yyjson_doc_get_root(doc), key);
if (v && yyjson_is_str(v)) {
const char *sv = yyjson_get_str(v);
out = cbm_strndup(sv, strlen(sv));
out = cbm_mem_strdup(CBM_MEM_CLASS_DUMP, sv);
}
yyjson_doc_free(doc);
return out;
@@ -1477,11 +1693,11 @@ static CBMDumpNode *build_dump_nodes(cbm_gbuf_t *gb, int live_count, int64_t *te
int64_t max_temp_id, int *out_count,
cbm_gbuf_node_t ***src_out) {
size_t cap = (size_t)(live_count > 0 ? live_count : SKIP_ONE);
CBMDumpNode *dump_nodes = malloc(cap * sizeof(CBMDumpNode));
CBMDumpNode *dump_nodes = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(CBMDumpNode));
/* Parallel gbuf-node pointers so a streamed partition can free its heavy
* properties_json after the rows are persisted. NULL on OOM disables the
* per-partition free (the dump still succeeds). */
cbm_gbuf_node_t **src = malloc(cap * sizeof(cbm_gbuf_node_t *));
cbm_gbuf_node_t **src = cbm_alloc(CBM_MEM_CLASS_DUMP, cap * sizeof(cbm_gbuf_node_t *));
int idx = 0;
for (int i = 0; i < gb->nodes.count; i++) {
@@ -1534,10 +1750,10 @@ static CBMDumpEdge *build_dump_edges(cbm_gbuf_t *gb, const int64_t *temp_to_fina
}
}
CBMDumpEdge *dump_edges =
malloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE) * sizeof(CBMDumpEdge));
char **url_paths = calloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE), sizeof(char *));
char **local_names = calloc((size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE), sizeof(char *));
size_t edge_cap = (size_t)(valid_edges > 0 ? valid_edges : SKIP_ONE);
CBMDumpEdge *dump_edges = cbm_alloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(CBMDumpEdge));
char **url_paths = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *));
char **local_names = cbm_calloc(CBM_MEM_CLASS_DUMP, edge_cap * sizeof(char *));
int idx = 0;
for (int i = 0; i < gb->edges.count; i++) {
@@ -1629,14 +1845,14 @@ static void free_dump_resources(char **url_paths, char **local_names, int edge_c
CBMDumpEdge *dump_edges, CBMDumpNode *dump_nodes,
int64_t *temp_to_final) {
for (int i = 0; i < edge_count; i++) {
free(url_paths[i]);
free(local_names[i]);
cbm_free(CBM_MEM_CLASS_DUMP, url_paths[i]);
cbm_free(CBM_MEM_CLASS_DUMP, local_names[i]);
}
free(url_paths);
free(local_names);
free(dump_edges);
free(dump_nodes);
free(temp_to_final);
cbm_free(CBM_MEM_CLASS_DUMP, url_paths);
cbm_free(CBM_MEM_CLASS_DUMP, local_names);
cbm_free(CBM_MEM_CLASS_DUMP, dump_edges);
cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes);
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final);
}
static int count_live_nodes(cbm_gbuf_t *gb) {
@@ -1678,7 +1894,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
CBM_PROF_START(t_build_nodes);
int64_t max_temp_id = gb->next_id;
int64_t *temp_to_final = calloc((size_t)max_temp_id, sizeof(int64_t));
int64_t *temp_to_final = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
if (!temp_to_final) {
return CBM_NOT_FOUND;
}
@@ -1710,9 +1926,9 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
* uninitialized budget from ever triggering the free). */
cbm_db_writer_t *w = cbm_writer_open(path);
if (!w) {
free(src_nodes);
free(dump_nodes);
free(temp_to_final);
cbm_free(CBM_MEM_CLASS_DUMP, src_nodes);
cbm_free(CBM_MEM_CLASS_DUMP, dump_nodes);
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_final);
return CBM_NOT_FOUND;
}
@@ -1732,7 +1948,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
free_heavy = free_heavy || (cbm_mem_budget() > 0 && cbm_mem_over_budget());
if (free_heavy && src_nodes) {
for (int j = off; j < off + chunk; j++) {
free(src_nodes[j]->properties_json);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, src_nodes[j]->properties_json);
src_nodes[j]->properties_json = NULL;
dump_nodes[j].properties = NULL;
}
@@ -1766,7 +1982,7 @@ int cbm_gbuf_dump_to_sqlite(cbm_gbuf_t *gb, const char *path) {
log_dump_summary(node_idx, edge_idx);
free_dump_resources(url_paths, local_names, edge_idx, dump_edges, dump_nodes, temp_to_final);
free(src_nodes);
cbm_free(CBM_MEM_CLASS_DUMP, src_nodes);
return rc;
}
@@ -1791,7 +2007,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) {
* Temp IDs start at 1 and are sequential, but can have gaps from edge inserts.
* Use max_id as size. */
int64_t max_temp_id = gb->next_id;
int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t));
int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
for (int i = 0; i < gb->nodes.count; i++) {
cbm_gbuf_node_t *n = gb->nodes.items[i];
@@ -1840,7 +2056,7 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store) {
cbm_store_create_indexes(store);
cbm_store_end_bulk(store);
free(temp_to_real);
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real);
return 0;
}
@@ -1854,7 +2070,7 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) {
/* Build temp_id → real_id map */
int64_t max_temp_id = gb->next_id;
int64_t *temp_to_real = calloc(max_temp_id, sizeof(int64_t));
int64_t *temp_to_real = cbm_calloc(CBM_MEM_CLASS_DUMP, (size_t)max_temp_id * sizeof(int64_t));
for (int i = 0; i < gb->nodes.count; i++) {
cbm_gbuf_node_t *n = gb->nodes.items[i];
@@ -1899,6 +2115,6 @@ int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store) {
cbm_store_commit(store);
free(temp_to_real);
cbm_free(CBM_MEM_CLASS_DUMP, temp_to_real);
return 0;
}
src/graph_buffer/graph_buffer.h
+13
View File
@@ -50,6 +50,12 @@ cbm_gbuf_t *cbm_gbuf_new(const char *project, const char *root_path);
* IDs are allocated via atomic_fetch_add on *id_source.
* Used for parallel extraction where multiple gbufs need unique IDs.
* If id_source is NULL, behaves like cbm_gbuf_new(). */
/* A worker buffer: shared ids, no secondary indexes (by label / name / edge
* type). It is filled by one worker and merged into the main buffer; the
* finders that need those indexes return nothing on it. */
cbm_gbuf_t *cbm_gbuf_new_worker(const char *project, const char *root_path,
_Atomic int64_t *id_source);
cbm_gbuf_t *cbm_gbuf_new_shared_ids(const char *project, const char *root_path,
_Atomic int64_t *id_source);
@@ -175,4 +181,11 @@ int cbm_gbuf_flush_to_store(cbm_gbuf_t *gb, cbm_store_t *store);
* Returns 0 on success. */
int cbm_gbuf_merge_into_store(cbm_gbuf_t *gb, cbm_store_t *store);
/* Replace a node's properties_json with a buffer-owned copy of json (NULL
* maps to "{}"). The buffer allocates and frees every string it owns through
* the memory core; passes that rewrite properties MUST go through here rather
* than freeing node->properties_json themselves. Returns 0, or -1 on OOM
* (the old value is kept). */
int cbm_gbuf_node_set_properties_json(cbm_gbuf_node_t *node, const char *json);
#endif /* CBM_GRAPH_BUFFER_H */
src/main.c
+5
View File
@@ -854,6 +854,11 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo
const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG);
const char *worker_quarantine =
cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG);
if (index_worker) {
/* The graph lives on this process's heaps: SQLite gets heaps of its
* own here, and only here (cbm_sqlite_dedicated_heap). */
cbm_sqlite_dedicated_heap(true);
}
cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread,
worker_marker, worker_quarantine,
cbm_index_worker_memory_budget_bytes());
src/mcp/mcp.c
+28 -5
View File
@@ -11228,11 +11228,34 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
yyjson_mut_obj_add_str(doc, root, "previous_index", "preserved");
yyjson_mut_obj_add_int(doc, root, "budget_mb", budget_mb);
yyjson_mut_obj_add_int(doc, root, "peak_rss_mb", peak_rss_mb);
yyjson_mut_obj_add_str(doc, root, "hint",
"Indexing stopped: resident memory stayed above the budget after "
"backpressure; no partial graph was published and the previous "
"index still serves. Raise CBM_MEM_BUDGET_MB, lower CBM_WORKERS, "
"or exclude large subtrees.");
/* A CONCRETE retry value, because "raise CBM_MEM_BUDGET_MB" alone makes
* the caller guess — and the obvious guess is wrong. peak_rss_mb is
* where the run was STOPPED (it is pinned just above the budget by
* construction), not what the repo needs, so retrying at peak+10% fails
* again. Measured 2026-09-13 on the linux kernel: aborted at 25622 MB
* against a 24576 MB budget, but completing it actually took 31.75 GB —
* 1.32x the budget, 1.24x the reported peak. Suggest 1.5x the budget so
* the first retry has a real chance, and say plainly that the peak is a
* floor rather than a requirement. */
/* (3*b+1)/2 rather than b + b/2: integer division makes the latter
* degenerate to b for b == 1, so the "suggestion" would repeat the
* budget that just failed. Rounding up keeps it strictly larger for
* every positive budget. */
int suggested_budget_mb = budget_mb > 0 ? (budget_mb * 3 + 1) / 2 : 0;
char hint_text[CBM_SZ_512];
(void)snprintf(hint_text, sizeof(hint_text),
"Indexing stopped: resident memory stayed above the budget after "
"backpressure; no partial graph was published and the previous index "
"still serves. peak_rss_mb is where indexing was STOPPED, not what this "
"repo needs — the real requirement is higher, so retrying just above the "
"peak will fail again. Retry with CBM_MEM_BUDGET_MB=%d (1.5x the current "
"budget) if the machine has the RAM, or lower CBM_WORKERS, or exclude "
"large subtrees.",
suggested_budget_mb);
if (suggested_budget_mb > 0) {
yyjson_mut_obj_add_int(doc, root, "suggested_budget_mb", suggested_budget_mb);
}
yyjson_mut_obj_add_strcpy(doc, root, "hint", hint_text);
} else if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB) {
/* The truthful abort message (#2020): the old generic "check repo_path"
* hint sent people debugging a path that was fine, when the run
src/pipeline/lsp_surface.c
+8 -5
View File
@@ -17,6 +17,7 @@
* bytes is the early-cutoff key: a body edit reserializes identically.
*/
#include "pipeline/lsp_surface.h"
#include "pipeline/pipeline_internal.h"
#include <stdlib.h>
#include <string.h>
@@ -126,8 +127,8 @@ static char *surface_file_to_json(const CBMFileResult *result, const CBMLSPDef *
return json;
}
int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
const cbm_file_info_t *files, int file_count,
int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project,
CBMFileResult **cache, const cbm_file_info_t *files, int file_count,
const CBMLSPDef *all_defs, const int *def_starts,
cbm_lsp_surface_row_t **out_rows, int *out_count) {
*out_rows = NULL;
@@ -141,7 +142,9 @@ int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
}
int n = 0;
for (int i = 0; i < file_count; i++) {
if (!cache[i]) {
bool loaded = false;
CBMFileResult *fr = cbm_pipeline_result_acquire(ctx, cache, i, NULL, &loaded);
if (!fr) {
/* Never parsed this run (read/extract skip): no surface claim.
* The routing layer treats a missing row as "must full-rebuild
* before this file can be reasoned about", which is the correct
@@ -150,8 +153,8 @@ int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
}
int start = def_starts ? def_starts[i] : 0;
int end = def_starts ? def_starts[i + 1] : 0;
char *json =
surface_file_to_json(cache[i], all_defs ? all_defs + start : NULL, end - start);
char *json = surface_file_to_json(fr, all_defs ? all_defs + start : NULL, end - start);
cbm_pipeline_result_release(fr, loaded);
if (!json) {
cbm_store_free_lsp_surfaces(rows, n);
return -1;
src/pipeline/lsp_surface.h
+2 -2
View File
@@ -29,8 +29,8 @@
* heap strings; release with cbm_store_free_lsp_surfaces. Files with an
* empty surface still get a row (empty arrays hash too — "no defs" must be
* distinguishable from "no data"). Returns 0, or -1 on allocation failure. */
int cbm_lsp_surface_build_rows(const char *project, CBMFileResult **cache,
const cbm_file_info_t *files, int file_count,
int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project,
CBMFileResult **cache, const cbm_file_info_t *files, int file_count,
const CBMLSPDef *all_defs, const int *def_starts,
cbm_lsp_surface_row_t **out_rows, int *out_count);
src/pipeline/pass_complexity.c
+2 -2
View File
@@ -95,8 +95,8 @@ static void append_complexity_props(cbm_gbuf_node_t *node, int tld, bool recursi
free(neu);
return;
}
free(node->properties_json);
node->properties_json = neu;
(void)cbm_gbuf_node_set_properties_json(node, neu);
free(neu);
}
/* Content-only node order: qualified_name, then file path and start line.
src/pipeline/pass_importance.c
+2 -2
View File
@@ -292,8 +292,8 @@ void cbm_pipeline_importance_append_prop(cbm_gbuf_node_t *node, double score) {
if (!neu) {
return;
}
free(node->properties_json);
node->properties_json = neu;
(void)cbm_gbuf_node_set_properties_json(node, neu);
free(neu);
}
/* ── The scoring rule ─────────────────────────────────────────────────
src/pipeline/pass_lsp_cross.c
+173 -87
View File
@@ -16,6 +16,7 @@
#include "pipeline/pass_lsp_cross.h"
#include "pipeline/lsp_surface.h"
#include "result_spill.h"
#include "pipeline/pipeline_internal.h"
#include "pipeline/lsp_resolve.h"
#include "lsp/go_lsp.h"
@@ -104,7 +105,11 @@ static char *pxc_read_file(const char *path, int *out_len) {
* plus Protocol/Function/Method — variables, modules, decorators, etc. are
* skipped. Struct passes through so Rust/Go struct type-registration via the
* cross-file LSP path is not dropped. */
static const char *pxc_map_label(const char *label) {
/* The label a cross def keeps is a static spelling, never the result's own
* pointer: in spill mode the CBMFileResult is freed (parked on disk) the
* moment the collector is done with it, and the defs outlive it. A kept
* spelling outside the table falls back to an arena copy. */
static const char *pxc_map_label(CBMArena *arena, const char *label) {
if (!label)
return NULL;
if (cbm_label_is_type_like(label) || strcmp(label, "Protocol") == 0 ||
@@ -115,7 +120,14 @@ static const char *pxc_map_label(const char *label) {
* name-only fallback). Every registrar filters by explicit label, so
* the other languages ignore Variable defs untouched. */
strcmp(label, "Variable") == 0) {
return label;
static const char *const canon[] = {"Class", "Struct", "Interface", "Enum", "Type",
"Trait", "Protocol", "Function", "Method", "Variable"};
for (size_t i = 0; i < sizeof(canon) / sizeof(canon[0]); i++) {
if (strcmp(label, canon[i]) == 0) {
return canon[i];
}
}
return arena ? cbm_arena_strdup(arena, label) : NULL;
}
return NULL;
}
@@ -268,22 +280,28 @@ static const char *pxc_last_component(const char *qn) {
return dot ? dot + 1 : qn;
}
/* Every return is arena-owned: the fallback spelling is a copy, never the
* result's pointer (the result may be parked on disk before the def is read). */
static const char *pxc_jvm_type_qn(CBMArena *arena, const char *namespace_name,
const char *type_qn_or_name) {
if (!arena || !namespace_name || !namespace_name[0] || !type_qn_or_name) {
return type_qn_or_name;
if (!arena || !type_qn_or_name) {
return NULL;
}
const char *short_name = pxc_last_component(type_qn_or_name);
const char *short_name =
(namespace_name && namespace_name[0]) ? pxc_last_component(type_qn_or_name) : NULL;
if (!short_name || !short_name[0]) {
return type_qn_or_name;
return cbm_arena_strdup(arena, type_qn_or_name);
}
return cbm_arena_sprintf(arena, "%s.%s", namespace_name, short_name);
}
static const char *pxc_jvm_def_qn(CBMArena *arena, const CBMDefinition *src,
const char *namespace_name, const char *label) {
if (!arena || !src || !namespace_name || !namespace_name[0]) {
return src ? src->qualified_name : NULL;
if (!arena || !src) {
return NULL;
}
if (!namespace_name || !namespace_name[0]) {
return cbm_arena_strdup(arena, src->qualified_name); /* a copy, see above */
}
if (strcmp(label, "Method") == 0 || strcmp(label, "Function") == 0 ||
strcmp(label, "Constructor") == 0) {
@@ -373,7 +391,7 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch
const char *namespace_name, CBMLanguage lang, CBMLSPDef *dst,
const cbm_registry_t *reg, const char **imp_keys,
const char **imp_vals, int imp_count) {
const char *label = pxc_map_label(src->label);
const char *label = pxc_map_label(arena, src->label);
if (!label || !src->qualified_name || !src->name)
return -1;
memset(dst, 0, sizeof(*dst));
@@ -381,18 +399,20 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch
dst->qualified_name = pxc_jvm_def_qn(arena, src, namespace_name, label);
dst->receiver_type = pxc_jvm_type_qn(arena, namespace_name, src->parent_class);
} else {
dst->qualified_name = src->qualified_name;
dst->receiver_type = src->parent_class;
dst->qualified_name = cbm_arena_strdup(arena, src->qualified_name);
dst->receiver_type = src->parent_class ? cbm_arena_strdup(arena, src->parent_class) : NULL;
}
dst->short_name = src->name;
/* Copies, not borrows: a result may be parked on disk (spill mode) the
* moment the collector is done with it; the def list outlives it. */
dst->short_name = cbm_arena_strdup(arena, src->name);
dst->label = label;
dst->def_module_qn = module_qn;
dst->namespace_name = namespace_name;
dst->namespace_name = namespace_name ? cbm_arena_strdup(arena, namespace_name) : NULL;
dst->is_interface = (strcmp(label, "Interface") == 0 || strcmp(label, "Protocol") == 0);
/* Single return-type string. The per-language registrars split on '|'
* for multi-return languages (Go); single-return languages just see one
* piece, which is what's already stored. */
dst->return_types = src->return_type;
dst->return_types = src->return_type ? cbm_arena_strdup(arena, src->return_type) : NULL;
/* Languages whose cross registrars read embedded_types as QNs get their
* bases resolved against the project registry; everyone else keeps the
* raw source spelling their own registrar already knows how to handle. */
@@ -400,14 +420,42 @@ static int pxc_build_lsp_def(CBMArena *arena, const CBMDefinition *src, const ch
? pxc_join_base_qns(arena, src->base_classes, reg, module_qn,
imp_keys, imp_vals, imp_count)
: pxc_join_pipe(arena, src->base_classes);
dst->signature_param_types = src->signature_param_types;
dst->signature_param_count = src->signature_param_count;
dst->signature_param_types = NULL;
dst->signature_param_count = 0;
if (src->signature_param_types && src->signature_param_count > 0) {
const char **types = (const char **)cbm_arena_alloc(
arena, (size_t)src->signature_param_count * sizeof(const char *));
if (types) {
for (int i = 0; i < src->signature_param_count; i++) {
types[i] = src->signature_param_types[i]
? cbm_arena_strdup(arena, src->signature_param_types[i])
: NULL;
}
dst->signature_param_types = types;
dst->signature_param_count = src->signature_param_count;
}
}
dst->lang = lang;
dst->decorators = src->decorators;
dst->decorators = NULL;
if (src->decorators) {
int n = 0;
while (src->decorators[n]) {
n++;
}
const char **decos =
(const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *));
if (decos) {
for (int i = 0; i < n; i++) {
decos[i] = cbm_arena_strdup(arena, src->decorators[i]);
}
decos[n] = NULL;
dst->decorators = decos;
}
}
if (lang == CBM_LANG_RUST) {
/* Exact impl-block provenance is captured while the Rust impl node is
* still on hand. Do not reconstruct it later from leaf names. */
dst->trait_qn = src->impl_trait;
dst->trait_qn = src->impl_trait ? cbm_arena_strdup(arena, src->impl_trait) : NULL;
dst->is_abstract = src->is_abstract;
}
return 0;
@@ -490,14 +538,14 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp
if (!arena || !impl || !impl->trait_name || !impl->struct_name || !impl->struct_qn) {
return -1;
}
const char *receiver_qn = impl->struct_qn;
const char *receiver_qn = cbm_arena_strdup(arena, impl->struct_qn);
memset(dst, 0, sizeof(*dst));
dst->qualified_name = receiver_qn;
dst->short_name = pxc_qn_leaf(receiver_qn);
dst->label = "RustImpl";
dst->receiver_type = receiver_qn;
dst->def_module_qn = module_qn;
dst->trait_qn = impl->trait_name; /* raw; canonicalized by Rust registry */
dst->trait_qn = cbm_arena_strdup(arena, impl->trait_name); /* raw; canonicalized later */
dst->lang = CBM_LANG_RUST;
dst->is_rust_impl_relation = true;
return 0;
@@ -506,17 +554,24 @@ static int pxc_build_rust_impl_relation(CBMArena *arena, const CBMImplTrait *imp
/* Collect a project-wide CBMLSPDef[] from all cached results. Returns a
* malloc'd array (caller frees) of length *out_count. String fields are
* borrowed from cache[i]->arena and from def_modules[i] (also borrowed). */
CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache,
const cbm_file_info_t *files, int file_count,
const char *project_name, char **def_modules, int *out_count,
int *out_def_starts) {
CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *arena,
CBMFileResult **cache, const cbm_file_info_t *files,
int file_count, const char *project_name, char **def_modules,
int *out_count, int *out_def_starts) {
const cbm_result_spill_t *spill = ctx ? ctx->spill : NULL;
int total = 0;
for (int i = 0; i < file_count; i++) {
int defs = 0;
int impls = 0;
if (cache[i]) {
total += cache[i]->defs.count;
if (files[i].language == CBM_LANG_RUST) {
total += cache[i]->impl_traits.count;
}
defs = cache[i]->defs.count;
impls = cache[i]->impl_traits.count;
} else if (spill && cbm_result_spill_has(spill, i)) {
cbm_result_spill_peek_counts(spill, i, &defs, &impls);
}
total += defs;
if (files[i].language == CBM_LANG_RUST) {
total += impls;
}
}
if (total == 0) {
@@ -540,19 +595,21 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
out_def_starts[fi] = idx;
}
const int file_start = idx;
if (!cache[fi])
CBMFileResult *fr = cache[fi];
bool fr_loaded = false;
if (!fr && spill && cbm_result_spill_has(spill, fi)) {
fr = cbm_result_spill_load(spill, fi);
fr_loaded = fr != NULL;
}
if (!fr)
continue;
if (!def_modules[fi]) {
def_modules[fi] = cbm_pipeline_fqn_module_dir(project_name, files[fi].rel_path,
pxc_module_is_dir(files[fi].language));
}
const char *namespace_name = cache[fi]->namespace_name;
const char *namespace_name = fr->namespace_name;
if ((!namespace_name || !namespace_name[0]) && files[fi].rel_path) {
namespace_name =
pxc_infer_jvm_namespace(&cache[fi]->arena, files[fi].rel_path, files[fi].language);
if (namespace_name && namespace_name[0]) {
cache[fi]->namespace_name = namespace_name;
}
namespace_name = pxc_infer_jvm_namespace(arena, files[fi].rel_path, files[fi].language);
}
/* One import map per FILE (not per def, and not per base name): the
* cross-file base-class resolution below needs the same local-name →
@@ -567,29 +624,31 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult
if (ctx && ctx->registry && pxc_lang_resolves_base_qns(files[fi].language)) {
base_reg = ctx->registry;
cbm_pxc_build_import_map(ctx->gbuf, project_name, files[fi].rel_path,
files[fi].language, cache[fi], &imp_keys, &imp_vals,
&imp_count);
files[fi].language, fr, &imp_keys, &imp_vals, &imp_count);
}
for (int di = 0; di < cache[fi]->defs.count; di++) {
if (pxc_build_lsp_def(&cache[fi]->arena, &cache[fi]->defs.items[di], def_modules[fi],
namespace_name, files[fi].language, &defs[idx], base_reg,
imp_keys, imp_vals, imp_count) == 0) {
for (int di = 0; di < fr->defs.count; di++) {
if (pxc_build_lsp_def(arena, &fr->defs.items[di], def_modules[fi], namespace_name,
files[fi].language, &defs[idx], base_reg, imp_keys, imp_vals,
imp_count) == 0) {
idx++;
}
}
cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */
if (files[fi].language == CBM_LANG_GO) {
pxc_fold_go_struct_fields(&cache[fi]->arena, cache[fi], defs, file_start, idx);
pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx);
}
if (files[fi].language == CBM_LANG_RUST) {
for (int ii = 0; ii < cache[fi]->impl_traits.count; ii++) {
if (pxc_build_rust_impl_relation(
&cache[fi]->arena, &cache[fi]->impl_traits.items[ii], project_name,
files[fi].rel_path, def_modules[fi], &defs[idx]) == 0) {
for (int ii = 0; ii < fr->impl_traits.count; ii++) {
if (pxc_build_rust_impl_relation(arena, &fr->impl_traits.items[ii], project_name,
files[fi].rel_path, def_modules[fi],
&defs[idx]) == 0) {
idx++;
}
}
}
if (fr_loaded) {
cbm_free_result(fr);
}
}
if (out_def_starts) {
out_def_starts[file_count] = idx;
@@ -1065,7 +1124,15 @@ static void pxc_append_synthetic_calls(CBMArena *dst_arena, CBMCallArray *dst_ca
src->first_string_arg ? cbm_arena_strdup(dst_arena, src->first_string_arg) : NULL;
dst.second_arg_name =
src->second_arg_name ? cbm_arena_strdup(dst_arena, src->second_arg_name) : NULL;
for (int ai = 0; ai < CBM_MAX_CALL_ARGS; ai++) {
dst.args = NULL;
if (src->args && src->arg_count > 0) {
dst.args = cbm_arena_calloc(dst_arena, (size_t)src->arg_count * sizeof(CBMCallArg));
if (!dst.args) {
dst.arg_count = 0;
}
}
for (int ai = 0; dst.args && ai < src->arg_count; ai++) {
dst.args[ai].index = src->args[ai].index;
dst.args[ai].expr =
src->args[ai].expr ? cbm_arena_strdup(dst_arena, src->args[ai].expr) : NULL;
dst.args[ai].value =
@@ -1272,6 +1339,10 @@ void cbm_pxc_filter_stats(uint64_t *defs_registered, uint64_t *build_files, uint
*filter_failed = atomic_load_explicit(&g_pxc_filter_failed, memory_order_relaxed);
}
/* A file the extractor marked lsp_skipped (its parse alone used more than its
* share of the per-file budget) takes no cross-file resolve either: the same
* tree the per-file walk could not afford. One check for every language and
* both drivers. */
void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *source,
int source_len, const char *rel, const char *def_module,
const CBMCrossLspRegistries *cross_registries,
@@ -1279,6 +1350,9 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
int all_def_count, const char **imp_keys, const char **imp_vals,
int imp_count, CBMTypeRegistry *(*rust_shared_get)(void *),
void *rust_shared_ctx) {
if (result && result->lsp_skipped) {
return;
}
if (!result) {
return;
}
@@ -1286,63 +1360,64 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
CBMTypeRegistry *prebuilt =
cross_registries ? cbm_pxc_registry_for_lang(cross_registries, lang) : NULL;
if (prebuilt) {
/* One lifecycle for every language (2026-09-13): the walk runs in a
* scratch arena against a per-file OVERLAY registry chained to the
* immutable shared base; whatever it registers or refines lands in
* the overlay and dies with the file; only the resolved calls it
* produces are copied into the result. Before this, Go/C/C#/Java/TS
* were handed the RESULT arena (and C/C#/TS the shared base to write
* into): the kernel proof measured that as 4.4 GB of resolve-time
* growth in retained results plus arena pointers stored into the
* shared registry -- a use-after-free class the moment the arena was
* not the result arena (ASan, lsp_resolution_probe). */
CBMArena scratch;
cbm_arena_init(&scratch);
CBMTypeRegistry overlay;
cbm_registry_init(&overlay, &scratch);
overlay.fallback = prebuilt;
CBMResolvedCallArray out = {0};
CBMCallArray synthetic_calls = {0};
switch (lang) {
case CBM_LANG_GO:
/* Tier 3 (metadata-driven): pure lookup over the Tier-1
* lsp_unresolved entries — no parse, no AST walk. Then the
* AST walk on the shared Tier-2 registry (mirroring every
* other language) so NAMED receivers evaluated against
* project-wide defs also resolve. The walk variant below is
* read-only — the sealed registry is safe for parallel
* workers. */
* lsp_unresolved entries -- no parse, no AST walk -- then the
* AST walk against the overlay (chained to the sealed base). */
cbm_go_fast_resolve_qualified_calls(result, prebuilt, imp_keys, imp_vals, imp_count);
cbm_run_go_lsp_cross_with_registry(&result->arena, source, source_len, def_module,
prebuilt, imp_keys, imp_vals, imp_count,
result->cached_tree, &result->resolved_calls);
cbm_run_go_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay,
imp_keys, imp_vals, imp_count, result->cached_tree,
&out);
used_prebuilt = true;
break;
case CBM_LANG_PYTHON: {
CBMArena scratch;
cbm_arena_init(&scratch);
CBMResolvedCallArray out = {0};
CBMCallArray synthetic_calls = {0};
cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, prebuilt,
case CBM_LANG_PYTHON:
cbm_run_py_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay,
imp_keys, imp_vals, imp_count, result->cached_tree,
&out, &synthetic_calls);
pxc_append_results(&result->arena, &result->resolved_calls, &out);
pxc_append_synthetic_calls(&result->arena, &result->calls, &synthetic_calls);
cbm_arena_destroy(&scratch);
used_prebuilt = true;
break;
}
case CBM_LANG_C:
case CBM_LANG_CPP:
case CBM_LANG_CUDA:
cbm_run_c_lsp_cross_with_registry(
&result->arena, source, source_len, def_module, (lang != CBM_LANG_C), prebuilt,
imp_keys, imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
cbm_run_c_lsp_cross_with_registry(&scratch, source, source_len, def_module,
(lang != CBM_LANG_C), &overlay, imp_keys, imp_vals,
imp_count, result->cached_tree, &out);
used_prebuilt = true;
break;
case CBM_LANG_CSHARP:
cbm_run_cs_lsp_cross_with_registry(&result->arena, source, source_len, def_module,
prebuilt, imp_vals, imp_count, result->cached_tree,
&result->resolved_calls);
cbm_run_cs_lsp_cross_with_registry(&scratch, source, source_len, def_module, &overlay,
imp_vals, imp_count, result->cached_tree, &out);
used_prebuilt = true;
break;
case CBM_LANG_JAVA:
/* Own-module defs go into a per-file overlay; imports and stdlib
* resolve through the shared base (#1669). */
cbm_run_java_lsp_cross_with_registry(
&result->arena, result, source, source_len, def_module, prebuilt, imp_keys,
imp_vals, imp_count, result->cached_tree, &result->resolved_calls);
/* Java builds its own-module overlay on top of the one it is
* handed; both live in scratch now. */
cbm_run_java_lsp_cross_with_registry(&scratch, result, source, source_len, def_module,
&overlay, imp_keys, imp_vals, imp_count,
result->cached_tree, &out);
used_prebuilt = true;
break;
case CBM_LANG_JAVASCRIPT:
case CBM_LANG_TYPESCRIPT:
case CBM_LANG_TSX: {
/* TS: per-file OVERLAY chained to the shared base. Filter to
* own+imports so the overlay builder can pick out own-module
* defs without scanning the whole project. */
bool js;
bool jsx;
bool dts;
@@ -1361,10 +1436,9 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
ts_def_count = fc;
}
}
cbm_run_ts_lsp_cross_with_registry(&result->arena, source, source_len, def_module, js,
jsx, dts, prebuilt, ts_defs, ts_def_count, imp_keys,
imp_vals, imp_count, result->cached_tree,
&result->resolved_calls);
cbm_run_ts_lsp_cross_with_registry(&scratch, source, source_len, def_module, js, jsx,
dts, &overlay, ts_defs, ts_def_count, imp_keys,
imp_vals, imp_count, result->cached_tree, &out);
free(ts_filtered);
used_prebuilt = true;
break;
@@ -1374,6 +1448,11 @@ void cbm_pxc_dispatch_file(CBMLanguage lang, CBMFileResult *result, const char *
default:
break;
}
if (used_prebuilt) {
pxc_append_results(&result->arena, &result->resolved_calls, &out);
pxc_append_synthetic_calls(&result->arena, &result->calls, &synthetic_calls);
}
cbm_arena_destroy(&scratch);
}
if (used_prebuilt) {
@@ -1498,15 +1577,22 @@ int cbm_pipeline_pass_lsp_cross(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *
int def_count = 0;
int *def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int));
CBMLSPDef *all_defs = cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name,
def_modules, &def_count, def_starts);
/* The defs own their strings in the caller-owned seq_cross_arena, which
* the registries and later passes already borrow from (see below). */
if (!ctx->seq_cross_arena_live) {
cbm_arena_init(&ctx->seq_cross_arena);
ctx->seq_cross_arena_live = true;
}
CBMLSPDef *all_defs =
cbm_pxc_collect_all_defs(ctx, &ctx->seq_cross_arena, cache, files, file_count,
ctx->project_name, def_modules, &def_count, def_starts);
/* Same seam as the parallel driver: serialize per-file surfaces while the
* result cache is alive. Failure only degrades to a full rebuild on the
* next incremental run. */
if (ctx->pipeline && all_defs && def_starts) {
cbm_lsp_surface_row_t *surface_rows = NULL;
int surface_count = 0;
if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs,
if (cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, files, file_count, all_defs,
def_starts, &surface_rows, &surface_count) == 0) {
cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count);
}
src/pipeline/pass_lsp_cross.h
+4 -4
View File
@@ -63,10 +63,10 @@ bool cbm_pxc_has_cross_lsp(CBMLanguage lang);
* inputs pass_semantic uses to draw its INHERITS edge, so the LSP's
* inheritance view and the graph's cannot diverge. Pass NULL to keep the raw
* source spelling (surface-probe paths that build no registry). */
CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache,
const cbm_file_info_t *files, int file_count,
const char *project_name, char **def_modules, int *out_count,
int *out_def_starts);
CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *arena,
CBMFileResult **cache, const cbm_file_info_t *files,
int file_count, const char *project_name, char **def_modules,
int *out_count, int *out_def_starts);
/* Detect TS dialect flags from a relative path. */
void cbm_pxc_ts_modes(CBMLanguage lang, const char *rel_path, bool *out_js, bool *out_jsx,
src/pipeline/pass_parallel.c
+317 -8
View File
@@ -70,6 +70,8 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
#define PP_RETAIN_PER_FILE_HARD_MAX_BYTES (32ULL * 1024 * 1024) /* 32 MiB per file */
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include "result_spill.h"
#include "foundation/platform.h" /* cbm_resolve_cache_dir */
#include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */
#include "pipeline/lsp_resolve.h"
#include "lsp/rust_cargo.h"
@@ -78,6 +80,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 };
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/mem_core.h"
#include "graph_buffer/graph_buffer.h"
#include "service_patterns.h"
#include "foundation/platform.h"
@@ -649,6 +652,12 @@ typedef struct {
_Atomic int64_t *shared_ids;
_Atomic int *cancelled;
_Atomic int next_file_idx;
cbm_pipeline_ctx_t *pctx; /* spill store + mode latch live here */
_Atomic uint8_t *slot_state; /* per file: 0 in progress, 1 cached, 2 parked */
_Atomic int spill_sweeps_running; /* workers inside pp_spill_sweep right now */
int spill_env; /* CBM_MEM_SPILL for this run: 1 force, 0 off, 2 budget decides */
cbm_mutex_t spill_mu; /* serializes the one store open */
_Atomic int spill_unavailable; /* store could not open: never retry */
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
@@ -762,13 +771,204 @@ static void pp_fail_whole_over_budget(extract_ctx_t *ec) {
}
}
/* ── Spill / admission control ──────────────────────────────────────── */
/* Spill mode is entered this fraction short of the budget: 1/16 = 6.25%,
* ~940 MB at 15 GB, above the 4% the kernel's in-flight files were measured
* to carry past the line. */
enum { PP_SPILL_EARLY_DIV = 16 };
/* CBM_MEM_SPILL=1 forces spill mode from the first file (tests, small
* machines); CBM_MEM_SPILL=0 keeps results in memory even over budget. Read
* once per extraction run, never cached per process: a test toggles it. */
static int pp_spill_env_read(void) {
char buf[CBM_SZ_16];
if (cbm_safe_getenv("CBM_MEM_SPILL", buf, sizeof(buf), NULL)) {
return buf[0] == '1' ? 1 : 0;
}
return 2; /* unset: the budget decides */
}
static bool pp_spill_forced_by_env(const extract_ctx_t *ec) {
return ec->spill_env == 1;
}
static bool pp_spill_allowed(const extract_ctx_t *ec) {
return ec->spill_env != 0;
}
/* Latch spill mode (once) and open the store. The store is open BEFORE the
* latch is visible: a peer that reads the latch must also find the store, or
* it sees "nothing to park" while this worker is still opening files and runs
* the futility cycle instead of joining the sweep (kernel, 16 GB budget,
* 2026-09-13: the abort fired before the first sweep had logged). Never fails
* the run: without a store the results simply stay in memory and the nap /
* futility path applies as before. */
static void pp_spill_enter(extract_ctx_t *ec, const char *reason) {
if (!ec->pctx || !ec->pctx->spill_allowed || !pp_spill_allowed(ec)) {
return;
}
if (atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) != 0 ||
atomic_load_explicit(&ec->spill_unavailable, memory_order_relaxed) != 0) {
return;
}
cbm_mutex_lock(&ec->spill_mu);
if (atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) == 0 &&
atomic_load_explicit(&ec->spill_unavailable, memory_order_relaxed) == 0) {
if (!ec->pctx->spill) {
ec->pctx->spill =
cbm_result_spill_open(cbm_resolve_cache_dir(), ec->max_workers, ec->file_count);
}
size_t mb = (size_t)1024 * 1024;
cbm_log_warn("mem.spill.on", "reason", reason, "charged_mb",
itoa_log((int)(cbm_mem_charged() / mb)), "budget_mb",
itoa_log((int)(cbm_mem_budget() / mb)), "store",
ec->pctx->spill ? "open" : "unavailable");
if (ec->pctx->spill) {
atomic_store_explicit(&ec->pctx->spill_mode, 1, memory_order_release);
} else {
atomic_store_explicit(&ec->spill_unavailable, 1, memory_order_relaxed);
}
}
cbm_mutex_unlock(&ec->spill_mu);
}
static bool pp_spill_active(const extract_ctx_t *ec) {
return ec->pctx && ec->pctx->spill &&
atomic_load_explicit(&ec->pctx->spill_mode, memory_order_acquire) != 0;
}
CBMFileResult *cbm_pipeline_result_acquire(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache,
int i, cbm_result_want_fn want, bool *loaded) {
*loaded = false;
if (cache && cache[i]) {
return (!want || want(cache[i])) ? cache[i] : NULL;
}
if (!ctx || !ctx->spill || !cbm_result_spill_has(ctx->spill, i)) {
return NULL;
}
if (want) {
CBMFileResult hdr;
if (!cbm_result_spill_peek_header(ctx->spill, i, &hdr) || !want(&hdr)) {
return NULL;
}
}
CBMFileResult *r = cbm_result_spill_load(ctx->spill, i);
*loaded = r != NULL;
return r;
}
void cbm_pipeline_result_release(CBMFileResult *r, bool loaded) {
if (r && loaded) {
cbm_free_result(r);
}
}
void cbm_pipeline_spill_close(cbm_pipeline_ctx_t *ctx) {
if (!ctx || !ctx->spill) {
return;
}
int64_t parked = 0;
int64_t bytes = 0;
int64_t loads = 0;
cbm_result_spill_stats(ctx->spill, &parked, &bytes, &loads);
cbm_log_info("mem.spill.done", "parked", itoa_log((int)parked), "mb",
itoa_log((int)(bytes / (1024 * 1024))), "loads", itoa_log((int)loads));
cbm_result_spill_close(ctx->spill);
ctx->spill = NULL;
atomic_store_explicit(&ctx->spill_mode, 0, memory_order_release);
cbm_work_arena_release();
}
/* Park every cached result that is complete. Slots are claimed 1 -> 2 so two
* workers never park the same result; a failed park hands the slot back. */
/* True while a cached (not yet parked) result exists or a sweep is running:
* memory can still come down without anyone napping. */
static bool pp_spill_work_remains(const extract_ctx_t *ec) {
if (!ec->slot_state) {
return false;
}
if (atomic_load_explicit(&ec->spill_sweeps_running, memory_order_acquire) > 0) {
return true;
}
for (int i = 0; i < ec->file_count; i++) {
if (atomic_load_explicit(&ec->slot_state[i], memory_order_relaxed) == 1) {
return true;
}
}
return false;
}
static int pp_spill_sweep(extract_ctx_t *ec, int worker_id) {
if (!pp_spill_active(ec) || !ec->slot_state) {
return 0;
}
atomic_fetch_add_explicit(&ec->spill_sweeps_running, 1, memory_order_acq_rel);
int parked = 0;
for (int i = 0; i < ec->file_count; i++) {
if (atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) {
break;
}
uint8_t expected = 1;
if (!atomic_compare_exchange_strong_explicit(&ec->slot_state[i], &expected, (uint8_t)2,
memory_order_acq_rel, memory_order_relaxed)) {
continue;
}
CBMFileResult *r = ec->result_cache[i];
if (r && cbm_result_spill_park(ec->pctx->spill, worker_id, i, r)) {
ec->result_cache[i] = NULL;
parked++;
if ((parked & 255) == 0) {
cbm_mem_release_to_os();
}
} else {
atomic_store_explicit(&ec->slot_state[i], (uint8_t)1, memory_order_release);
}
}
if (parked > 0) {
cbm_mem_release_to_os();
cbm_log_info("mem.spill.sweep", "parked", itoa_log(parked), "charged_mb",
itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024))));
}
atomic_fetch_sub_explicit(&ec->spill_sweeps_running, 1, memory_order_acq_rel);
return parked;
}
/* Diagnostic (CBM_MEM_PHASES=1): where does the charge go between the
* near-budget latch and the first over-budget observation? One line per
* 256 MB step of the charge above its last logged value while spill mode is
* on, and one at the first over-budget observation, each with the
* footprint / commit / tracked breakdown and the class table. */
static _Atomic size_t g_probe_last_mb = 0;
static _Atomic int g_probe_over_logged = 0;
static void pp_charge_probe(extract_ctx_t *ec, bool over) {
if (!cbm_mem_phases_enabled() || !pp_spill_active(ec)) {
return;
}
enum { PROBE_STEP_MB = 256, PROBE_MB = 1024 * 1024 };
size_t charged_mb = cbm_mem_charged() / PROBE_MB;
size_t last = atomic_load_explicit(&g_probe_last_mb, memory_order_relaxed);
bool step = charged_mb >= last + PROBE_STEP_MB &&
atomic_compare_exchange_strong_explicit(&g_probe_last_mb, &last, charged_mb,
memory_order_relaxed, memory_order_relaxed);
bool first_over =
over && atomic_exchange_explicit(&g_probe_over_logged, 1, memory_order_relaxed) == 0;
if (!step && !first_over) {
return;
}
cbm_log_info("mem.charge.probe", "event", first_over ? "first_over" : "step", "charged_mb",
itoa_log((int)charged_mb), "footprint_mb",
itoa_log((int)(cbm_mem_footprint() / PROBE_MB)), "commit_mb",
itoa_log((int)(cbm_mem_allocator_committed() / PROBE_MB)), "tracked_mb",
itoa_log((int)(cbm_mem_tracked_live_bytes() / PROBE_MB)));
cbm_mem_class_log(first_over ? "charge.first_over" : "charge.step");
}
static void extract_worker(int worker_id, void *ctx_ptr) {
extract_ctx_t *ec = ctx_ptr;
extract_worker_state_t *ws = &ec->workers[worker_id];
/* Lazy gbuf creation */
if (!ws->local_gbuf) {
ws->local_gbuf = cbm_gbuf_new_shared_ids(ec->project_name, ec->repo_path, ec->shared_ids);
ws->local_gbuf = cbm_gbuf_new_worker(ec->project_name, ec->repo_path, ec->shared_ids);
}
/* Pull files from shared atomic counter */
@@ -805,6 +1005,35 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
* and the previously serving index keeps answering. */
if (cbm_mem_budget() > 0) {
bool over = cbm_mem_over_budget();
pp_charge_probe(ec, over);
/* Anticipation: the gate sees the crossing per file pull, and the
* workers' in-flight files carry the charge past the line before
* the first sweep lands (kernel, 15 GB budget: high-water 15.65
* GB, 4% over, all of it set in that window). Spill mode is
* entered PP_SPILL_EARLY_DIV-th short of the budget, so results
* stop accumulating in memory while the headroom still covers the
* in-flight work. The nap / futility path still keys on the
* budget itself. */
if (!over && !pp_spill_active(ec)) {
size_t budget = cbm_mem_budget();
if (cbm_mem_charged() > budget - budget / PP_SPILL_EARLY_DIV) {
pp_spill_enter(ec, "near_budget");
}
}
bool settling = false;
if (over) {
/* Admission control, first response: park what can be parked.
* Only what is still over budget after that -- the floor --
* reaches the nap / futility / abort path below, and only
* once nothing is left to park anywhere. */
pp_spill_enter(ec, "over_budget");
int parked = pp_spill_sweep(ec, worker_id);
over = cbm_mem_over_budget();
if (over && pp_spill_active(ec) && (parked > 0 || pp_spill_work_remains(ec))) {
over = false; /* memory is still on its way down */
settling = true;
}
}
bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0;
if (over && !futile) {
/* Act only on the 0→1 transition: all workers race into the
@@ -819,7 +1048,13 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
}
}
} else if (!over && futile) {
} else if (!over && !settling && futile) {
/* Re-arm only on a genuine under-budget reading. A reading the
* spill shortcut produced ("still on its way down") is not
* one: re-arming on it made the next over-budget pull pay a
* full nap cycle again -- the gate re-paid per pull that
* pipeline_backpressure_futile_nap_disengages guards against
* (TSan lane on PR #2202: 8 cycles against a bound of 7). */
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
}
}
@@ -1005,8 +1240,24 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
* and the retention copy (if any) lives in result->arena. */
free_source(source);
/* Cache result (arena + extracted data, no tree) for Phase 3B and Phase 4 */
ec->result_cache[file_idx] = result;
/* Everything this file will ever contribute has been written; drop
* the working arena (node-text copies, abandoned array generations)
* and keep only the reachable result. See cbm_result_compact. */
cbm_result_compact(result);
/* Cache result (arena + extracted data, no tree) for Phase 3B and
* Phase 4 -- or park it straight to disk in spill mode. */
if (pp_spill_active(ec) &&
cbm_result_spill_park(ec->pctx->spill, worker_id, file_idx, result)) {
if (ec->slot_state) {
atomic_store_explicit(&ec->slot_state[file_idx], (uint8_t)2, memory_order_release);
}
} else {
ec->result_cache[file_idx] = result;
if (ec->slot_state) {
atomic_store_explicit(&ec->slot_state[file_idx], (uint8_t)1, memory_order_release);
}
}
/* Progress logging: log every 10 files (atomic read, no contention) */
if ((sort_pos + SKIP_ONE) % PP_LOG_INTERVAL == 0 || sort_pos + SKIP_ONE == ec->file_count) {
@@ -1032,6 +1283,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
}
/* Final cleanup (parser already destroyed in loop, just slab state) */
cbm_work_arena_release(); /* the working arena kept between files */
cbm_slab_destroy_thread();
cbm_kind_in_set_free_cache(); /* free this worker thread's node-type bitset cache */
}
@@ -1158,6 +1410,8 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
.result_cache = result_cache,
.shared_ids = shared_ids,
.cancelled = ctx->cancelled,
.pctx = ctx,
.slot_state = NULL,
.pkg_entries = pkg_entries,
.err_lists = err_lists,
.retain_sources = resolved_opts.retain_sources,
@@ -1173,12 +1427,43 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
atomic_init(&ec.oversized_warned, 0);
atomic_init(&ec.bp_futile, 0);
atomic_init(&ec.over_budget_abort, 0);
atomic_init(&ec.spill_sweeps_running, 0);
atomic_init(&ec.spill_unavailable, 0);
cbm_mutex_init(&ec.spill_mu);
ec.slot_state = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)file_count * sizeof(_Atomic uint8_t));
ec.spill_env = pp_spill_env_read();
if (pp_spill_forced_by_env(&ec)) {
pp_spill_enter(&ec, "env");
}
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
CBM_PROF_START(t_dispatch);
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_scale_begin(&ec.scale, "parallel_extract", (long)file_count);
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
if (pp_spill_active(&ec) &&
!atomic_load_explicit(&ec.over_budget_abort, memory_order_relaxed)) {
/* Spill mode was entered, so results belong on disk: park every
* result still cached before the phases that cannot park (registry
* build, resolve, the semantic pass) inherit them. The sweeps above
* run only on an over-budget observation; a run that latched early
* and then stayed under budget through extraction (kernel, 15 GB,
* 2026-09-14: 14,949 MB at this point, 44,797 results = 8 GB still
* cached) reached resolve with no headroom and aborted there. */
int parked = pp_spill_sweep(&ec, 0);
cbm_log_info("mem.spill.final_sweep", "parked", itoa_log(parked), "charged_mb",
itoa_log((int)(cbm_mem_charged() / ((size_t)1024 * 1024))));
}
if (ctx->spill) {
int64_t parked = 0;
int64_t bytes = 0;
cbm_result_spill_stats(ctx->spill, &parked, &bytes, NULL);
cbm_log_info("mem.spill.extract_done", "parked", itoa_log((int)parked), "mb",
itoa_log((int)(bytes / (1024 * 1024))));
}
cbm_mutex_destroy(&ec.spill_mu);
cbm_free(CBM_MEM_CLASS_OTHER, ec.slot_state);
ec.slot_state = NULL;
cbm_scale_end(&ec.scale);
CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
@@ -1378,6 +1663,11 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
}
CBMFileResult *result = result_cache[i];
bool loaded = false;
if (!result && ctx->spill && cbm_result_spill_has(ctx->spill, i)) {
result = cbm_result_spill_load(ctx->spill, i);
loaded = result != NULL;
}
if (!result) {
continue;
}
@@ -1392,6 +1682,9 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
imports_edges += create_imports_edges(ctx, result, rel, namespace_map);
create_channel_edges(ctx, result, rel);
cbm_pipeline_create_env_configures_for_file(ctx, result, rel);
if (loaded) {
cbm_free_result(result);
}
}
cbm_pipeline_namespace_map_free(namespace_map);
@@ -1414,7 +1707,8 @@ typedef struct __attribute__((aligned(CBM_CACHE_LINE))) {
* registry's textual matcher. Surfaced in the parallel.resolve.done
* log line so divergence between pipelines becomes observable. */
int lsp_overrides;
char _pad[CBM_CACHE_LINE - sizeof(cbm_gbuf_t *) - ((PP_RING + 1) * sizeof(int))];
CBMFileResult *loaded; /* spill-loaded result this worker is using */
char _pad[CBM_CACHE_LINE - 2 * sizeof(void *) - ((PP_RING + 1) * sizeof(int))];
} resolve_worker_state_t;
typedef struct {
@@ -1502,6 +1796,7 @@ typedef struct {
* necessary (#1669: 87% of a Java index), so it is the one that must never
* again grow superlinear without saying so. */
cbm_scale_probe_t scale;
cbm_pipeline_ctx_t *pctx; /* spill store access */
} resolve_ctx_t;
/* Minimum buffer space needed per arg JSON object */
@@ -3048,8 +3343,7 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
cbm_pxc_set_rust_manifest(rc->rust_manifest);
if (!ws->local_edge_buf) {
ws->local_edge_buf =
cbm_gbuf_new_shared_ids(rc->project_name, rc->repo_path, rc->shared_ids);
ws->local_edge_buf = cbm_gbuf_new_worker(rc->project_name, rc->repo_path, rc->shared_ids);
}
/* Per-worker service-pattern result cache. The same resolved QN
@@ -3073,7 +3367,16 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
uint64_t _loop_t0 = extract_now_ns();
if (ws->loaded) {
cbm_free_result(ws->loaded);
ws->loaded = NULL;
}
CBMFileResult *result = rc->result_cache[file_idx];
if (!result && rc->pctx && rc->pctx->spill &&
cbm_result_spill_has(rc->pctx->spill, file_idx)) {
result = cbm_result_spill_load(rc->pctx->spill, file_idx);
ws->loaded = result;
}
if (!result) {
atomic_fetch_add_explicit(&rc->time_ns_total_loop, extract_now_ns() - _loop_t0,
memory_order_relaxed);
@@ -3293,6 +3596,10 @@ static void resolve_worker(int worker_id, void *ctx_ptr) {
* into dead TLS and are never retired, so a later cross-thread free can
* never bring their refcount to zero (leak). Retiring them here releases
* each page as its final chunk returns. */
if (ws->loaded) {
cbm_free_result(ws->loaded);
ws->loaded = NULL;
}
cbm_pxc_set_rust_manifest(NULL);
cbm_destroy_thread_parser();
cbm_slab_destroy_thread();
@@ -3322,7 +3629,8 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
bool have_rust = false;
for (int i = 0; i < file_count; i++) {
if (result_cache[i] && files[i].language == CBM_LANG_RUST) {
if (files[i].language == CBM_LANG_RUST &&
(result_cache[i] || (ctx->spill && cbm_result_spill_has(ctx->spill, i)))) {
have_rust = true;
break;
}
@@ -3340,6 +3648,7 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files,
}
resolve_ctx_t rc = {
.pctx = ctx,
.files = files,
.file_count = file_count,
.project_name = ctx->project_name,
src/pipeline/pass_semantic.c
+17 -10
View File
@@ -12,6 +12,7 @@
* Depends on: pass_definitions having populated the registry and graph buffer
*/
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "foundation/str_util.h" // cbm_json_escape
#include "pipeline/pipeline.h"
#include <stdint.h>
@@ -49,7 +50,7 @@ static char *read_file(const char *path, int *out_len) {
}
/* +pad: tree-sitter lexer lookahead reads past EOF; keep it in-bounds */
enum { CBM_TS_LOOKAHEAD_PAD = 16 };
char *buf = malloc((size_t)size + CBM_TS_LOOKAHEAD_PAD);
char *buf = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)size + CBM_TS_LOOKAHEAD_PAD);
if (!buf) {
(void)fclose(f);
return NULL;
@@ -84,8 +85,12 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
/* Fast path: build from cached extraction result (no JSON parsing) */
if (result && result->imports.count > 0) {
const char **keys = calloc((size_t)result->imports.count, sizeof(const char *));
const char **vals = calloc((size_t)result->imports.count, sizeof(const char *));
const char **keys =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
(size_t)((size_t)result->imports.count) * (sizeof(const char *)));
const char **vals =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
(size_t)((size_t)result->imports.count) * (sizeof(const char *)));
int count = 0;
for (int i = 0; i < result->imports.count; i++) {
@@ -99,7 +104,7 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
if (!target) {
continue;
}
keys[count] = strdup(imp->local_name);
keys[count] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, imp->local_name);
vals[count] = target->qualified_name;
count++;
}
@@ -126,8 +131,10 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
return 0;
}
const char **keys = calloc(edge_count, sizeof(const char *));
const char **vals = calloc(edge_count, sizeof(const char *));
const char **keys =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(edge_count) * (sizeof(const char *)));
const char **vals =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(edge_count) * (sizeof(const char *)));
int count = 0;
for (int i = 0; i < edge_count; i++) {
@@ -158,12 +165,12 @@ static int build_import_map(cbm_pipeline_ctx_t *ctx, const char *rel_path,
static void free_import_map(const char **keys, const char **vals, int count) {
if (keys) {
for (int i = 0; i < count; i++) {
free((void *)keys[i]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)keys[i]);
}
free((void *)keys);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)keys);
}
if (vals) {
free((void *)vals);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)vals);
}
}
@@ -496,7 +503,7 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx,
}
CBMFileResult *r = cbm_extract_file(source, source_len, fi->language, ctx->project_name,
fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL);
free(source);
cbm_free(CBM_MEM_CLASS_SEMANTIC, source);
if (r) {
*owned = true;
}
src/pipeline/pass_semantic_edges.c
+348 -78
View File
@@ -10,6 +10,8 @@
* Runs in moderate and full modes (not fast). Controlled by pipeline mode.
*/
#include "foundation/constants.h"
#include "foundation/mem.h"
#include "foundation/mem_core.h"
#include "pipeline/pipeline.h"
#include <stdint.h>
#include "pipeline/pipeline_internal.h"
@@ -93,7 +95,8 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt
bool same_file, int i, int j, int c) {
if (buf->count >= buf->cap) {
int nc = buf->cap < CBM_SZ_256 ? CBM_SZ_256 : buf->cap * GROW;
deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(deferred_edge_t));
deferred_edge_t *grown =
cbm_realloc(CBM_MEM_CLASS_SEMANTIC, buf->edges, (size_t)nc * sizeof(deferred_edge_t));
if (!grown) {
return;
}
@@ -110,7 +113,7 @@ static void deferred_buf_push(deferred_edge_buf_t *buf, int64_t src, int64_t tgt
}
static void deferred_buf_free(deferred_edge_buf_t *buf) {
free(buf->edges);
cbm_free(CBM_MEM_CLASS_SEMANTIC, buf->edges);
buf->edges = NULL;
buf->count = buf->cap = 0;
}
@@ -128,7 +131,7 @@ static int push_pattern_token(char **tokens, int count, int max_tokens, const ch
if (count >= max_tokens) {
return count;
}
tokens[count] = strdup(text);
tokens[count] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, text);
return tokens[count] ? count + SKIP_ONE : count;
}
@@ -357,7 +360,7 @@ static int json_str_array(const char *json, const char *key, char **out, int max
break;
}
int len = (int)(end - start);
out[count] = malloc((size_t)len + SKIP_ONE);
out[count] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)len + SKIP_ONE);
memcpy(out[count], start, (size_t)len);
out[count][len] = '\0';
count++;
@@ -398,7 +401,7 @@ static int tokenize_json_array_field(const char *json, const char *key, char **t
if (count < max_tokens) {
count += cbm_sem_tokenize(arr[p], tokens + count, max_tokens - count);
}
free(arr[p]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, arr[p]);
}
return count;
}
@@ -428,7 +431,7 @@ static const char **collect_sorted_call_neighbors(const cbm_gbuf_t *gbuf, int64_
if (rc != 0 || ec <= 0) {
return NULL;
}
const char **names = malloc((size_t)ec * sizeof(char *));
const char **names = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)ec * sizeof(char *));
if (!names) {
return NULL;
}
@@ -458,7 +461,7 @@ static int tokenize_call_neighbors(const cbm_gbuf_node_t *n, const cbm_gbuf_t *g
for (int e = 0; e < nn && e < MAX_CALLEES && count < max_tokens; e++) {
count += cbm_sem_tokenize(names[e], tokens + count, max_tokens - count);
}
free((void *)names);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names);
return count;
}
@@ -511,7 +514,7 @@ static void build_api_vec(const cbm_gbuf_t *gbuf, int64_t node_id, cbm_sem_vec_t
cbm_sem_random_index(names[i], &callee_ri);
cbm_sem_vec_add_scaled(out, &callee_ri, PSE_UNIT_POS);
}
free((void *)names);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)names);
cbm_sem_normalize(out);
}
@@ -533,7 +536,7 @@ static void build_type_vec(const char *props_json, cbm_sem_vec_t *out) {
cbm_sem_vec_t ri;
cbm_sem_random_index(ptypes[i], &ri);
cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS);
free(ptypes[i]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, ptypes[i]);
}
cbm_sem_normalize(out);
}
@@ -549,7 +552,7 @@ static void build_deco_vec(const char *props_json, cbm_sem_vec_t *out) {
cbm_sem_vec_t ri;
cbm_sem_random_index(decos[i], &ri);
cbm_sem_vec_add_scaled(out, &ri, PSE_UNIT_POS);
free(decos[i]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, decos[i]);
}
cbm_sem_normalize(out);
}
@@ -594,10 +597,41 @@ static void decode_minhash(const char *props_json, cbm_sem_func_t *func) {
/* ── Parallel Phase 2: Tokenize nodes ────────────────────────────── */
/* One growable token-pointer buffer per worker. A function's tokens are
* appended contiguously; where they landed is recorded per function so a
* single pass after the parallel phase can pack everything into one array
* with offsets. This replaces a fixed CBM_SEM_MAX_TOKENS-slot stride per
* function (4 KB each, 7.4 GB on the kernel). */
typedef struct {
char **items;
size_t count;
size_t cap;
} tok_buf_t;
static bool tok_buf_append(tok_buf_t *b, char **tokens, int count) {
if (b->count + (size_t)count > b->cap) {
size_t new_cap = b->cap ? b->cap * 2 : (size_t)CBM_SZ_4K;
while (new_cap < b->count + (size_t)count) {
new_cap *= 2;
}
char **grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, b->items, new_cap * sizeof(char *));
if (!grown) {
return false;
}
b->items = grown;
b->cap = new_cap;
}
memcpy(b->items + b->count, tokens, (size_t)count * sizeof(char *));
b->count += (size_t)count;
return true;
}
typedef struct {
const cbm_gbuf_node_t **node_ptrs; /* node pointer per function index */
cbm_gbuf_t *gbuf; /* read-only during tokenization */
char **all_tokens; /* output: all_tokens[f * MAX + t] */
tok_buf_t *bufs; /* per worker: appended token pointers */
int *tok_worker; /* per function: which worker's buffer */
size_t *tok_off; /* per function: offset inside that buffer */
int *token_counts; /* output: token count per function */
int func_count;
_Atomic int next_idx;
@@ -617,11 +651,10 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) {
}
const cbm_gbuf_node_t *n = tc->node_ptrs[f];
/* Write directly into the shared buffer slice for this function — the
* strdup'd tokens are owned by all_tokens[] from the moment they land
* in this slot, which avoids a spurious analyzer "leak" diagnostic on
* the previous stack-local relay pattern. */
char **dst = &tc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS];
/* Tokenize into a per-call scratch slice, intern, then append the
* surviving pointers to this worker's buffer. The strings are owned by
* the worker's intern pool (or by the buffer until interned). */
char *dst[CBM_SEM_MAX_TOKENS];
int count = tokenize_node(n, tc->gbuf, dst, CBM_SEM_MAX_TOKENS);
count = inject_pattern_tokens(n, tc->gbuf, dst, count, CBM_SEM_MAX_TOKENS);
if (tc->pools && tc->pools[worker_id]) {
@@ -629,13 +662,19 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) {
for (int t = 0; t < count; t++) {
char *canon = cbm_ht_get(pool, dst[t]);
if (canon) {
free(dst[t]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, dst[t]);
dst[t] = canon;
} else {
cbm_ht_set(pool, dst[t], dst[t]); /* key borrows the value */
}
}
}
tok_buf_t *b = &tc->bufs[worker_id];
tc->tok_worker[f] = worker_id;
tc->tok_off[f] = b->count;
if (!tok_buf_append(b, dst, count)) {
count = 0; /* OOM: the function contributes no tokens, never garbage */
}
tc->token_counts[f] = count;
}
}
@@ -645,6 +684,7 @@ static void tokenize_worker(int worker_id, void *ctx_ptr) {
typedef struct {
cbm_sem_func_t *funcs;
char **all_tokens;
const size_t *offsets; /* function f = all_tokens[offsets[f] ..] */
int *token_counts;
cbm_sem_corpus_t *corpus;
uint8_t *qvecs; /* output: pre-quantized int8 vectors [func_count * CBM_SEM_DIM] */
@@ -662,11 +702,11 @@ static void vec_build_worker(int worker_id, void *ctx_ptr) {
}
int tc = vc->token_counts[f];
char **tokens = &vc->all_tokens[(ptrdiff_t)f * CBM_SEM_MAX_TOKENS];
char **tokens = &vc->all_tokens[vc->offsets[f]];
/* TF-IDF weights */
int *indices = malloc((size_t)tc * sizeof(int));
float *weights = malloc((size_t)tc * sizeof(float));
int *indices = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(int));
float *weights = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)tc * sizeof(float));
int tfidf_len = 0;
for (int t = 0; t < tc; t++) {
float idf = cbm_sem_corpus_idf(vc->corpus, tokens[t]);
@@ -964,13 +1004,14 @@ static int phase1_scan_functions(cbm_gbuf_t *gbuf, cbm_sem_func_t **out_funcs,
for (int i = 0; i < node_count; i++) {
if (func_count >= func_cap) {
int new_cap = func_cap < MAX_FUNCS_INIT ? MAX_FUNCS_INIT : func_cap * GROW;
cbm_sem_func_t *grown = realloc(funcs, (size_t)new_cap * sizeof(cbm_sem_func_t));
cbm_sem_func_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, funcs,
(size_t)new_cap * sizeof(cbm_sem_func_t));
if (!grown) {
break;
}
funcs = grown;
const cbm_gbuf_node_t **np_grown =
realloc(node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *));
const cbm_gbuf_node_t **np_grown = cbm_realloc(
CBM_MEM_CLASS_SEMANTIC, node_ptrs, (size_t)new_cap * sizeof(cbm_gbuf_node_t *));
if (!np_grown) {
break;
}
@@ -1023,7 +1064,8 @@ static void phase5c_build_lsh_buckets(const uint64_t *signatures, int func_count
if (bucket->count >= bucket->cap) {
int nc =
bucket->cap < SEM_BUCKET_CAP_INIT ? SEM_BUCKET_CAP_INIT : bucket->cap * GROW;
int *ni = realloc(bucket->items, (size_t)nc * sizeof(int));
int *ni =
cbm_realloc(CBM_MEM_CLASS_SEMANTIC, bucket->items, (size_t)nc * sizeof(int));
if (!ni) {
continue;
}
@@ -1065,7 +1107,7 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
}
deferred_edge_t *pairs = NULL;
if (total_pairs > 0) {
pairs = malloc((size_t)total_pairs * sizeof(deferred_edge_t));
pairs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)total_pairs * sizeof(deferred_edge_t));
}
if (!pairs) {
for (int w = 0; w < worker_count; w++) {
@@ -1096,7 +1138,7 @@ static int phase6b_merge_edges(cbm_gbuf_t *gbuf, deferred_edge_buf_t *worker_buf
edge_counts[de->j]++;
total_edges++;
}
free(pairs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, pairs);
return total_edges;
}
@@ -1130,7 +1172,8 @@ static void phase3c_export_token_vectors(cbm_gbuf_t *gbuf, cbm_sem_corpus_t *cor
/* Phase 5a: generate NUM_HYPERPLANES × CBM_SEM_DIM deterministic random
* float hyperplanes seeded from XXH3 so signatures are reproducible. */
static hyperplane_row_t *phase5a_build_hyperplanes(void) {
hyperplane_row_t *hyperplanes = malloc(sizeof(hyperplane_row_t) * NUM_HYPERPLANES);
hyperplane_row_t *hyperplanes =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, sizeof(hyperplane_row_t) * NUM_HYPERPLANES);
if (!hyperplanes) {
return NULL;
}
@@ -1164,13 +1207,31 @@ static void phase1b_decode_and_build(cbm_sem_func_t *funcs, const cbm_gbuf_node_
/* Phase 2: tokenize each function's metadata in parallel, filling
* all_tokens[] and token_counts[]. Caller allocates the arrays. */
static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char **all_tokens,
int *token_counts, int func_count, int worker_count,
CBMHashTable **pools) {
static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, char ***out_tokens,
size_t **out_offsets, int *token_counts, int func_count,
int worker_count, CBMHashTable **pools) {
*out_tokens = NULL;
*out_offsets = NULL;
tok_buf_t *bufs = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)worker_count * sizeof(tok_buf_t));
int *tok_worker = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(int));
size_t *tok_off = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(size_t));
size_t *offsets = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * sizeof(size_t));
if (!bufs || !tok_worker || !tok_off || !offsets) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_worker);
cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_off);
cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets);
for (int f = 0; f < func_count; f++) {
token_counts[f] = 0;
}
return;
}
tokenize_ctx_t tc = {
.node_ptrs = node_ptrs,
.gbuf = gbuf,
.all_tokens = all_tokens,
.bufs = bufs,
.tok_worker = tok_worker,
.tok_off = tok_off,
.token_counts = token_counts,
.func_count = func_count,
.pools = pools,
@@ -1178,22 +1239,60 @@ static void phase2_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf,
atomic_init(&tc.next_idx, 0);
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, tokenize_worker, &tc, opts);
/* Pack: one array, functions addressed by offset. Worker w's buffer lands
* at base[w]; a function's tokens sit at base[worker] + its offset. */
size_t total = 0;
for (int w = 0; w < worker_count; w++) {
total += bufs[w].count;
}
char **packed = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (total ? total : 1) * sizeof(char *));
if (packed) {
size_t base = 0;
for (int w = 0; w < worker_count; w++) {
if (bufs[w].count) {
memcpy(packed + base, bufs[w].items, bufs[w].count * sizeof(*bufs[w].items));
}
/* Copied: give the worker buffer back now, so the packing transient
* is the packed array plus ONE worker buffer, not plus all of them. */
cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs[w].items);
bufs[w].items = NULL;
bufs[w].cap = base; /* reuse: base offset of this worker's tokens */
base += bufs[w].count;
}
for (int f = 0; f < func_count; f++) {
offsets[f] = bufs[tok_worker[f]].cap + tok_off[f];
}
} else {
for (int f = 0; f < func_count; f++) {
token_counts[f] = 0;
}
}
for (int w = 0; w < worker_count; w++) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs[w].items);
}
cbm_free(CBM_MEM_CLASS_SEMANTIC, bufs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_worker);
cbm_free(CBM_MEM_CLASS_SEMANTIC, tok_off);
*out_tokens = packed;
*out_offsets = offsets;
}
/* Phase 4a: build per-function TF-IDF + RI vectors in parallel, producing
* int8-quantized qvecs for subsequent storage. Phase 4b runs sequentially
* to store them in gbuf because gbuf is not thread-safe. */
static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs,
char **all_tokens, int *token_counts,
cbm_sem_corpus_t *corpus, int func_count,
int worker_count) {
uint8_t *qvecs = malloc((size_t)func_count * CBM_SEM_DIM);
char **all_tokens, const size_t *offsets,
int *token_counts, cbm_sem_corpus_t *corpus,
int func_count, int worker_count) {
uint8_t *qvecs = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)func_count * CBM_SEM_DIM);
if (!qvecs) {
return;
}
vec_build_ctx_t vc = {
.funcs = funcs,
.all_tokens = all_tokens,
.offsets = offsets,
.token_counts = token_counts,
.corpus = corpus,
.qvecs = qvecs,
@@ -1206,7 +1305,7 @@ static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *fun
cbm_gbuf_store_vector(gbuf, funcs[f].node_id, &qvecs[(ptrdiff_t)f * CBM_SEM_DIM],
CBM_SEM_DIM);
}
free(qvecs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, qvecs);
}
/* Phase 5: hyperplane generation → signatures → LSH bucket population.
@@ -1215,7 +1314,8 @@ static void phase4_build_and_store_vectors(cbm_gbuf_t *gbuf, cbm_sem_func_t *fun
static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_count,
uint64_t **out_signatures, sem_bucket_t ***out_buckets) {
hyperplane_row_t *hyperplanes = phase5a_build_hyperplanes();
uint64_t *signatures = calloc((size_t)func_count, sizeof(uint64_t));
uint64_t *signatures =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(uint64_t)));
if (hyperplanes && signatures) {
sig_build_ctx_t sc = {
.funcs = funcs,
@@ -1227,12 +1327,14 @@ static void phase5_lsh_build(cbm_sem_func_t *funcs, int func_count, int worker_c
cbm_parallel_for_opts_t opts = {.max_workers = worker_count, .force_pthreads = false};
cbm_parallel_for(worker_count, sig_build_worker, &sc, opts);
}
free(hyperplanes);
cbm_free(CBM_MEM_CLASS_SEMANTIC, hyperplanes);
sem_bucket_t **band_buckets = calloc(SEM_LSH_BANDS, sizeof(sem_bucket_t *));
sem_bucket_t **band_buckets =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SEM_LSH_BANDS) * (sizeof(sem_bucket_t *)));
if (band_buckets) {
for (int b = 0; b < SEM_LSH_BANDS; b++) {
band_buckets[b] = calloc(SEM_BUCKET_COUNT, sizeof(sem_bucket_t));
band_buckets[b] = cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
(size_t)(SEM_BUCKET_COUNT) * (sizeof(sem_bucket_t)));
}
phase5c_build_lsh_buckets(signatures, func_count, band_buckets);
}
@@ -1270,21 +1372,22 @@ static void free_lsh_buckets(sem_bucket_t **band_buckets) {
continue;
}
for (int h = 0; h < SEM_BUCKET_COUNT; h++) {
free(band_buckets[b][h].items);
cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets[b][h].items);
}
free(band_buckets[b]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets[b]);
}
free(band_buckets);
cbm_free(CBM_MEM_CLASS_SEMANTIC, band_buckets);
}
/* Phases 3a/3b/3c bundled: create corpus, batch-add docs, finalize, export
* enriched token vectors to the graph buffer. Returns the new corpus, which
* the caller must cbm_sem_corpus_free() later. */
static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, int *token_counts,
static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens,
const size_t *offsets, int *token_counts,
int func_count) {
CBM_PROF_START(t_phase3a);
cbm_sem_corpus_t *corpus = cbm_sem_corpus_new();
cbm_sem_corpus_add_docs_batch(corpus, all_tokens, token_counts, func_count, CBM_SEM_MAX_TOKENS);
cbm_sem_corpus_add_docs_batch(corpus, all_tokens, offsets, token_counts, func_count);
CBM_PROF_END_N("semantic_edges", "3a_corpus_batch", t_phase3a, func_count);
CBM_PROF_START(t_phase3b);
@@ -1304,11 +1407,13 @@ static cbm_sem_corpus_t *run_corpus_phase(cbm_gbuf_t *gbuf, char **all_tokens, i
static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *signatures,
sem_bucket_t **band_buckets, cbm_sem_config_t cfg, int func_count,
int worker_count) {
int *edge_counts = calloc((size_t)func_count, sizeof(int));
deferred_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(deferred_edge_buf_t));
int *edge_counts =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(int)));
deferred_edge_buf_t *worker_bufs = cbm_calloc(
CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) * (sizeof(deferred_edge_buf_t)));
if (!edge_counts || !worker_bufs) {
free(edge_counts);
free(worker_bufs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs);
return 0;
}
for (int w = 0; w < worker_count; w++) {
@@ -1324,8 +1429,8 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *
int total = phase6b_merge_edges(gbuf, worker_bufs, worker_count, edge_counts, cfg.max_edges);
CBM_PROF_END_N("semantic_edges", "6b_edge_merge_seq", t_phase6b, total);
free(worker_bufs);
free(edge_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts);
return total;
}
@@ -1333,20 +1438,22 @@ static int run_scoring_phase(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs, uint64_t *
static void free_token_pool_entry(const char *key, void *value, void *ud) {
(void)key; /* key == value: one owned string per unique token */
(void)ud;
free(value);
cbm_free(CBM_MEM_CLASS_SEMANTIC, value);
}
/* all_tokens slots BORROW their strings from the per-worker intern pools;
* the pools own exactly one copy per unique token per worker. */
static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char **all_tokens,
const int *token_counts, CBMHashTable **pools, int worker_count) {
size_t *offsets, const int *token_counts, CBMHashTable **pools,
int worker_count) {
(void)token_counts;
cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets);
for (int f = 0; f < func_count; f++) {
free(funcs[f].tfidf_indices);
free(funcs[f].tfidf_weights);
cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs[f].tfidf_indices);
cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs[f].tfidf_weights);
}
free(all_tokens);
free(funcs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, all_tokens);
cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs);
if (pools) {
for (int w = 0; w < worker_count; w++) {
if (pools[w]) {
@@ -1354,10 +1461,153 @@ static void free_funcs_and_tokens(cbm_sem_func_t *funcs, int func_count, char **
cbm_ht_free(pools[w]);
}
}
free(pools);
cbm_free(CBM_MEM_CLASS_SEMANTIC, pools);
}
}
/* ── Memory checkpoints inside the pass (CBM_MEM_PHASES=1) ─────────── */
/* The pass is one phase mark to the pipeline; its transient lives between
* marks. These lines put the semantic class's live/peak bytes and the charged
* footprint at every sub-phase boundary, so the peak is attributable. */
static void sem_mem_mark(const char *step) {
if (!cbm_mem_phases_enabled()) {
return;
}
enum { MB = 1024 * 1024 };
char live[CBM_SZ_16];
char peak[CBM_SZ_16];
char charged[CBM_SZ_16];
snprintf(live, sizeof(live), "%zu", cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC) / MB);
snprintf(peak, sizeof(peak), "%zu", cbm_mem_class_peak_bytes(CBM_MEM_CLASS_SEMANTIC) / MB);
snprintf(charged, sizeof(charged), "%zu", cbm_mem_charged() / MB);
cbm_log_info("mem.semantic.step", "step", step, "sem_live_mb", live, "sem_peak_mb", peak,
"charged_mb", charged);
}
/* ── Headroom-sized batches ───────────────────────────────────────── */
/* Phases 2-4 hold, per function, its token pointers, the corpus's per-doc
* token ids and the quantized vector being stored: ~3 KB per function on the
* kernel (783k functions -> the 4.9 GB transient the 15 GB budget run
* overshot with, 2026-09-13). The retained part -- the funcs array and the
* corpus entries -- is what scoring needs whole and stays. When the charged
* footprint plus that transient would cross the budget, the token phases run
* per batch of functions: tokenize -> count into the corpus -> free, then,
* after finalize, tokenize again -> vectorize -> store -> free. Tokenization
* is deterministic, so both passes see the same tokens and the graph does
* not change; the run pays with the second tokenize. */
enum {
SEM_BATCH_BYTES_PER_FUNC = 4096, /* measured 3 KB, rounded up */
SEM_BATCH_MIN_FUNCS = 4096,
};
static int sem_batch_size(int func_count) {
/* CBM_SEM_BATCH=<n> forces the batch size regardless of budget (the
* batched-equals-unbatched test, small-machine iteration). */
char forced_buf[CBM_SZ_16];
if (cbm_safe_getenv("CBM_SEM_BATCH", forced_buf, sizeof(forced_buf), NULL)) {
int forced = atoi(forced_buf);
if (forced > 0 && forced < func_count) {
cbm_log_info("mem.semantic.batches", "functions", itoa_log(func_count), "batch",
itoa_log(forced), "reason", "env");
return forced;
}
}
size_t budget = cbm_mem_budget();
if (budget == 0 || func_count <= SEM_BATCH_MIN_FUNCS) {
return func_count;
}
size_t charged = cbm_mem_charged();
size_t headroom = budget > charged ? budget - charged : 0;
size_t transient = (size_t)func_count * SEM_BATCH_BYTES_PER_FUNC;
/* Half the headroom is the transient's share; the other half is what the
* later phases (signatures, buckets, deferred edges) and mimalloc keep. */
size_t share = headroom / 2;
if (transient <= share) {
return func_count;
}
size_t batch = share / SEM_BATCH_BYTES_PER_FUNC;
if (batch < SEM_BATCH_MIN_FUNCS) {
batch = SEM_BATCH_MIN_FUNCS;
}
if (batch > (size_t)func_count) {
batch = (size_t)func_count;
}
enum { MB = 1024 * 1024 };
cbm_log_info("mem.semantic.batches", "functions", itoa_log(func_count), "batch",
itoa_log((int)batch), "headroom_mb", itoa_log((int)(headroom / MB)),
"transient_mb", itoa_log((int)(transient / MB)));
return (int)batch;
}
/* One batch of the token phases: tokenize functions [first, first+count) into
* a fresh packed array the caller frees with sem_batch_free_tokens(). */
static void sem_batch_tokenize(const cbm_gbuf_node_t **node_ptrs, cbm_gbuf_t *gbuf, int first,
int count, int worker_count, CBMHashTable **pools, int *token_counts,
char ***out_tokens, size_t **out_offsets) {
phase2_tokenize(node_ptrs + first, gbuf, out_tokens, out_offsets, token_counts + first, count,
worker_count, pools);
}
static void sem_batch_free_tokens(char **tokens, size_t *offsets) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, tokens);
cbm_free(CBM_MEM_CLASS_SEMANTIC, offsets);
}
/* Phases 2-4 in headroom-sized batches. The corpus is fed batch by batch
* (its per-doc token ids are ints, kept for co-occurrence), finalized once,
* then vectors are built batch by batch from a second tokenization. */
static cbm_sem_corpus_t *run_token_phases_batched(cbm_gbuf_t *gbuf, cbm_sem_func_t *funcs,
const cbm_gbuf_node_t **node_ptrs,
int *token_counts, int func_count, int batch,
int worker_count, CBMHashTable **token_pools) {
CBM_PROF_START(t_phase2);
cbm_sem_corpus_t *corpus = cbm_sem_corpus_new();
for (int first = 0; first < func_count; first += batch) {
int count = func_count - first < batch ? func_count - first : batch;
char **tokens = NULL;
size_t *offsets = NULL;
sem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts,
&tokens, &offsets);
if (tokens && offsets) {
cbm_sem_corpus_add_docs_batch(corpus, tokens, offsets, token_counts + first, count);
}
sem_batch_free_tokens(tokens, offsets);
cbm_mem_release_to_os(); /* the batch's pages leave the charge now, not at the mark */
}
CBM_PROF_END_N("semantic_edges", "2+3a_tokenize_count_batched", t_phase2, func_count);
sem_mem_mark("3a_corpus_batched");
CBM_PROF_START(t_phase3b);
cbm_sem_corpus_finalize(corpus);
CBM_PROF_END_N("semantic_edges", "3b_corpus_finalize_seq", t_phase3b,
cbm_sem_corpus_token_count(corpus));
CBM_PROF_START(t_phase3c);
phase3c_export_token_vectors(gbuf, corpus);
CBM_PROF_END_N("semantic_edges", "3c_token_vec_export_seq", t_phase3c,
cbm_sem_corpus_token_count(corpus));
sem_mem_mark("3c_export");
CBM_PROF_START(t_phase4);
for (int first = 0; first < func_count; first += batch) {
int count = func_count - first < batch ? func_count - first : batch;
char **tokens = NULL;
size_t *offsets = NULL;
sem_batch_tokenize(node_ptrs, gbuf, first, count, worker_count, token_pools, token_counts,
&tokens, &offsets);
if (tokens && offsets) {
phase4_build_and_store_vectors(gbuf, funcs + first, tokens, offsets,
token_counts + first, corpus, count, worker_count);
}
sem_batch_free_tokens(tokens, offsets);
cbm_mem_release_to_os();
}
CBM_PROF_END_N("semantic_edges", "2+4_tokenize_vectorize_batched", t_phase4, func_count);
sem_mem_mark("4_vectors");
return corpus;
}
/* ── Pass entry point ────────────────────────────────────────────── */
int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
@@ -1381,37 +1631,53 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
cbm_log_info("pass.semantic.collected", "functions", itoa_log(func_count));
if (func_count < PSE_MIN_FUNCS_FOR_PAIR) {
free(funcs);
free(node_ptrs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, funcs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs);
cbm_log_info("pass.done", "pass", "semantic_edges", "edges", "0");
return 0;
}
sem_mem_mark("1b_decode_build");
/* Phase 2: Tokenize all nodes (PARALLEL) */
int worker_count = cbm_default_worker_count(false);
char **all_tokens = malloc((size_t)func_count * sizeof(char *) * CBM_SEM_MAX_TOKENS);
int *token_counts = calloc((size_t)func_count, sizeof(int));
char **all_tokens = NULL; /* packed by phase2_tokenize */
size_t *tok_offsets = NULL; /* per function: index into all_tokens */
int *token_counts =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)func_count) * (sizeof(int)));
CBM_PROF_START(t_phase2);
CBMHashTable **token_pools = calloc((size_t)worker_count, sizeof(CBMHashTable *));
CBMHashTable **token_pools = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) *
(sizeof(CBMHashTable *)));
if (token_pools) {
for (int w = 0; w < worker_count; w++) {
token_pools[w] = cbm_ht_create(CBM_SZ_1K);
}
}
phase2_tokenize(node_ptrs, gbuf, all_tokens, token_counts, func_count, worker_count,
token_pools);
CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count);
free(node_ptrs);
cbm_sem_corpus_t *corpus = NULL;
int batch = sem_batch_size(func_count);
if (batch < func_count) {
corpus = run_token_phases_batched(gbuf, funcs, node_ptrs, token_counts, func_count, batch,
worker_count, token_pools);
cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs);
} else {
CBM_PROF_START(t_phase2);
phase2_tokenize(node_ptrs, gbuf, &all_tokens, &tok_offsets, token_counts, func_count,
worker_count, token_pools);
CBM_PROF_END_N("semantic_edges", "2_tokenize_parallel", t_phase2, func_count);
cbm_free(CBM_MEM_CLASS_SEMANTIC, node_ptrs);
sem_mem_mark("2_tokenize");
/* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */
cbm_sem_corpus_t *corpus = run_corpus_phase(gbuf, all_tokens, token_counts, func_count);
/* Phase 3: Build corpus (batch add), finalize, export enriched token vectors. */
corpus = run_corpus_phase(gbuf, all_tokens, tok_offsets, token_counts, func_count);
sem_mem_mark("3_corpus");
/* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */
CBM_PROF_START(t_phase4);
phase4_build_and_store_vectors(gbuf, funcs, all_tokens, token_counts, corpus, func_count,
worker_count);
CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count);
/* Phase 4: Build per-function TF-IDF + RI vectors (PARALLEL) and store them. */
CBM_PROF_START(t_phase4);
phase4_build_and_store_vectors(gbuf, funcs, all_tokens, tok_offsets, token_counts, corpus,
func_count, worker_count);
CBM_PROF_END_N("semantic_edges", "4_build_and_store_vec", t_phase4, func_count);
sem_mem_mark("4_vectors");
}
cbm_log_info("pass.semantic.vectors_stored", "count", itoa_log(func_count));
@@ -1424,18 +1690,22 @@ int cbm_pipeline_pass_semantic_edges(cbm_pipeline_ctx_t *ctx) {
cbm_log_info("pass.semantic.lsh_built", "functions", itoa_log(func_count), "bands",
itoa_log(SEM_LSH_BANDS));
sem_mem_mark("5_lsh");
/* Phase 6: Parallel scoring + sequential edge merge. */
int total_edges =
run_scoring_phase(gbuf, funcs, signatures, band_buckets, cfg, func_count, worker_count);
sem_mem_mark("6_score");
/* Phase 7: Cleanup */
CBM_PROF_START(t_phase7);
free_lsh_buckets(band_buckets);
free(signatures);
cbm_free(CBM_MEM_CLASS_SEMANTIC, signatures);
cbm_log_info("pass.done", "pass", "semantic_edges", "edges", itoa_log(total_edges));
free_funcs_and_tokens(funcs, func_count, all_tokens, token_counts, token_pools, worker_count);
free(token_counts);
free_funcs_and_tokens(funcs, func_count, all_tokens, tok_offsets, token_counts, token_pools,
worker_count);
cbm_free(CBM_MEM_CLASS_SEMANTIC, token_counts);
cbm_sem_corpus_free(corpus);
CBM_PROF_END("semantic_edges", "7_cleanup", t_phase7);
src/pipeline/pass_similarity.c
+18 -12
View File
@@ -8,6 +8,7 @@
* Runs as a post-pass after enrichment (both full and incremental).
*/
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "pipeline/pipeline.h"
#include <stdint.h>
#include "pipeline/pipeline_internal.h"
@@ -132,7 +133,8 @@ static int collect_fp_entries(cbm_gbuf_t *gbuf, fp_entry_t **out_entries) {
}
if (count >= cap) {
int new_cap = cap < FP_ENTRY_INIT_CAP ? FP_ENTRY_INIT_CAP : cap * FP_ENTRY_GROW;
fp_entry_t *grown = realloc(entries, (size_t)new_cap * sizeof(fp_entry_t));
fp_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, entries,
(size_t)new_cap * sizeof(fp_entry_t));
if (!grown) {
break;
}
@@ -178,7 +180,8 @@ static void sim_edge_buf_push(sim_edge_buf_t *buf, int64_t src, int64_t tgt, dou
bool same_file) {
if (buf->count >= buf->cap) {
int nc = buf->cap < SIM_EDGE_INIT_CAP ? SIM_EDGE_INIT_CAP : buf->cap * SIM_EDGE_GROW;
sim_deferred_edge_t *grown = realloc(buf->edges, (size_t)nc * sizeof(sim_deferred_edge_t));
sim_deferred_edge_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, buf->edges,
(size_t)nc * sizeof(sim_deferred_edge_t));
if (!grown) {
return;
}
@@ -271,9 +274,9 @@ static int merge_sim_edges(cbm_gbuf_t *gbuf, sim_edge_buf_t *worker_bufs, int wo
cbm_gbuf_insert_edge(gbuf, de->source_id, de->target_id, "SIMILAR_TO", props);
total++;
}
free(worker_bufs[w].edges);
cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs[w].edges);
}
free(worker_bufs);
cbm_free(CBM_MEM_CLASS_SEMANTIC, worker_bufs);
return total;
}
@@ -293,7 +296,7 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) {
cbm_log_info("pass.similarity.collected", "nodes_with_fp", itoa_log(entry_count));
if (entry_count < MIN_FP_ENTRIES) {
free(entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, entries);
cbm_log_info("pass.done", "pass", "similarity", "edges", "0");
return 0;
}
@@ -301,9 +304,10 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) {
/* Phase 2: Build LSH index (sequential — cbm_lsh_insert mutates shared state) */
CBM_PROF_START(t_lsh_build);
cbm_lsh_index_t *lsh = cbm_lsh_new();
cbm_lsh_entry_t *lsh_entries = malloc((size_t)entry_count * sizeof(cbm_lsh_entry_t));
cbm_lsh_entry_t *lsh_entries =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)entry_count * sizeof(cbm_lsh_entry_t));
if (!lsh_entries) {
free(entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, entries);
cbm_lsh_free(lsh);
return CBM_NOT_FOUND;
}
@@ -325,9 +329,11 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) {
* in its own deferred buffer. Shared edge_counts is atomic.
* Final merge into gbuf is sequential (gbuf not thread-safe). */
CBM_PROF_START(t_query_emit);
_Atomic int *edge_counts = calloc((size_t)entry_count, sizeof(_Atomic int));
_Atomic int *edge_counts =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)entry_count) * (sizeof(_Atomic int)));
int worker_count = cbm_default_worker_count(false);
sim_edge_buf_t *worker_bufs = calloc((size_t)worker_count, sizeof(sim_edge_buf_t));
sim_edge_buf_t *worker_bufs = cbm_calloc(
CBM_MEM_CLASS_SEMANTIC, (size_t)((size_t)worker_count) * (sizeof(sim_edge_buf_t)));
{
sim_query_ctx_t sc = {
@@ -349,9 +355,9 @@ int cbm_pipeline_pass_similarity(cbm_pipeline_ctx_t *ctx) {
cbm_log_info("pass.done", "pass", "similarity", "edges", itoa_log(total_edges));
free(edge_counts);
free(lsh_entries);
free(entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, edge_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, lsh_entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, entries);
cbm_lsh_free(lsh);
return 0;
}
src/pipeline/pipeline.c
+317 -27
View File
@@ -10,6 +10,8 @@
* 6. Post-passes: tests, communities, HTTP links, git history
* 7. Dump graph buffer to SQLite
*/
#include "foundation/arena.h" // FIRST: internal/cbm/arena.h shares the CBM_ARENA_H guard and lacks cbm_arena_total
#include "foundation/constants.h"
enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 };
@@ -37,6 +39,8 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 };
#include "foundation/compat_thread.h"
#include "foundation/profile.h"
#include "foundation/mem.h"
#include "foundation/mem_core.h"
#include "result_spill.h"
#include "foundation/secure_random.h"
#include <ctype.h>
@@ -293,9 +297,26 @@ static const char *itoa_buf(int val) {
/* Log current + peak RSS at a pipeline phase boundary (memory profiling). */
static void log_phase_mem(const char *phase) {
enum { PL_BYTES_PER_MB = 1024 * 1024 };
cbm_log_info("mem.phase", "phase", phase, "rss_mb",
itoa_buf((int)(cbm_mem_rss() / PL_BYTES_PER_MB)), "peak_mb",
itoa_buf((int)(cbm_mem_peak_rss() / PL_BYTES_PER_MB)));
/* tracked_mb is what the memory core can account for; rss_mb - tracked_mb
* is the part of the process no class explains yet. */
/* itoa_buf is a 4-slot ring: two calls per line, never more. */
char rss_mb[CBM_SZ_32];
char footprint_mb[CBM_SZ_32];
char commit_mb[CBM_SZ_32];
char tracked_mb[CBM_SZ_32];
char peak_mb[CBM_SZ_32];
char peak_charged_mb[CBM_SZ_32];
snprintf(rss_mb, sizeof(rss_mb), "%zu", cbm_mem_rss() / PL_BYTES_PER_MB);
snprintf(footprint_mb, sizeof(footprint_mb), "%zu", cbm_mem_footprint() / PL_BYTES_PER_MB);
snprintf(commit_mb, sizeof(commit_mb), "%zu", cbm_mem_allocator_committed() / PL_BYTES_PER_MB);
snprintf(tracked_mb, sizeof(tracked_mb), "%zu", cbm_mem_tracked_live_bytes() / PL_BYTES_PER_MB);
snprintf(peak_mb, sizeof(peak_mb), "%zu", cbm_mem_peak_rss() / PL_BYTES_PER_MB);
(void)cbm_mem_charged(); /* fold this mark into the high-water mark */
snprintf(peak_charged_mb, sizeof(peak_charged_mb), "%zu",
cbm_mem_peak_charged() / PL_BYTES_PER_MB);
cbm_log_info("mem.phase", "phase", phase, "rss_mb", rss_mb, "footprint_mb", footprint_mb,
"commit_mb", commit_mb, "tracked_mb", tracked_mb, "peak_mb", peak_mb,
"peak_charged_mb", peak_charged_mb);
}
/* ── Lifecycle ──────────────────────────────────────────────────── */
@@ -803,19 +824,32 @@ static int process_one_infra_binding(cbm_gbuf_t *gbuf, const CBMInfraBinding *ib
return SKIP_ONE;
}
static void cbm_pipeline_process_infra_bindings(cbm_gbuf_t *gbuf, const cbm_file_info_t *files,
static bool want_infra_bindings(const CBMFileResult *header) {
return header->infra_bindings.count > 0;
}
static bool want_string_refs(const CBMFileResult *header) {
return header->string_refs.count > 0;
}
static void cbm_pipeline_process_infra_bindings(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf,
const cbm_file_info_t *files,
CBMFileResult **result_cache, int file_count) {
int bindings = 0;
for (int i = 0; i < file_count; i++) {
if (!result_cache[i]) {
bool loaded = false;
const CBMFileResult *r =
cbm_pipeline_result_acquire(ctx, result_cache, i, want_infra_bindings, &loaded);
if (!r) {
continue;
}
for (int bi = 0; bi < result_cache[i]->infra_bindings.count; bi++) {
const CBMInfraBinding *ib = &result_cache[i]->infra_bindings.items[bi];
for (int bi = 0; bi < r->infra_bindings.count; bi++) {
const CBMInfraBinding *ib = &r->infra_bindings.items[bi];
if (ib->source_name && ib->target_url) {
bindings += process_one_infra_binding(gbuf, ib, files[i].rel_path);
}
}
cbm_pipeline_result_release((CBMFileResult *)r, loaded);
}
if (bindings > 0) {
char buf[CBM_SZ_16];
@@ -928,7 +962,8 @@ static bool route_sr_denied(const CBMStringRef *sr) {
return is_upstream_config_key(sr->key_path);
}
static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_info_t *files,
static void cbm_pipeline_extract_infra_routes(const cbm_pipeline_ctx_t *ctx, cbm_gbuf_t *gbuf,
const cbm_file_info_t *files,
CBMFileResult **result_cache, int file_count) {
/* DENY-WINS-BY-VALUE: the same URL is often extracted as several string_refs
* at different key_path granularities (full path, leaf key, flat). The Route
@@ -936,29 +971,45 @@ static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_i
* per-ref guard — e.g. a denied full path `registries.terraform-registry.url`
* is defeated by a sibling leaf `url`. So pass 1 collects every URL value
* denied under ANY of its refs; pass 2 mints only values never denied. (#521) */
/* The table borrows nothing: a key is copied into `denied_keys`, because
* the result that holds sr->value is released after its file (spill
* mode loads it only for that moment) while the table spans both
* passes. A borrowed key hashed freed memory and the insert spun. */
CBMArena denied_keys;
cbm_arena_init(&denied_keys);
CBMHashTable *denied = cbm_ht_create(16);
for (int pass = 0; pass < 2; pass++) {
for (int i = 0; i < file_count; i++) {
if (!result_cache[i] || !is_infra_file(files[i].rel_path) ||
is_ci_tooling_config(files[i].rel_path)) {
if (!is_infra_file(files[i].rel_path) || is_ci_tooling_config(files[i].rel_path)) {
continue;
}
for (int si = 0; si < result_cache[i]->string_refs.count; si++) {
const CBMStringRef *sr = &result_cache[i]->string_refs.items[si];
bool loaded = false;
const CBMFileResult *r =
cbm_pipeline_result_acquire(ctx, result_cache, i, want_string_refs, &loaded);
if (!r) {
continue;
}
for (int si = 0; si < r->string_refs.count; si++) {
const CBMStringRef *sr = &r->string_refs.items[si];
if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) {
continue;
}
if (pass == 0) {
if (denied && route_sr_denied(sr)) {
cbm_ht_set(denied, sr->value, (void *)1);
const char *key = cbm_arena_strdup(&denied_keys, sr->value);
if (key && !cbm_ht_has(denied, key)) {
cbm_ht_set(denied, key, (void *)1);
}
}
} else if (!denied || !cbm_ht_has(denied, sr->value)) {
try_upsert_infra_route(gbuf, sr, files[i].rel_path);
}
}
cbm_pipeline_result_release((CBMFileResult *)r, loaded);
}
}
cbm_ht_free(denied);
cbm_arena_destroy(&denied_keys);
}
/* Run decorator_tags, configlink, and route matching passes. */
@@ -988,6 +1039,222 @@ static void predump_importance(cbm_pipeline_ctx_t *ctx) {
cbm_pipeline_pass_importance(ctx);
}
/* Phase boundary for memory attribution. Two instruments, both already in
* foundation/, both previously wired ONLY into MCP request handling and never
* into the index pipeline -- which is where the memory is (a kernel index
* peaks at 35 GB in extraction, measured 2026-09-13 with an external sampler
* because nothing in-process could say which phase it was in):
* - cbm_mem_phase_mark attributes the committed-bytes delta since the last
* mark to the phase just ended. Off unless CBM_MEM_PHASES=1.
* - cbm_mem_class_log prints the mem_core class table, so the log answers
* WHICH class grew in WHICH pass. Logs nothing until a class has activity,
* so it is silent on a tree that has not migrated yet.
* Marks must bracket the whole path with no unlabelled gaps (mem.h), hence a
* mark at every pass.timing site plus pipeline.begin at the top. */
static void pipeline_phase_mark(const char *pass) {
cbm_mem_phase_mark(pass);
/* A phase boundary is where memory is genuinely idle (the results after
* resolve, the semantic transient): hand it back before reading the
* numbers. Measured 2026-09-13 with the mimalloc-backed core: the Go worker
* floor went 15.8 -> 1.0 GB RSS. Milliseconds per phase. */
cbm_mem_release_to_os();
log_phase_mem(pass);
cbm_mem_class_log(pass);
}
/* Research census (2026-09-13): what the retained per-file results are made
* of. Records: count x sizeof per kind, plus the capacity the growable arrays
* reserved (each growth leaves the previous generation dead in the arena).
* Strings: bytes per field family, counted once per record (a pointer that
* is shared between records is charged every time it appears, so a family
* that is really shared shows up LARGER than its arena bytes -- that is the
* signal that interning would win). Measurement only. */
static size_t census_len(const char *sv) {
return sv ? strlen(sv) + SKIP_ONE : 0;
}
static size_t census_list(const char **list) {
size_t n = 0;
if (!list) {
return 0;
}
for (int i = 0; list[i]; i++) {
n += census_len(list[i]) + sizeof(char *);
}
return n + sizeof(char *);
}
static void log_result_census(const char *tag, CBMFileResult **cache, int file_count) {
enum { PL_BYTES_PER_MB = 1024 * 1024 };
size_t rec_defs = 0, rec_calls = 0, rec_usages = 0, rec_rw = 0, rec_typerefs = 0;
size_t rec_imports = 0, rec_resolved = 0, rec_other = 0;
size_t cap_defs = 0, cap_calls = 0, cap_usages = 0, cap_rw = 0, cap_typerefs = 0;
size_t cap_other = 0;
size_t n_defs = 0, n_calls = 0, n_usages = 0, n_rw = 0, n_typerefs = 0, n_resolved = 0;
size_t str_def_names = 0, str_def_sig = 0, str_def_doc = 0, str_def_tokens = 0;
size_t str_def_profile = 0, str_def_lists = 0, str_def_fp = 0, str_def_misc = 0;
size_t str_call_names = 0, str_call_enclosing = 0, str_call_args = 0;
size_t str_usage_names = 0, str_usage_enclosing = 0, str_rw = 0, str_typeref = 0;
size_t str_resolved = 0, str_source = 0, str_module = 0;
for (int i = 0; i < file_count; i++) {
const CBMFileResult *r = cache ? cache[i] : NULL;
if (!r) {
continue;
}
rec_defs += (size_t)r->defs.count * sizeof(CBMDefinition);
cap_defs += (size_t)r->defs.cap * sizeof(CBMDefinition);
rec_calls += (size_t)r->calls.count * sizeof(CBMCall);
cap_calls += (size_t)r->calls.cap * sizeof(CBMCall);
rec_usages += (size_t)r->usages.count * sizeof(CBMUsage);
cap_usages += (size_t)r->usages.cap * sizeof(CBMUsage);
rec_rw += (size_t)r->rw.count * sizeof(CBMReadWrite);
cap_rw += (size_t)r->rw.cap * sizeof(CBMReadWrite);
rec_typerefs += (size_t)r->type_refs.count * sizeof(CBMTypeRef);
cap_typerefs += (size_t)r->type_refs.cap * sizeof(CBMTypeRef);
rec_imports += (size_t)r->imports.count * sizeof(CBMImport);
rec_resolved += (size_t)r->resolved_calls.count * sizeof(CBMResolvedCall);
rec_other += (size_t)r->throws.count * sizeof(CBMThrow) +
(size_t)r->env_accesses.count * sizeof(CBMEnvAccess) +
(size_t)r->type_assigns.count * sizeof(CBMTypeAssign) +
(size_t)r->string_refs.count * sizeof(CBMStringRef) +
(size_t)r->impl_traits.count * sizeof(CBMImplTrait) +
(size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) +
(size_t)r->channels.count * sizeof(CBMChannel);
cap_other += (size_t)r->imports.cap * sizeof(CBMImport) +
(size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) +
(size_t)r->throws.cap * sizeof(CBMThrow) +
(size_t)r->env_accesses.cap * sizeof(CBMEnvAccess) +
(size_t)r->type_assigns.cap * sizeof(CBMTypeAssign) +
(size_t)r->string_refs.cap * sizeof(CBMStringRef) +
(size_t)r->impl_traits.cap * sizeof(CBMImplTrait) +
(size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) +
(size_t)r->channels.cap * sizeof(CBMChannel);
n_defs += (size_t)r->defs.count;
n_calls += (size_t)r->calls.count;
n_usages += (size_t)r->usages.count;
n_rw += (size_t)r->rw.count;
n_typerefs += (size_t)r->type_refs.count;
n_resolved += (size_t)r->resolved_calls.count;
str_source += (size_t)(r->source ? r->source_len + 1 : 0);
str_module += census_len(r->module_qn) + census_len(r->namespace_name) +
census_list(r->exports) + census_list(r->constants) +
census_list(r->global_vars) + census_list(r->macros);
for (int d = 0; d < r->defs.count; d++) {
const CBMDefinition *def = &r->defs.items[d];
str_def_names += census_len(def->name) + census_len(def->qualified_name) +
census_len(def->label) + census_len(def->file_path) +
census_len(def->parent_class);
str_def_sig += census_len(def->signature) + census_len(def->return_type) +
census_len(def->receiver);
str_def_doc += census_len(def->docstring);
str_def_tokens += census_len(def->body_tokens);
str_def_profile += census_len(def->structural_profile);
str_def_lists += census_list(def->decorators) + census_list(def->base_classes) +
census_list(def->param_names) + census_list(def->param_types) +
census_list(def->return_types);
for (int k = 0; k < def->signature_param_count; k++) {
str_def_lists += census_len(def->signature_param_types[k]) + sizeof(char *);
}
str_def_fp += def->fingerprint ? (size_t)def->fingerprint_k * sizeof(uint32_t) : 0;
str_def_misc += census_len(def->route_path) + census_len(def->route_method) +
census_len(def->impl_trait);
}
for (int c = 0; c < r->calls.count; c++) {
const CBMCall *call = &r->calls.items[c];
str_call_names += census_len(call->callee_name) + census_len(call->first_string_arg) +
census_len(call->second_arg_name);
str_call_enclosing += census_len(call->enclosing_func_qn);
for (int a = 0; a < call->arg_count && a < CBM_MAX_CALL_ARGS; a++) {
str_call_args += census_len(call->args[a].expr) + census_len(call->args[a].value) +
census_len(call->args[a].keyword);
}
}
for (int u = 0; u < r->usages.count; u++) {
str_usage_names += census_len(r->usages.items[u].ref_name);
str_usage_enclosing += census_len(r->usages.items[u].enclosing_func_qn);
}
for (int w = 0; w < r->rw.count; w++) {
str_rw +=
census_len(r->rw.items[w].var_name) + census_len(r->rw.items[w].enclosing_func_qn);
}
for (int t = 0; t < r->type_refs.count; t++) {
str_typeref += census_len(r->type_refs.items[t].type_name) +
census_len(r->type_refs.items[t].enclosing_func_qn);
}
for (int q = 0; q < r->resolved_calls.count; q++) {
const CBMResolvedCall *rc = &r->resolved_calls.items[q];
str_resolved += census_len(rc->caller_qn) + census_len(rc->callee_qn) +
census_len(rc->strategy) + census_len(rc->reason);
}
}
/* One snprintf per line: itoa_buf is a small TLS ring and a line with a
* dozen values would overwrite its own earlier fields. */
char line[CBM_SZ_1K];
#define MB(x) ((unsigned long)((x) / PL_BYTES_PER_MB))
snprintf(line, sizeof(line), "defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu resolved=%lu",
(unsigned long)n_defs, (unsigned long)n_calls, (unsigned long)n_usages,
(unsigned long)n_rw, (unsigned long)n_typerefs, (unsigned long)n_resolved);
cbm_log_info("extract.census.records", "tag", tag, "v", line);
snprintf(
line, sizeof(line),
"defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu imports=%lu resolved=%lu other=%lu",
MB(rec_defs), MB(rec_calls), MB(rec_usages), MB(rec_rw), MB(rec_typerefs), MB(rec_imports),
MB(rec_resolved), MB(rec_other));
cbm_log_info("extract.census.record_mb", "tag", tag, "v", line);
snprintf(line, sizeof(line), "defs=%lu calls=%lu usages=%lu rw=%lu type_refs=%lu other=%lu",
MB(cap_defs), MB(cap_calls), MB(cap_usages), MB(cap_rw), MB(cap_typerefs),
MB(cap_other));
cbm_log_info("extract.census.array_cap_mb", "tag", tag, "v", line);
snprintf(line, sizeof(line),
"names=%lu signature=%lu docstring=%lu body_tokens=%lu structural_profile=%lu "
"lists=%lu fingerprint=%lu misc=%lu",
MB(str_def_names), MB(str_def_sig), MB(str_def_doc), MB(str_def_tokens),
MB(str_def_profile), MB(str_def_lists), MB(str_def_fp), MB(str_def_misc));
cbm_log_info("extract.census.def_strings_mb", "tag", tag, "v", line);
snprintf(line, sizeof(line),
"call_names=%lu call_enclosing=%lu call_args=%lu usage_names=%lu "
"usage_enclosing=%lu rw=%lu type_refs=%lu resolved=%lu source=%lu module=%lu",
MB(str_call_names), MB(str_call_enclosing), MB(str_call_args), MB(str_usage_names),
MB(str_usage_enclosing), MB(str_rw), MB(str_typeref), MB(str_resolved), MB(str_source),
MB(str_module));
cbm_log_info("extract.census.ref_strings_mb", "tag", tag, "v", line);
#undef MB
}
/* The per-file result arenas are the largest retained structure of an index
* (Go corpus, 2026-09-13: 28.8 GB of arena capacity live at the end of
* extraction against 15 GB resident). Capacity is what the core charges
* (block sizes); used is what extraction wrote; trees counts results still
* holding a tree-sitter tree. The three numbers together say whether the cost
* is the data, the block-doubling headroom, or retained trees. */
static void log_result_arenas(const char *tag, CBMFileResult **cache, int file_count) {
enum { PL_BYTES_PER_MB = 1024 * 1024 };
if (!cbm_mem_phases_enabled()) {
return; /* a walk over every result: diagnostics only (CBM_MEM_PHASES=1) */
}
size_t used = 0;
size_t capacity = 0;
int results = 0;
int trees = 0;
for (int i = 0; i < file_count; i++) {
const CBMFileResult *r = cache ? cache[i] : NULL;
if (!r) {
continue;
}
results++;
used += cbm_arena_total(&r->arena);
for (int b = 0; b < r->arena.nblocks; b++) {
capacity += r->arena.block_sizes[b];
}
if (r->cached_tree) {
trees++;
}
}
cbm_log_info("extract.arenas", "tag", tag, "results", itoa_buf(results), "used_mb",
itoa_buf((int)(used / PL_BYTES_PER_MB)), "capacity_mb",
itoa_buf((int)(capacity / PL_BYTES_PER_MB)), "trees", itoa_buf(trees));
log_result_census(tag, cache, file_count);
}
/* Results are gone after resolve; so is the store. Logs what spill did. */
static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
static const struct {
predump_pass_fn fn;
@@ -1024,6 +1291,7 @@ static void run_predump_passes(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
passes[i].fn(ctx);
cbm_log_info("pass.timing", "pass", passes[i].name, "elapsed_ms",
itoa_buf((int)elapsed_ms(t)));
pipeline_phase_mark(passes[i].name);
}
}
@@ -1145,6 +1413,7 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
}
cbm_log_info("pass.timing", "pass", seq_passes[si].name, "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
pipeline_phase_mark(seq_passes[si].name);
if (check_cancel(p)) {
rc = CBM_NOT_FOUND;
}
@@ -1154,8 +1423,8 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
* one. process_one_infra_binding self-creates the topic Route node when no
* code-side dispatch created it (e.g. a standalone scheduler manifest). */
if (seq_cache && rc == 0) {
cbm_pipeline_extract_infra_routes(p->gbuf, files, seq_cache, file_count);
cbm_pipeline_process_infra_bindings(p->gbuf, files, seq_cache, file_count);
cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, seq_cache, file_count);
cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, seq_cache, file_count);
}
if (seq_cache) {
for (int i = 0; i < file_count; i++) {
@@ -1219,14 +1488,20 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
return CBM_NOT_FOUND;
}
cbm_clock_gettime(CLOCK_MONOTONIC, t);
/* This driver is the spill owner: every consumer below reads the cache
* through cbm_pipeline_result_acquire() and the store is closed here. */
ctx->spill_allowed = true;
int rc = cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count);
cbm_log_info("pass.timing", "pass", "parallel_extract", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
pipeline_phase_mark("parallel_extract");
log_result_arenas("post_extract", cache, file_count);
if (rc != 0 || check_cancel(p)) {
for (int i = 0; i < file_count; i++) {
cbm_free_result(cache[i]);
}
free(cache);
cbm_pipeline_spill_close(ctx);
return rc != 0 ? rc : CBM_NOT_FOUND;
}
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
@@ -1242,7 +1517,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
rc = cbm_build_registry_from_cache(ctx, files, file_count, cache);
cbm_log_info("pass.timing", "pass", "registry_build", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
log_phase_mem("registry_build");
pipeline_phase_mark("registry_build");
if (rc != 0 || check_cancel(p)) {
for (int i = 0; i < file_count; i++) {
if (cache[i]) {
@@ -1250,6 +1525,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
}
}
free(cache);
cbm_pipeline_spill_close(ctx);
return rc != 0 ? rc : CBM_NOT_FOUND;
}
/* Registry consumers may materialize serial nodes (Channel, EnvVar, and
@@ -1287,13 +1563,17 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
int def_count = 0;
CBMLSPDef *all_defs = NULL;
int *def_starts = NULL;
/* The collected defs own their strings in this arena (results may be on
* disk by now); the per-language cross registries share it. */
CBMArena cross_lsp_arena;
cbm_arena_init(&cross_lsp_arena);
if (run_cross_lsp) {
def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
def_starts = (int *)calloc((size_t)file_count + 1, sizeof(int));
all_defs = def_modules
? cbm_pxc_collect_all_defs(ctx, cache, files, file_count, ctx->project_name,
def_modules, &def_count, def_starts)
: NULL;
all_defs = def_modules ? cbm_pxc_collect_all_defs(ctx, &cross_lsp_arena, cache, files,
file_count, ctx->project_name,
def_modules, &def_count, def_starts)
: NULL;
}
/* Serialize per-file LSP surfaces NOW — the result cache dies with this
* pass, and the rows are what lets an incremental run detect body-only
@@ -1302,7 +1582,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
if (ctx->pipeline && all_defs && def_starts) {
cbm_lsp_surface_row_t *surface_rows = NULL;
int surface_count = 0;
if (cbm_lsp_surface_build_rows(ctx->project_name, cache, files, file_count, all_defs,
if (cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, files, file_count, all_defs,
def_starts, &surface_rows, &surface_count) == 0) {
cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count);
} else {
@@ -1323,8 +1603,6 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
* during resolve. Per-file work is then: parse + AST walk + O(1) lookups
* — no registry build, no Phase 1b mutations. Languages added so far:
* Go, Python, C/C++, C#, TS/JS, Java. Others (Kotlin, PHP) fall back to per-file. */
CBMArena cross_lsp_arena;
cbm_arena_init(&cross_lsp_arena);
CBMCrossLspRegistries cross_registries = {0};
if (all_defs) {
/* Per-builder split of lsp_cross_prepare — attributes a slow prepare to
@@ -1366,13 +1644,14 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
}
cbm_log_info("pass.timing", "pass", "lsp_cross_prepare", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
log_phase_mem("lsp_cross_prepare");
pipeline_phase_mark("lsp_cross_prepare");
cbm_clock_gettime(CLOCK_MONOTONIC, t);
rc = cbm_parallel_resolve(ctx, files, file_count, cache, &shared_ids, worker_count, all_defs,
def_count, def_modules, module_def_index, &cross_registries);
cbm_log_info("pass.timing", "pass", "parallel_resolve", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)));
log_phase_mem("parallel_resolve");
pipeline_phase_mark("parallel_resolve");
log_result_arenas("post_resolve", cache, file_count);
cbm_pxc_free_module_def_index(module_def_index);
cbm_arena_destroy(&cross_lsp_arena); /* releases all per-lang registries */
free(all_defs);
@@ -1383,20 +1662,22 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
free(def_modules);
}
cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids));
cbm_pipeline_extract_infra_routes(p->gbuf, files, cache, file_count);
cbm_pipeline_process_infra_bindings(p->gbuf, files, cache, file_count);
cbm_pipeline_extract_infra_routes(ctx, p->gbuf, files, cache, file_count);
cbm_pipeline_process_infra_bindings(ctx, p->gbuf, files, cache, file_count);
for (int i = 0; i < file_count; i++) {
if (cache[i]) {
cbm_free_result(cache[i]);
}
}
free(cache);
cbm_pipeline_spill_close(ctx);
if (rc != 0) {
return rc;
}
cbm_clock_gettime(CLOCK_MONOTONIC, t);
cbm_pipeline_pass_k8s(ctx, files, file_count);
cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(*t)));
pipeline_phase_mark("k8s");
return check_cancel(p) ? CBM_NOT_FOUND : 0;
}
@@ -2238,6 +2519,7 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
}
cbm_log_info("pass.timing", "pass", "dump_and_persist", "elapsed_ms",
itoa_buf((int)elapsed_ms(*t)), "files", itoa_buf(manifest_count));
pipeline_phase_mark("dump_and_persist");
if (p->ignored_total > p->ignored_count) {
cbm_log_warn("index.ignored_capped", "stored", itoa_buf(p->ignored_count), "total",
itoa_buf(p->ignored_total));
@@ -2302,6 +2584,7 @@ static int run_tests_and_history(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
int rc = cbm_pipeline_pass_tests(ctx, files, file_count);
CBM_PROF_END_N("pipeline", "pass_tests", t_tests, file_count);
cbm_log_info("pass.timing", "pass", "tests", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
pipeline_phase_mark("tests");
if (rc == 0 && !check_cancel(p)) {
CBM_PROF_START(t_gh);
rc = run_githistory(p, ctx);
@@ -2354,6 +2637,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
pass_structure(p, files, file_count);
CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count);
cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
pipeline_phase_mark("structure");
if (check_cancel(p)) {
return CBM_NOT_FOUND;
}
@@ -3040,6 +3324,12 @@ static void sweep_orphan_stages(const char *final_path) {
}
int cbm_pipeline_run(cbm_pipeline_t *p) {
/* Per-index attribution: peaks and phase totals are about THIS index, not
* the process history, so they start clean here. The first mark opens
* the labelled path; every pass.timing site below closes a phase. */
cbm_mem_class_reset_peaks();
cbm_mem_phase_reset();
cbm_mem_phase_mark("pipeline.begin");
if (!p) {
return CBM_NOT_FOUND;
}
src/pipeline/pipeline_incremental.c
+11 -7
View File
@@ -1285,12 +1285,13 @@ static int run_extract_resolve(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed
int fresh_count = 0;
CBMLSPDef *fresh_defs =
def_modules && def_starts
? cbm_pxc_collect_all_defs(ctx, cache, changed_files, ci, ctx->project_name,
def_modules, &fresh_count, def_starts)
? cbm_pxc_collect_all_defs(ctx, &closure->arena, cache, changed_files, ci,
ctx->project_name, def_modules, &fresh_count,
def_starts)
: NULL;
if ((fresh_defs || fresh_count == 0) && def_starts &&
cbm_lsp_surface_build_rows(ctx->project_name, cache, changed_files, ci, fresh_defs,
def_starts, &closure->fresh_rows,
cbm_lsp_surface_build_rows(ctx, ctx->project_name, cache, changed_files, ci,
fresh_defs, def_starts, &closure->fresh_rows,
&closure->fresh_count) != 0) {
closure->fresh_rows = NULL;
closure->fresh_count = 0;
@@ -1587,15 +1588,18 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,
int *def_starts = (int *)calloc((size_t)probe_count + 1, sizeof(int));
int def_count = 0;
CBMLSPDef *defs = NULL;
CBMArena probe_arena;
cbm_arena_init(&probe_arena);
if (def_modules && def_starts) {
defs = cbm_pxc_collect_all_defs(NULL, cache, probe_files, probe_count, project,
def_modules, &def_count, def_starts);
rc = cbm_lsp_surface_build_rows(project, cache, probe_files, probe_count, defs,
defs = cbm_pxc_collect_all_defs(NULL, &probe_arena, cache, probe_files, probe_count,
project, def_modules, &def_count, def_starts);
rc = cbm_lsp_surface_build_rows(NULL, project, cache, probe_files, probe_count, defs,
def_starts, out_rows, out_count);
} else {
rc = -1;
}
free(defs);
cbm_arena_destroy(&probe_arena);
free(def_starts);
if (def_modules) {
for (int i = 0; i < probe_count; i++) {
src/pipeline/pipeline_internal.h
+33
View File
@@ -136,8 +136,41 @@ typedef struct {
/* ObjectScript method-return-type table built from extracted definitions
* (NULL until pass_calls builds it). Owned by pipeline.c. */
const CBMReturnTypeTable *return_type_table;
/* Spill / admission control (2026-09-13). spill_mode latches on the first
* over-budget observation in the extract gate (or on CBM_MEM_SPILL=1):
* from then on every compacted result is parked on disk instead of held
* in the cache, results already cached are swept out, and every later
* consumer (registry build, def collection, resolve) loads a result only
* for the moment it reads it. Memory then sits at the floor -- graph +
* registries + in-flight files -- and the run pays with disk reads.
* NULL/0 = results stay in memory as always. Owned by pipeline.c. */
struct cbm_result_spill *spill;
_Atomic int spill_mode;
/* Set by the ONE owner whose every result-cache consumer goes through
* cbm_pipeline_result_acquire()/release() and that closes the store
* (run_parallel_pipeline). An owner that leaves it false never spills:
* the incremental and probe routes still hand the cache array to passes
* that index it directly, so they keep results in memory (follow-up). */
bool spill_allowed;
} cbm_pipeline_ctx_t;
/* ── Result-cache access contract (spill mode) ────────────────────────
* After extraction a slot of the result cache is either the in-memory result
* or NULL with the result parked on disk (ctx->spill). Every consumer reads a
* slot through this pair; a pass that indexes the array itself is blind to
* parked results (the infra-route passes lost every __route__infra__ node
* that way, 2026-09-13). `want` (NULL = always) sees the parked HEADER first
* -- counts are valid, pointers are not -- and can veto the load, so a pass
* after one rare list does not read every parked result back from disk. */
typedef bool (*cbm_result_want_fn)(const CBMFileResult *header);
CBMFileResult *cbm_pipeline_result_acquire(const cbm_pipeline_ctx_t *ctx, CBMFileResult **cache,
int i, cbm_result_want_fn want, bool *loaded);
void cbm_pipeline_result_release(CBMFileResult *r, bool loaded);
/* Log the store counters, close and delete the store, drop the latch. */
void cbm_pipeline_spill_close(cbm_pipeline_ctx_t *ctx);
/* Transcode an ObjectScript Studio Export XML file and compose every generated
* UDL class into one cacheable result. The returned result owns all child
* extraction arenas and is released with the ordinary cbm_free_result(). */
src/pipeline/worker_pool.c
+4
View File
@@ -14,6 +14,7 @@
enum { WP_TRUE = 1, WP_MIN = 1, WP_STEP = 1 };
#include "foundation/platform.h"
#include "foundation/compat_thread.h"
#include "foundation/mem_core.h"
#include <stdatomic.h>
#include <stdlib.h>
@@ -48,6 +49,9 @@ static void *pthread_worker(void *arg) {
}
wa->fn(idx, wa->ctx);
}
/* This thread ends here: hand its pending memory-class deltas to the
* shared counters, so the phase mark that follows the join is exact. */
cbm_mem_class_flush_thread();
return NULL;
}
src/semantic/semantic.c
+87 -61
View File
@@ -6,6 +6,7 @@
* Uses xxHash for deterministic random vectors. Pure C, zero dependencies.
*/
#include "semantic/semantic.h"
#include "foundation/mem_core.h"
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/log.h"
@@ -158,7 +159,7 @@ static bool is_camel_break(const char *name, int i) {
static void flush_token(char *buf, int *blen, char **out, int *count, int max_out) {
if (*blen > 0 && *count < max_out) {
buf[*blen] = '\0';
out[(*count)++] = strdup(buf);
out[(*count)++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, buf);
}
*blen = 0;
}
@@ -358,7 +359,7 @@ int cbm_sem_tokenize(const char *name, char **out, int max_out) {
for (int t = 0; t < orig_count && count < max_out; t++) {
for (int a = 0; abbrevs[a].abbrev; a++) {
if (strcmp(out[t], abbrevs[a].abbrev) == 0) {
out[count++] = strdup(abbrevs[a].expanded);
out[count++] = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, abbrevs[a].expanded);
break;
}
}
@@ -422,7 +423,8 @@ static void ensure_pretrained_map(void) {
const char *tok = PRETRAINED_TOKENS[i];
if (tok && tok[0]) {
snprintf(idx_buf, sizeof(idx_buf), "%d", i);
cbm_ht_set(g_pretrained_map, strdup(tok), strdup(idx_buf));
cbm_ht_set(g_pretrained_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, tok),
cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
}
}
atomic_store_explicit(&g_pretrained_ready, MAP_READY, memory_order_release);
@@ -524,7 +526,8 @@ static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) {
}
if (c->entry_count >= c->entry_cap) {
int new_cap = c->entry_cap < CORPUS_INIT_CAP ? CORPUS_INIT_CAP : c->entry_cap * PAIR_LEN;
corpus_entry_t *grown = realloc(c->entries, (size_t)new_cap * sizeof(corpus_entry_t));
corpus_entry_t *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, c->entries,
(size_t)new_cap * sizeof(corpus_entry_t));
if (!grown) {
return CBM_NOT_FOUND;
}
@@ -532,16 +535,17 @@ static int corpus_get_or_add(cbm_sem_corpus_t *c, const char *token) {
c->entry_cap = new_cap;
}
int idx = c->entry_count++;
c->entries[idx].token = strdup(token);
c->entries[idx].token = cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token);
c->entries[idx].doc_freq = 0;
memset(&c->entries[idx].enriched_vec, 0, sizeof(cbm_sem_vec_t));
snprintf(idx_buf, sizeof(idx_buf), "%d", idx);
cbm_ht_set(c->token_map, strdup(token), strdup(idx_buf));
cbm_ht_set(c->token_map, cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, token),
cbm_mem_strdup(CBM_MEM_CLASS_SEMANTIC, idx_buf));
return idx;
}
cbm_sem_corpus_t *cbm_sem_corpus_new(void) {
cbm_sem_corpus_t *c = calloc(SKIP_ONE, sizeof(cbm_sem_corpus_t));
cbm_sem_corpus_t *c = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(cbm_sem_corpus_t));
if (c) {
c->token_map = cbm_ht_create(CORPUS_INIT_CAP);
}
@@ -556,11 +560,13 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c
if (corpus->doc_count >= corpus->doc_cap) {
int new_cap =
corpus->doc_cap < DOC_TOKENS_INIT ? DOC_TOKENS_INIT : corpus->doc_cap * PAIR_LEN;
int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *));
int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int));
int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
(size_t)new_cap * sizeof(int *));
int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
(size_t)new_cap * sizeof(int));
if (!grown_ids || !grown_counts) {
free(grown_ids);
free(grown_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
return;
}
corpus->doc_token_ids = grown_ids;
@@ -568,11 +574,13 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c
corpus->doc_cap = new_cap;
}
int doc_idx = corpus->doc_count++;
corpus->doc_token_ids[doc_idx] = malloc((size_t)count * sizeof(int));
corpus->doc_token_ids[doc_idx] = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
corpus->doc_token_counts[doc_idx] = count;
/* Per-doc unique set for IDF */
int *seen = calloc((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP, sizeof(int));
int *seen =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC,
((size_t)corpus->entry_cap + (size_t)count + CORPUS_INIT_CAP) * sizeof(int));
int seen_count = 0;
for (int i = 0; i < count; i++) {
@@ -594,7 +602,7 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c
corpus->entries[tid].doc_freq++;
}
}
free(seen);
cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}
/* ── Parallel corpus batch build ──────────────────────────────────── */
@@ -612,8 +620,8 @@ void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int c
typedef struct {
cbm_sem_corpus_t *corpus;
char **all_tokens;
const size_t *offsets; /* document d = all_tokens[offsets[d] ..] */
const int *token_counts;
int max_tokens;
int doc_count;
_Atomic int *doc_freq_atomic; /* per-entry atomic counter (entry_count long) */
_Atomic int next_idx;
@@ -630,12 +638,12 @@ static void batch_resolve_one_doc(batch_resolve_ctx_t *bc, int doc_index, int *s
bc->corpus->doc_token_counts[doc_index] = 0;
return;
}
int *ids = malloc((size_t)count * sizeof(int));
int *ids = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)count * sizeof(int));
bc->corpus->doc_token_ids[doc_index] = ids;
bc->corpus->doc_token_counts[doc_index] = count;
int seen_count = 0;
char **tokens = &bc->all_tokens[(ptrdiff_t)doc_index * bc->max_tokens];
char **tokens = &bc->all_tokens[bc->offsets[doc_index]];
for (int i = 0; i < count; i++) {
const char *idx_str = cbm_ht_get(bc->corpus->token_map, tokens[i]);
int tid = CBM_NOT_FOUND;
@@ -671,7 +679,7 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
batch_resolve_ctx_t *bc = ctx_ptr;
/* Per-worker scratch for unique-per-doc tracking */
int local_seen_cap = CBM_SEM_SEEN_INIT_CAP;
int *seen = malloc((size_t)local_seen_cap * sizeof(int));
int *seen = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)local_seen_cap * sizeof(int));
if (!seen) {
return;
}
@@ -689,7 +697,7 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
for (int d = start; d < end; d++) {
int count = bc->token_counts[d];
if (count > local_seen_cap) {
int *grown = realloc(seen, (size_t)count * sizeof(int));
int *grown = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, seen, (size_t)count * sizeof(int));
if (!grown) {
continue;
}
@@ -699,12 +707,12 @@ static void batch_resolve_worker(int worker_id, void *ctx_ptr) {
batch_resolve_one_doc(bc, d, seen);
}
}
free(seen);
cbm_free(CBM_MEM_CLASS_SEMANTIC, seen);
}
void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
const int *token_counts, int doc_count, int max_tokens_per_doc) {
if (!corpus || !all_tokens || !token_counts || doc_count <= 0) {
const size_t *offsets, const int *token_counts, int doc_count) {
if (!corpus || !all_tokens || !offsets || !token_counts || doc_count <= 0) {
return;
}
@@ -712,11 +720,13 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
* Hash table mutation can't be parallelized; strdup+insert is the cost. */
if (corpus->doc_cap < corpus->doc_count + doc_count) {
int new_cap = corpus->doc_count + doc_count;
int **grown_ids = realloc(corpus->doc_token_ids, (size_t)new_cap * sizeof(int *));
int *grown_counts = realloc(corpus->doc_token_counts, (size_t)new_cap * sizeof(int));
int **grown_ids = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids,
(size_t)new_cap * sizeof(int *));
int *grown_counts = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts,
(size_t)new_cap * sizeof(int));
if (!grown_ids || !grown_counts) {
free(grown_ids);
free(grown_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, grown_counts);
return;
}
corpus->doc_token_ids = grown_ids;
@@ -728,7 +738,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
for (int d = 0; d < doc_count; d++) {
int count = token_counts[d];
char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc];
char **tokens = &all_tokens[offsets[d]];
for (int i = 0; i < count; i++) {
/* Inserts token into token_map if new; we discard return here —
* Phase B will re-lookup in read-only mode to get the ID. */
@@ -739,14 +749,15 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
/* Phase B (PARALLEL): Resolve tokens → IDs and count doc_freq per entry.
* token_map is now read-only; each worker owns its doc range (no writes
* to shared state except atomic doc_freq counters). */
_Atomic int *doc_freq_atomic = calloc((size_t)corpus->entry_count, sizeof(_Atomic int));
_Atomic int *doc_freq_atomic =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(_Atomic int));
if (!doc_freq_atomic) {
/* OOM fallback: sequential path. Roll back doc_count first since
* add_doc increments it itself. */
corpus->doc_count = base_doc;
for (int d = 0; d < doc_count; d++) {
int count = token_counts[d];
char **tokens = &all_tokens[(ptrdiff_t)d * max_tokens_per_doc];
char **tokens = &all_tokens[offsets[d]];
cbm_sem_corpus_add_doc(corpus, (const char **)tokens, count);
}
return;
@@ -757,7 +768,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
.corpus = corpus,
.all_tokens = all_tokens,
.token_counts = token_counts,
.max_tokens = max_tokens_per_doc,
.offsets = offsets,
.doc_count = doc_count,
.doc_freq_atomic = doc_freq_atomic,
};
@@ -775,7 +786,7 @@ void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
corpus->entries[i].doc_freq +=
atomic_load_explicit(&doc_freq_atomic[i], memory_order_relaxed);
}
free(doc_freq_atomic);
cbm_free(CBM_MEM_CLASS_SEMANTIC, doc_freq_atomic);
}
/* ── Parallel corpus_finalize ─────────────────────────────────────── */
@@ -1205,14 +1216,15 @@ static void pass1_quantize_worker(int worker_id, void *ctx_ptr) {
/* Build reverse index: token_id → list of (doc_id, position) pairs.
* SEQUENTIAL (fast: just pointer arithmetic + flat array fill). */
static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
reverse_index_t *rev = calloc(SKIP_ONE, sizeof(reverse_index_t));
reverse_index_t *rev = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, sizeof(reverse_index_t));
if (!rev) {
return NULL;
}
/* Phase A: count occurrences per token */
int *counts = calloc((size_t)corpus->entry_count + SKIP_ONE, sizeof(int));
int *counts =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
if (!counts) {
free(rev);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
long total = 0;
@@ -1228,10 +1240,11 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
}
}
/* Phase B: exclusive prefix sum → offsets[] */
rev->offsets = malloc(((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
rev->offsets =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, ((size_t)corpus->entry_count + SKIP_ONE) * sizeof(int));
if (!rev->offsets) {
free(counts);
free(rev);
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
int running = 0;
@@ -1242,13 +1255,13 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
}
rev->offsets[corpus->entry_count] = running;
/* Phase C: fill flat array. Ensure allocation size > 0 even for empty
* corpora (avoids malloc(0) which is implementation-defined). */
* corpora (avoids cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 0) which is implementation-defined). */
size_t flat_bytes = (total > 0 ? (size_t)total : SKIP_ONE) * sizeof(cooccur_pos_t);
rev->flat = malloc(flat_bytes);
rev->flat = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, flat_bytes);
if (!rev->flat) {
free(rev->offsets);
free(counts);
free(rev);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
return NULL;
}
for (int d = 0; d < corpus->doc_count; d++) {
@@ -1263,7 +1276,7 @@ static reverse_index_t *build_reverse_index(cbm_sem_corpus_t *corpus) {
}
}
}
free(counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, counts);
return rev;
}
@@ -1271,9 +1284,9 @@ static void free_reverse_index(reverse_index_t *rev) {
if (!rev) {
return;
}
free(rev->offsets);
free(rev->flat);
free(rev);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->offsets);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev->flat);
cbm_free(CBM_MEM_CLASS_SEMANTIC, rev);
}
/* Bundle of parameters shared by the finalize sub-phases. */
@@ -1323,7 +1336,8 @@ static void finalize_pass1(finalize_params_t *p) {
/* Sub-phases 4+5: quantize pass1 to int8, run RRI pass 2, blend + normalize. */
static void finalize_pass2(finalize_params_t *p) {
int8_t *pass1_q = malloc((size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t));
int8_t *pass1_q = cbm_alloc(CBM_MEM_CLASS_SEMANTIC,
(size_t)p->corpus->entry_count * CBM_SEM_DIM * sizeof(int8_t));
if (pass1_q) {
pass1_quant_ctx_t qc = {
.entries = p->corpus->entries,
@@ -1334,7 +1348,8 @@ static void finalize_pass2(finalize_params_t *p) {
cbm_parallel_for(p->worker_count, pass1_quantize_worker, &qc, p->opts);
}
cbm_sem_vec_t *pass1 = malloc((size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t));
cbm_sem_vec_t *pass1 =
cbm_alloc(CBM_MEM_CLASS_SEMANTIC, (size_t)p->corpus->entry_count * sizeof(cbm_sem_vec_t));
if (pass1) {
for (int i = 0; i < p->corpus->entry_count; i++) {
pass1[i] = p->corpus->entries[i].enriched_vec;
@@ -1366,15 +1381,25 @@ static void finalize_pass2(finalize_params_t *p) {
};
atomic_init(&bc.next_idx, 0);
cbm_parallel_for(p->worker_count, blend_worker, &bc, p->opts);
free(pass1);
cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1);
}
free(pass1_q);
cbm_free(CBM_MEM_CLASS_SEMANTIC, pass1_q);
norm_ctx_t nc = {.entries = p->corpus->entries, .entry_count = p->corpus->entry_count};
atomic_init(&nc.next_idx, 0);
cbm_parallel_for(p->worker_count, normalize_worker, &nc, p->opts);
}
/* The per-document token id lists exist for the co-occurrence pass only;
* released at the end of finalize so the vector phase runs without them
* (783k small blocks on the kernel). */
static void corpus_release_docs(cbm_sem_corpus_t *corpus) {
for (int d = 0; d < corpus->doc_count; d++) {
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
corpus->doc_token_ids[d] = NULL;
}
}
void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) {
if (!corpus || corpus->finalized) {
return;
@@ -1401,8 +1426,8 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) {
corpus->finalized = true;
return;
}
cbm_sem_src_entry_t *src_entries =
calloc((size_t)corpus->entry_count, sizeof(cbm_sem_src_entry_t));
cbm_sem_src_entry_t *src_entries = cbm_calloc(
CBM_MEM_CLASS_SEMANTIC, (size_t)corpus->entry_count * sizeof(cbm_sem_src_entry_t));
if (!src_entries) {
free_reverse_index(rev);
corpus->finalized = true;
@@ -1423,8 +1448,9 @@ void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus) {
finalize_pass1(&params);
finalize_pass2(&params);
free(src_entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, src_entries);
free_reverse_index(rev);
corpus_release_docs(corpus);
corpus->finalized = true;
}
@@ -1490,8 +1516,8 @@ const char *cbm_sem_corpus_token_at(const cbm_sem_corpus_t *corpus, int index,
static void free_ht_kv(const char *key, void *value, void *userdata) {
(void)userdata;
free((void *)key);
free(value);
cbm_free(CBM_MEM_CLASS_SEMANTIC, (void *)key);
cbm_free(CBM_MEM_CLASS_SEMANTIC, value);
}
void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) {
@@ -1499,19 +1525,19 @@ void cbm_sem_corpus_free(cbm_sem_corpus_t *corpus) {
return;
}
for (int i = 0; i < corpus->entry_count; i++) {
free(corpus->entries[i].token);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries[i].token);
}
free(corpus->entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->entries);
for (int d = 0; d < corpus->doc_count; d++) {
free(corpus->doc_token_ids[d]);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids[d]);
}
free(corpus->doc_token_ids);
free(corpus->doc_token_counts);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_ids);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus->doc_token_counts);
if (corpus->token_map) {
cbm_ht_foreach(corpus->token_map, free_ht_kv, NULL);
cbm_ht_free(corpus->token_map);
}
free(corpus);
cbm_free(CBM_MEM_CLASS_SEMANTIC, corpus);
}
/* ── Combined scoring ────────────────────────────────────────────── */
src/semantic/semantic.h
+6 -3
View File
@@ -24,6 +24,7 @@
#ifndef CBM_SEMANTIC_H
#define CBM_SEMANTIC_H
#include <stddef.h> /* size_t */
#include <stdbool.h>
#include <stdint.h>
@@ -166,11 +167,13 @@ cbm_sem_corpus_t *cbm_sem_corpus_new(void);
void cbm_sem_corpus_add_doc(cbm_sem_corpus_t *corpus, const char **tokens, int count);
/* Batch-build the corpus from pre-tokenized documents (PARALLEL variant).
* `all_tokens` layout: all_tokens[f * max_tokens_per_doc + t] = token pointer.
* `token_counts[f]` = number of tokens in document f.
* Document f's tokens are all_tokens[offsets[f] .. offsets[f] + token_counts[f]).
* Packed, not strided: a fixed CBM_SEM_MAX_TOKENS (512) slots per document
* was 4 KB per function up front -- 7.4 GB on the kernel for tokens that
* average a few dozen per function.
* This replaces a loop of cbm_sem_corpus_add_doc() calls. */
void cbm_sem_corpus_add_docs_batch(cbm_sem_corpus_t *corpus, char **all_tokens,
const int *token_counts, int doc_count, int max_tokens_per_doc);
const size_t *offsets, const int *token_counts, int doc_count);
/* Finalize: compute IDF, build enriched token vectors via co-occurrence. */
void cbm_sem_corpus_finalize(cbm_sem_corpus_t *corpus);
src/simhash/minhash.c
+15 -11
View File
@@ -9,6 +9,7 @@
*/
#include "simhash/minhash.h"
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "foundation/log.h"
/* Inline all xxHash functions — avoids separate compilation unit. */
#define XXH_INLINE_ALL
@@ -355,7 +356,8 @@ static uint32_t band_hash(const cbm_minhash_t *fp, int band) {
static void bucket_push(lsh_bucket_t *bucket, int entry_index) {
if (bucket->count >= bucket->cap) {
int new_cap = bucket->cap < BUCKET_INIT_CAP ? BUCKET_INIT_CAP : bucket->cap * GROW_FACTOR;
int *new_items = realloc(bucket->items, (size_t)new_cap * sizeof(int));
int *new_items =
cbm_realloc(CBM_MEM_CLASS_SEMANTIC, bucket->items, (size_t)new_cap * sizeof(int));
if (!new_items) {
return;
}
@@ -366,7 +368,8 @@ static void bucket_push(lsh_bucket_t *bucket, int entry_index) {
}
cbm_lsh_index_t *cbm_lsh_new(void) {
cbm_lsh_index_t *idx = calloc(SKIP_ONE, sizeof(cbm_lsh_index_t));
cbm_lsh_index_t *idx =
cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SKIP_ONE) * (sizeof(cbm_lsh_index_t)));
return idx;
}
@@ -379,8 +382,8 @@ void cbm_lsh_insert(cbm_lsh_index_t *idx, const cbm_lsh_entry_t *entry) {
if (idx->entry_count >= idx->entry_cap) {
int new_cap =
idx->entry_cap < ENTRY_INIT_CAP ? ENTRY_INIT_CAP : idx->entry_cap * GROW_FACTOR;
cbm_lsh_entry_t *new_entries =
realloc(idx->entries, (size_t)new_cap * sizeof(cbm_lsh_entry_t));
cbm_lsh_entry_t *new_entries = cbm_realloc(CBM_MEM_CLASS_SEMANTIC, idx->entries,
(size_t)new_cap * sizeof(cbm_lsh_entry_t));
if (!new_entries) {
return;
}
@@ -405,7 +408,7 @@ typedef struct {
} seen_set_t;
static void seen_set_init(seen_set_t *s) {
s->slots = calloc(SEEN_SET_SIZE, sizeof(int64_t));
s->slots = cbm_calloc(CBM_MEM_CLASS_SEMANTIC, (size_t)(SEEN_SET_SIZE) * (sizeof(int64_t)));
s->cap = SEEN_SET_SIZE;
/* 0 means empty — node_ids are always > 0 */
}
@@ -429,7 +432,7 @@ static bool seen_set_insert(seen_set_t *s, int64_t node_id) {
}
static void seen_set_free(seen_set_t *s) {
free(s->slots);
cbm_free(CBM_MEM_CLASS_SEMANTIC, s->slots);
s->slots = NULL;
}
@@ -439,7 +442,8 @@ static bool result_push(cbm_lsh_index_t *idx, const cbm_lsh_entry_t *candidate)
int new_cap =
idx->result_cap < RESULT_INIT_CAP ? RESULT_INIT_CAP : idx->result_cap * GROW_FACTOR;
const cbm_lsh_entry_t **new_buf =
realloc(idx->result_buf, (size_t)new_cap * sizeof(const cbm_lsh_entry_t *));
cbm_realloc(CBM_MEM_CLASS_SEMANTIC, idx->result_buf,
(size_t)new_cap * sizeof(const cbm_lsh_entry_t *));
if (!new_buf) {
return false;
}
@@ -529,10 +533,10 @@ void cbm_lsh_free(cbm_lsh_index_t *idx) {
/* Free bucket arrays */
for (int b = 0; b < CBM_LSH_BANDS; b++) {
for (int h = 0; h < LSH_BUCKET_COUNT; h++) {
free(idx->bands[b][h].items);
cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->bands[b][h].items);
}
}
free(idx->entries);
free(idx->result_buf);
free(idx);
cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->entries);
cbm_free(CBM_MEM_CLASS_SEMANTIC, idx->result_buf);
cbm_free(CBM_MEM_CLASS_SEMANTIC, idx);
}
tests/test_arena.c
+41
View File
@@ -5,6 +5,45 @@
#include "../src/foundation/arena.h"
#include <stdint.h>
TEST(arena_init_exact_one_block_then_default_growth) {
CBMArena a;
cbm_arena_init_exact(&a, 100);
ASSERT_EQ(a.nblocks, 1);
ASSERT_EQ(a.block_sizes[0], 104); /* rounded to the 8-byte allocation grain */
ASSERT_EQ(a.grow_size, CBM_ARENA_DEFAULT_BLOCK_SIZE);
void *p = cbm_arena_alloc(&a, 100);
ASSERT_NOT_NULL(p);
ASSERT_EQ(a.used, 104);
ASSERT_EQ(a.nblocks, 1); /* the exact block is full, nothing wasted */
void *q = cbm_arena_alloc(&a, 8);
ASSERT_NOT_NULL(q);
ASSERT_EQ(a.nblocks, 2);
ASSERT_EQ(a.block_sizes[1], CBM_ARENA_DEFAULT_BLOCK_SIZE); /* not 2 x 104 */
ASSERT_EQ(a.grow_size, 2 * CBM_ARENA_DEFAULT_BLOCK_SIZE);
cbm_arena_destroy(&a);
PASS();
}
TEST(arena_growth_doubles_grow_size_and_reset_restores_it) {
CBMArena a;
cbm_arena_init_sized(&a, 64);
ASSERT_EQ(a.grow_size, 128);
ASSERT_NOT_NULL(cbm_arena_alloc(&a, 64));
ASSERT_NOT_NULL(cbm_arena_alloc(&a, 8)); /* -> block 128 */
ASSERT_EQ(a.nblocks, 2);
ASSERT_EQ(a.block_sizes[1], 128);
ASSERT_EQ(a.grow_size, 256);
ASSERT_NOT_NULL(cbm_arena_alloc(&a, 1000)); /* larger than grow_size: exact */
ASSERT_EQ(a.nblocks, 3);
ASSERT_EQ(a.block_sizes[2], 1000);
ASSERT_EQ(a.grow_size, 2000);
cbm_arena_reset(&a);
ASSERT_EQ(a.nblocks, 1);
ASSERT_EQ(a.grow_size, 128);
cbm_arena_destroy(&a);
PASS();
}
TEST(arena_init_default) {
CBMArena a;
cbm_arena_init(&a);
@@ -428,6 +467,8 @@ TEST(arena_strndup_zero_len) {
}
SUITE(arena) {
RUN_TEST(arena_init_exact_one_block_then_default_growth);
RUN_TEST(arena_growth_doubles_grow_size_and_reset_restores_it);
RUN_TEST(arena_init_default);
RUN_TEST(arena_init_sized);
RUN_TEST(arena_alloc_basic);
tests/test_c_lsp.c
+163
View File
@@ -15250,6 +15250,167 @@ TEST(clsp_tier2_shared_registry_readonly_c) {
PASS();
}
/* The method-return refinement in the C++ class walk used to cast the chained
* lookup result and write a scratch-arena signature into it. With the entry in
* the sealed shared base, every other worker read that signature after the
* arena of this file had died (ASan heap-use-after-free in c_adl_resolve on
* dotnet/runtime, 2026-09-14). Contract: the refinement is copy-on-write into
* the overlay; the base signature pointer never changes. min_params = 7 is a
* marker only a COPY of the base entry carries (a fresh registration sets -1),
* so the assertion proves the upgrade path ran, not a re-registration. */
TEST(clsp_method_return_refinement_is_copy_on_write) {
CBMArena arena;
cbm_arena_init(&arena);
CBMTypeRegistry base;
cbm_registry_init(&base, &arena);
const CBMType *rets[2] = {cbm_type_named(&arena, "test.mod.Item"), NULL};
CBMRegisteredFunc f;
memset(&f, 0, sizeof(f));
f.qualified_name = "test.mod.Box.items";
f.short_name = "items";
f.receiver_type = "test.mod.Box";
f.signature = cbm_type_func(&arena, NULL, NULL, rets);
f.min_params = 7;
cbm_registry_add_func(&base, f);
cbm_registry_finalize(&base);
base.read_only = true;
const CBMRegisteredFunc *base_entry = cbm_registry_lookup_func(&base, "test.mod.Box.items");
ASSERT_NOT_NULL(base_entry);
const CBMType *base_sig = base_entry->signature;
ASSERT_EQ(base_sig->data.func.return_types[0]->kind, CBM_TYPE_NAMED);
CBMArena scratch;
cbm_arena_init(&scratch);
CBMTypeRegistry overlay;
cbm_registry_init(&overlay, &scratch);
overlay.fallback = &base;
/* The in-class declaration refines the NAMED return to a POINTER. */
const char *src = "struct Item { int v; };\n"
"struct Box {\n"
" Item *items();\n"
"};\n";
CBMResolvedCallArray out = {0};
cbm_run_c_lsp_cross_with_registry(&scratch, src, (int)strlen(src), "test.mod",
/*cpp_mode=*/true, &overlay, NULL, NULL, 0, NULL, &out);
ASSERT(base_entry->signature == base_sig);
ASSERT_EQ(base_sig->data.func.return_types[0]->kind, CBM_TYPE_NAMED);
const CBMRegisteredFunc *refined = cbm_registry_lookup_func(&overlay, "test.mod.Box.items");
ASSERT_NOT_NULL(refined);
ASSERT(refined >= overlay.funcs && refined < overlay.funcs + overlay.func_count);
ASSERT_EQ(refined->min_params, 7);
ASSERT_EQ(refined->signature->data.func.return_types[0]->kind, CBM_TYPE_POINTER);
cbm_arena_destroy(&scratch);
cbm_arena_destroy(&arena);
PASS();
}
/* The per-file overlay contract (type_registry.c chain API): a walk is handed
* an overlay chained to a sealed base. Lookups and iterators see the base
* through the overlay, every yielded index belongs to it.reg, a refinement is
* copy-on-write into the overlay (the base never changes), and an overlay copy
* shadows its base original in chained iteration. */
TEST(registry_overlay_chain_iterates_and_copies_on_write) {
CBMArena arena;
cbm_arena_init(&arena);
CBMTypeRegistry base;
cbm_registry_init(&base, &arena);
CBMRegisteredFunc f;
memset(&f, 0, sizeof(f));
f.qualified_name = "pkg.alpha";
f.short_name = "alpha";
f.min_params = -1;
cbm_registry_add_func(&base, f);
f.qualified_name = "pkg.beta";
f.short_name = "beta";
cbm_registry_add_func(&base, f);
CBMRegisteredType t;
memset(&t, 0, sizeof(t));
t.qualified_name = "pkg.T";
t.short_name = "T";
cbm_registry_add_type(&base, t);
cbm_registry_finalize(&base);
base.read_only = true;
CBMArena scratch;
cbm_arena_init(&scratch);
CBMTypeRegistry overlay;
cbm_registry_init(&overlay, &scratch);
overlay.fallback = &base;
/* Lookups chain through the empty overlay. */
ASSERT_NOT_NULL(cbm_registry_lookup_func(&overlay, "pkg.alpha"));
ASSERT_NOT_NULL(cbm_registry_lookup_type(&overlay, "pkg.T"));
/* Chained iteration reaches the base; it.reg names where the index lives. */
CBMFreeFuncIter it;
cbm_registry_free_funcs_by_short_name_chain(&overlay, "alpha", &it);
int i = cbm_free_func_iter_next(&it);
ASSERT(i >= 0);
ASSERT(it.reg == &base);
ASSERT(strcmp(it.reg->funcs[i].qualified_name, "pkg.alpha") == 0);
ASSERT_EQ(cbm_free_func_iter_next(&it), -1);
/* The plain iterator on the overlay alone sees nothing -- unchanged. */
cbm_registry_free_funcs_by_short_name(&overlay, "alpha", &it);
ASSERT_EQ(cbm_free_func_iter_next(&it), -1);
/* Copy-on-write: the refinement lands in the overlay, the base is untouched,
* and chained lookup now returns the refined copy. */
CBMRegisteredFunc *w = cbm_registry_func_for_update(&overlay, "pkg.alpha");
ASSERT_NOT_NULL(w);
ASSERT(w >= overlay.funcs && w < overlay.funcs + overlay.func_count);
w->min_params = 2;
ASSERT_EQ(base.funcs[0].min_params, -1);
ASSERT_EQ(cbm_registry_lookup_func(&overlay, "pkg.alpha")->min_params, 2);
ASSERT(cbm_registry_func_for_update(&overlay, "pkg.alpha") ==
w); /* own entry, no second copy */
ASSERT_EQ(overlay.func_count, 1);
/* The base original is shadowed: chained iteration yields exactly one alpha,
* from the overlay. */
cbm_registry_free_funcs_by_short_name_chain(&overlay, "alpha", &it);
int seen = 0;
while (cbm_free_func_iter_next(&it) >= 0) {
ASSERT(it.reg == &overlay);
seen++;
}
ASSERT_EQ(seen, 1);
/* Linear chain over everything: alpha (overlay copy) + beta (base). */
cbm_registry_all_funcs_chain(&overlay, &it);
seen = 0;
while (cbm_free_func_iter_next(&it) >= 0) {
seen++;
}
ASSERT_EQ(seen, 2);
/* Types: same contract. */
CBMTypeShortIter ti;
cbm_registry_types_by_short_name_chain(&overlay, "T", &ti);
int k = cbm_type_short_iter_next(&ti);
ASSERT(k >= 0);
ASSERT(ti.reg == &base);
CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T");
ASSERT_NOT_NULL(wt);
ASSERT_EQ(overlay.type_count, 1);
cbm_registry_all_types_chain(&overlay, &ti);
seen = 0;
while (cbm_type_short_iter_next(&ti) >= 0) {
seen++;
}
ASSERT_EQ(seen, 1);
/* A sealed head refuses refinement; an unknown QN yields nothing. */
overlay.read_only = true;
ASSERT(cbm_registry_func_for_update(&overlay, "pkg.beta") == NULL);
overlay.read_only = false;
ASSERT(cbm_registry_func_for_update(&overlay, "pkg.nope") == NULL);
ASSERT_EQ(base.func_count, 2);
cbm_arena_destroy(&scratch);
cbm_arena_destroy(&arena);
PASS();
}
/* Direct guard for the type-name / embedded-type / free-function registry
* indexes and their iterators (type_registry.c). Verifies that every iterator
* preserves ascending registry order, which is part of resolver tie-breaking. */
@@ -16216,6 +16377,8 @@ SUITE(c_lsp) {
RUN_TEST(clsp_tier2_shared_registry_readonly_c);
RUN_TEST(clsp_tier2_shared_registry_readonly_cpp);
RUN_TEST(seal_py_shared_registry_readonly);
RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write);
RUN_TEST(clsp_method_return_refinement_is_copy_on_write);
RUN_TEST(seal_py_shared_registry_readonly_fields);
RUN_TEST(seal_cs_shared_registry_readonly);
RUN_TEST(seal_ts_shared_registry_readonly);
tests/test_extraction.c
+370
View File
@@ -12,6 +12,8 @@
#include "../src/foundation/compat_fs.h"
#include <time.h>
#include "macro_table.h"
#include "result_spill.h"
#include "pipeline/pass_lsp_cross.h"
#include "iris_export_xml.h"
/* ── Helpers ───────────────────────────────────────────────────── */
@@ -7120,7 +7122,375 @@ TEST(non_config_language_module_has_no_promoted_description_issue519) {
PASS();
}
/* ── Result compaction (cbm_result_compact) ────────────────────────────── */
static const char *COMPACT_PY_SRC = "import os\n"
"from typing import List\n"
"\n"
"@app.route(\"/items\")\n"
"def list_items(limit: int, offset: int = 0) -> List[str]:\n"
" \"\"\"Return items.\"\"\"\n"
" rows = fetch(limit, offset=offset)\n"
" total = len(rows)\n"
" for r in rows:\n"
" print(r, total)\n"
" return rows\n"
"\n"
"class Store(Base):\n"
" def get(self, key):\n"
" return self.data.get(key, None)\n"
"\n"
" def put(self, key, value):\n"
" self.data[key] = value\n"
" return fetch(key, value)\n";
static bool cmp_str_eq(const char *a, const char *b) {
if (!a || !b) {
return a == b;
}
return strcmp(a, b) == 0;
}
static bool cmp_list_eq(const char **a, const char **b) {
if (!a || !b) {
return a == b;
}
int i = 0;
for (; a[i] && b[i]; i++) {
if (strcmp(a[i], b[i]) != 0) {
return false;
}
}
return a[i] == NULL && b[i] == NULL;
}
static size_t arena_capacity(const CBMArena *a) {
size_t total = 0;
for (int i = 0; i < a->nblocks; i++) {
total += a->block_sizes[i];
}
return total;
}
TEST(extract_compact_keeps_every_field_and_shrinks_the_arena) {
CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py");
CBMFileResult *ref = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(ref);
ASSERT_FALSE(r->has_error);
ASSERT(r->defs.count >= 4); /* list_items, Store, get, put (+ module) */
ASSERT(r->calls.count >= 4); /* fetch x2, len, print, get */
ASSERT(r->usages.count >= 1);
ASSERT(r->imports.count >= 2);
bool saw_args = false;
for (int i = 0; i < ref->calls.count; i++) {
saw_args = saw_args || ref->calls.items[i].arg_count > 0;
}
ASSERT(saw_args);
size_t used_before = cbm_arena_total(&r->arena);
size_t cap_before = arena_capacity(&r->arena);
cbm_result_compact(r);
/* One exact block: capacity == bytes used, no dead headroom. */
ASSERT_EQ(r->arena.nblocks, 1);
ASSERT_EQ(arena_capacity(&r->arena), cbm_arena_total(&r->arena));
ASSERT(cbm_arena_total(&r->arena) < used_before);
ASSERT(arena_capacity(&r->arena) < cap_before);
ASSERT_EQ(r->defs.cap, r->defs.count);
ASSERT_EQ(r->calls.cap, r->calls.count);
ASSERT_EQ(r->usages.cap, r->usages.count);
/* Every field survives, by value. */
ASSERT_EQ(r->defs.count, ref->defs.count);
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *a = &r->defs.items[i];
const CBMDefinition *b = &ref->defs.items[i];
ASSERT(cmp_str_eq(a->name, b->name));
ASSERT(cmp_str_eq(a->qualified_name, b->qualified_name));
ASSERT(cmp_str_eq(a->label, b->label));
ASSERT(cmp_str_eq(a->file_path, b->file_path));
ASSERT(cmp_str_eq(a->signature, b->signature));
ASSERT(cmp_str_eq(a->return_type, b->return_type));
ASSERT(cmp_str_eq(a->docstring, b->docstring));
ASSERT(cmp_str_eq(a->parent_class, b->parent_class));
ASSERT(cmp_str_eq(a->route_path, b->route_path));
ASSERT(cmp_str_eq(a->body_tokens, b->body_tokens));
ASSERT(cmp_str_eq(a->structural_profile, b->structural_profile));
ASSERT(cmp_list_eq(a->decorators, b->decorators));
ASSERT(cmp_list_eq(a->base_classes, b->base_classes));
ASSERT(cmp_list_eq(a->param_names, b->param_names));
ASSERT(cmp_list_eq(a->param_types, b->param_types));
ASSERT(cmp_list_eq(a->return_types, b->return_types));
ASSERT_EQ(a->signature_param_count, b->signature_param_count);
for (int k = 0; k < a->signature_param_count; k++) {
ASSERT(cmp_str_eq(a->signature_param_types[k], b->signature_param_types[k]));
}
ASSERT_EQ(a->start_line, b->start_line);
ASSERT_EQ(a->end_line, b->end_line);
ASSERT_EQ(a->complexity, b->complexity);
ASSERT_EQ(a->lines, b->lines);
ASSERT_EQ(a->is_exported, b->is_exported);
ASSERT_EQ(a->fingerprint_k, b->fingerprint_k);
if (a->fingerprint_k > 0) {
ASSERT_NOT_NULL(a->fingerprint);
ASSERT(memcmp(a->fingerprint, b->fingerprint,
(size_t)a->fingerprint_k * sizeof(uint32_t)) == 0);
}
}
ASSERT_EQ(r->calls.count, ref->calls.count);
for (int i = 0; i < r->calls.count; i++) {
const CBMCall *a = &r->calls.items[i];
const CBMCall *b = &ref->calls.items[i];
ASSERT(cmp_str_eq(a->callee_name, b->callee_name));
ASSERT(cmp_str_eq(a->enclosing_func_qn, b->enclosing_func_qn));
ASSERT(cmp_str_eq(a->first_string_arg, b->first_string_arg));
ASSERT_EQ(a->arg_count, b->arg_count);
ASSERT_EQ(a->start_line, b->start_line);
ASSERT_EQ(a->site_start_byte, b->site_start_byte);
ASSERT_EQ(a->site_end_byte, b->site_end_byte);
ASSERT_EQ(a->is_method, b->is_method);
for (int k = 0; k < a->arg_count; k++) {
ASSERT(cmp_str_eq(a->args[k].expr, b->args[k].expr));
ASSERT(cmp_str_eq(a->args[k].value, b->args[k].value));
ASSERT(cmp_str_eq(a->args[k].keyword, b->args[k].keyword));
ASSERT_EQ(a->args[k].index, b->args[k].index);
}
}
ASSERT_EQ(r->usages.count, ref->usages.count);
for (int i = 0; i < r->usages.count; i++) {
ASSERT(cmp_str_eq(r->usages.items[i].ref_name, ref->usages.items[i].ref_name));
ASSERT(cmp_str_eq(r->usages.items[i].enclosing_func_qn,
ref->usages.items[i].enclosing_func_qn));
ASSERT_EQ(r->usages.items[i].kind, ref->usages.items[i].kind);
ASSERT_EQ(r->usages.items[i].site_start_byte, ref->usages.items[i].site_start_byte);
ASSERT_EQ(r->usages.items[i].is_member_access, ref->usages.items[i].is_member_access);
}
ASSERT_EQ(r->imports.count, ref->imports.count);
for (int i = 0; i < r->imports.count; i++) {
ASSERT(cmp_str_eq(r->imports.items[i].local_name, ref->imports.items[i].local_name));
ASSERT(cmp_str_eq(r->imports.items[i].module_path, ref->imports.items[i].module_path));
}
ASSERT_EQ(r->rw.count, ref->rw.count);
ASSERT_EQ(r->type_refs.count, ref->type_refs.count);
ASSERT(cmp_str_eq(r->module_qn, ref->module_qn));
ASSERT(cmp_list_eq(r->exports, ref->exports));
/* Interned by content: two records with the same enclosing QN share one
* copy after compaction. */
bool shared = false;
for (int i = 0; i < r->calls.count && !shared; i++) {
for (int j = i + 1; j < r->calls.count && !shared; j++) {
if (r->calls.items[i].enclosing_func_qn && r->calls.items[j].enclosing_func_qn &&
strcmp(r->calls.items[i].enclosing_func_qn, r->calls.items[j].enclosing_func_qn) ==
0) {
shared = r->calls.items[i].enclosing_func_qn == r->calls.items[j].enclosing_func_qn;
}
}
}
ASSERT(shared);
/* The arena stays usable for the cross-file pass: growth restarts at the
* default block, never at twice the compact block. */
char *later = cbm_arena_strdup(&r->arena, "appended after compaction");
ASSERT_NOT_NULL(later);
ASSERT_EQ(r->arena.nblocks, 2);
ASSERT_EQ(r->arena.block_sizes[1], CBM_ARENA_DEFAULT_BLOCK_SIZE);
cbm_free_result(r);
cbm_free_result(ref);
PASS();
}
TEST(extract_compact_is_idempotent_and_survives_empty_results) {
CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py");
ASSERT_NOT_NULL(r);
cbm_result_compact(r);
size_t once = cbm_arena_total(&r->arena);
int defs = r->defs.count;
cbm_result_compact(r);
ASSERT_EQ(cbm_arena_total(&r->arena), once);
ASSERT_EQ(r->defs.count, defs);
ASSERT_EQ(r->arena.nblocks, 1);
cbm_free_result(r);
CBMFileResult *empty = extract("", CBM_LANG_PYTHON, "t", "empty.py");
ASSERT_NOT_NULL(empty);
cbm_result_compact(empty);
ASSERT_EQ(empty->defs.count + empty->calls.count, empty->defs.count + empty->calls.count);
ASSERT(empty->arena.nblocks >= 1);
cbm_free_result(empty);
cbm_result_compact(NULL); /* no-op */
PASS();
}
/* ── Result spill (result_spill.c): park -> load is a faithful round trip ── */
TEST(extract_spill_round_trip_keeps_every_field) {
CBMFileResult *r = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py");
CBMFileResult *ref = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "svc.py");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(ref);
cbm_result_compact(r);
cbm_result_compact(ref);
char dir[512];
snprintf(dir, sizeof(dir), "%s/cbm_spill_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(dir));
cbm_result_spill_t *sp = cbm_result_spill_open(dir, 2, 3);
ASSERT_NOT_NULL(sp);
ASSERT_FALSE(cbm_result_spill_has(sp, 1));
/* A result that is not compacted (two blocks) is refused, untouched. */
CBMFileResult *raw = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "raw.py");
ASSERT_NOT_NULL(raw);
if (raw->arena.nblocks > 1) {
ASSERT_FALSE(cbm_result_spill_park(sp, 0, 2, raw));
ASSERT_FALSE(cbm_result_spill_has(sp, 2));
}
cbm_free_result(raw);
/* Park frees the in-memory result; the slot is then on disk. Each
* precondition is named so a refusal says which one it was. */
int defs_before = r->defs.count;
int calls_before = r->calls.count;
ASSERT_EQ(r->arena.nblocks, 1);
ASSERT_EQ(r->owned_result_count, 0);
ASSERT_NOT_NULL(r->cached_tree); /* the extraction helper keeps the tree: park drops it */
ASSERT_TRUE(cbm_result_spill_park(sp, 1, 1, r));
r = NULL;
ASSERT_TRUE(cbm_result_spill_has(sp, 1));
int peek_defs = -1;
int peek_impls = -1;
cbm_result_spill_peek_counts(sp, 1, &peek_defs, &peek_impls);
ASSERT_EQ(peek_defs, defs_before);
ASSERT_EQ(peek_impls, 0);
CBMFileResult *back = cbm_result_spill_load(sp, 1);
ASSERT_NOT_NULL(back);
ASSERT_NULL(back->cached_tree);
ASSERT_EQ(back->arena.nblocks, 1);
ASSERT_EQ(back->defs.count, defs_before);
ASSERT_EQ(back->calls.count, calls_before);
for (int i = 0; i < back->defs.count; i++) {
const CBMDefinition *a = &back->defs.items[i];
const CBMDefinition *b = &ref->defs.items[i];
ASSERT(cmp_str_eq(a->name, b->name));
ASSERT(cmp_str_eq(a->qualified_name, b->qualified_name));
ASSERT(cmp_str_eq(a->label, b->label));
ASSERT(cmp_str_eq(a->file_path, b->file_path));
ASSERT(cmp_str_eq(a->signature, b->signature));
ASSERT(cmp_str_eq(a->docstring, b->docstring));
ASSERT(cmp_list_eq(a->decorators, b->decorators));
ASSERT(cmp_list_eq(a->param_names, b->param_names));
ASSERT_EQ(a->start_line, b->start_line);
ASSERT_EQ(a->fingerprint_k, b->fingerprint_k);
if (a->fingerprint_k > 0) {
ASSERT(memcmp(a->fingerprint, b->fingerprint,
(size_t)a->fingerprint_k * sizeof(uint32_t)) == 0);
}
/* Every pointer now lives in the loaded block, none in the old one. */
const char *lo = back->arena.blocks[0];
const char *hi = lo + back->arena.used;
ASSERT(a->name >= lo && a->name < hi);
ASSERT(a->qualified_name >= lo && a->qualified_name < hi);
}
for (int i = 0; i < back->calls.count; i++) {
const CBMCall *a = &back->calls.items[i];
const CBMCall *b = &ref->calls.items[i];
ASSERT(cmp_str_eq(a->callee_name, b->callee_name));
ASSERT(cmp_str_eq(a->enclosing_func_qn, b->enclosing_func_qn));
ASSERT_EQ(a->arg_count, b->arg_count);
for (int k = 0; k < a->arg_count; k++) {
ASSERT(cmp_str_eq(a->args[k].expr, b->args[k].expr));
}
}
ASSERT_EQ(back->usages.count, ref->usages.count);
for (int i = 0; i < back->usages.count; i++) {
ASSERT(cmp_str_eq(back->usages.items[i].ref_name, ref->usages.items[i].ref_name));
}
ASSERT(cmp_str_eq(back->module_qn, ref->module_qn));
ASSERT(cmp_list_eq(back->exports, ref->exports));
/* Loading twice yields two independent copies. */
CBMFileResult *again = cbm_result_spill_load(sp, 1);
ASSERT_NOT_NULL(again);
ASSERT(again->arena.blocks[0] != back->arena.blocks[0]);
ASSERT_EQ(again->defs.count, defs_before);
int64_t parked = 0;
int64_t bytes = 0;
int64_t loads = 0;
cbm_result_spill_stats(sp, &parked, &bytes, &loads);
ASSERT_EQ(parked, 1);
ASSERT(bytes > 0);
ASSERT_EQ(loads, 2);
cbm_free_result(again);
cbm_free_result(back);
cbm_free_result(ref);
cbm_result_spill_close(sp);
cbm_rmdir(dir);
PASS();
}
/* ── LSP budget share: an oversized parse disqualifies the file from the walks ── */
/* CBM_TEST_LSP_SKIP_ON names the file (no real timing): the result carries
* lsp_skipped, the per-file LSP walk did not run (no LSP-resolved calls),
* and the shared cross-file dispatcher returns without touching it. The
* unified extractor definitions are still there. */
TEST(extract_lsp_skipped_when_parse_used_its_budget_share) {
cbm_setenv("CBM_TEST_LSP_SKIP_ON", "budget_share.py", 1);
CBMFileResult *skipped = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "budget_share.py");
cbm_unsetenv("CBM_TEST_LSP_SKIP_ON");
CBMFileResult *walked = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walked.py");
ASSERT_NOT_NULL(skipped);
ASSERT_NOT_NULL(walked);
ASSERT_TRUE(skipped->lsp_skipped);
ASSERT_FALSE(walked->lsp_skipped);
ASSERT_GT(skipped->defs.count, 0); /* the unified extractor still ran */
ASSERT_TRUE(skipped->defs.count <= walked->defs.count); /* the LSP walk adds its own defs */
ASSERT_EQ(skipped->resolved_calls.count, 0);
/* The dispatcher is the one gate for every language and both drivers. */
int calls_before = skipped->calls.count;
cbm_pxc_dispatch_file(CBM_LANG_PYTHON, skipped, COMPACT_PY_SRC, (int)strlen(COMPACT_PY_SRC),
"budget_share.py", "t", NULL, NULL, NULL, 0, NULL, NULL, 0, NULL, NULL);
ASSERT_EQ(skipped->calls.count, calls_before);
ASSERT_EQ(skipped->resolved_calls.count, 0);
cbm_free_result(skipped);
cbm_free_result(walked);
PASS();
}
/* CBM_TEST_WALK_BUDGET_NODES stops the unified walk after that many nodes
* (no real timing): the result is walk_truncated, therefore lsp_skipped, and
* the definitions the walk had not reached are the only loss. */
TEST(extract_walk_truncated_at_its_cpu_budget) {
cbm_setenv("CBM_TEST_WALK_BUDGET_NODES", "8", 1);
CBMFileResult *cut = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walk_budget.py");
cbm_unsetenv("CBM_TEST_WALK_BUDGET_NODES");
CBMFileResult *full = extract(COMPACT_PY_SRC, CBM_LANG_PYTHON, "t", "walk_full.py");
ASSERT_NOT_NULL(cut);
ASSERT_NOT_NULL(full);
ASSERT_TRUE(cut->walk_truncated);
ASSERT_TRUE(cut->lsp_skipped);
ASSERT_FALSE(full->walk_truncated);
ASSERT_TRUE(cut->usages.count <= full->usages.count);
ASSERT_TRUE(cut->calls.count <= full->calls.count);
cbm_free_result(cut);
cbm_free_result(full);
PASS();
}
SUITE(extraction) {
RUN_TEST(extract_compact_keeps_every_field_and_shrinks_the_arena);
RUN_TEST(extract_compact_is_idempotent_and_survives_empty_results);
RUN_TEST(extract_spill_round_trip_keeps_every_field);
RUN_TEST(extract_lsp_skipped_when_parse_used_its_budget_share);
RUN_TEST(extract_walk_truncated_at_its_cpu_budget);
/* Initialize extraction library */
cbm_init();
tests/test_graph_buffer.c
+33
View File
@@ -6,6 +6,8 @@
*/
#include "test_framework.h"
#include "graph_buffer/graph_buffer.h"
#include "foundation/mem_core.h"
#include <stdatomic.h>
#include "store/store.h"
#include <string.h>
@@ -1120,7 +1122,38 @@ TEST(gbuf_flush_skips_orphan_edges) {
/* ── Suite ─────────────────────────────────────────────────────── */
/* A worker buffer draws ids from the shared counter, so a dense id -> node
* array in it spans the whole global id space: 18 workers x (next power of
* two above the highest id) x 8 B, doubling in lockstep -- a 1 GB step
* inside one gate interval on the kernel at 8M ids (2026-09-14). A worker
* buffer is never asked by id before the merge, so it keeps no such array;
* the main buffer it merges into still answers by id. One node at id 2M
* would cost a 16 MB array; the index class must not grow by even 1 MB. */
TEST(gbuf_worker_buffer_keeps_no_by_id_array) {
_Atomic int64_t ids;
atomic_init(&ids, (int64_t)1 << 21);
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);
cbm_gbuf_t *w = cbm_gbuf_new_worker("p", "/r", &ids);
ASSERT_NOT_NULL(w);
int64_t id = cbm_gbuf_upsert_node(w, "Function", "f", "p.f", "a.c", 1, 2, "{}");
ASSERT_TRUE(id >= ((int64_t)1 << 21));
size_t after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_INDEX);
size_t grown = after > before ? after - before : 0;
ASSERT_TRUE(grown < ((size_t)1 << 20));
ASSERT_TRUE(cbm_gbuf_find_by_id(w, id) == NULL);
ASSERT_NOT_NULL(cbm_gbuf_find_by_qn(w, "p.f"));
cbm_gbuf_t *main_gb = cbm_gbuf_new("p", "/r");
ASSERT_NOT_NULL(main_gb);
cbm_gbuf_merge(main_gb, w);
ASSERT_NOT_NULL(cbm_gbuf_find_by_id(main_gb, id));
cbm_gbuf_free(w);
cbm_gbuf_free(main_gb);
PASS();
}
SUITE(graph_buffer) {
RUN_TEST(gbuf_worker_buffer_keeps_no_by_id_array);
/* Original tests */
RUN_TEST(gbuf_create_free);
RUN_TEST(gbuf_free_null);
tests/test_mcp.c
+10
View File
@@ -17773,8 +17773,15 @@ TEST(index_repository_over_budget_reports_named_reason) {
bool reason_named = second_reason && strcmp(second_reason, "over_memory_budget") == 0;
bool previous_preserved = second_previous && strcmp(second_previous, "preserved") == 0;
bool hint_names_knob = second_hint && strstr(second_hint, "CBM_MEM_BUDGET_MB") != NULL;
/* The hint must also say that peak_rss_mb is NOT the requirement. The abort
* fires when RSS crosses the budget, so the peak is pinned just above it by
* construction; a caller who retries at peak+10% fails again. Measured on
* the linux kernel 2026-09-13: aborted at 25622 MB against a 24576 MB
* budget, but completing it took 31.75 GB. */
bool hint_warns_peak_is_not_need = second_hint && strstr(second_hint, "STOPPED") != NULL;
int budget_mb = budget_doc_int(second_doc, "budget_mb", -1);
int peak_rss_mb = budget_doc_int(second_doc, "peak_rss_mb", -1);
int suggested_mb = budget_doc_int(second_doc, "suggested_budget_mb", -1);
yyjson_doc_free(second_doc);
free(second);
long db_size_after = (long)cbm_file_size(db_path);
@@ -17817,8 +17824,11 @@ TEST(index_repository_over_budget_reports_named_reason) {
ASSERT_TRUE(reason_named);
ASSERT_TRUE(previous_preserved);
ASSERT_TRUE(hint_names_knob);
ASSERT_TRUE(hint_warns_peak_is_not_need);
ASSERT_EQ(budget_mb, 1);
ASSERT_GT(peak_rss_mb, budget_mb);
/* A CONCRETE retry value, not just the knob name: 1.5x the budget. */
ASSERT_GT(suggested_mb, budget_mb);
/* Preserved on disk and still served: same file size, same node count. */
ASSERT_GT(db_size_before, 0L);
ASSERT_EQ(db_size_after, db_size_before);
tests/test_mem.c
+249
View File
@@ -6,6 +6,8 @@
#include "test_framework.h"
#include "test_helpers.h"
#include "../src/foundation/mem.h"
#include "../src/foundation/platform.h" /* cbm_system_info, cbm_system_available_ram */
#include "../src/foundation/mem_core.h"
#include "../src/foundation/arena.h"
#include "../src/foundation/slab_alloc.h"
#include "../src/foundation/compat_thread.h"
@@ -1336,6 +1338,237 @@ TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010) {
PASS();
}
/* ── mem_core: the central allocation route ────────────────────────────
*
* Every assertion below is a DELTA, never an absolute. Other code in this
* process may allocate through the core concurrently, so a test that pinned an
* absolute total would be measuring the rest of the suite. */
TEST(mem_core_accounts_alloc_and_free) {
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE);
size_t blocks_before = cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE);
void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_NODE, 4096);
ASSERT_TRUE(p != NULL);
size_t during = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE);
/* Charged at least what was asked for; usable size may round UP, never
* down, so a strict >= is the honest assertion. */
ASSERT_TRUE(during >= before + 4096);
ASSERT_EQ((int)(cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE) - blocks_before), 1);
cbm_free(CBM_MEM_CLASS_GBUF_NODE, p);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE) - before), 0);
ASSERT_EQ((int)(cbm_mem_class_live_blocks(CBM_MEM_CLASS_GBUF_NODE) - blocks_before), 0);
PASS();
}
/* The whole point of classes: attribution. If a gbuf allocation could show up
* under semantic, the table could not choose between "park workers" and
* "stream the vectors" -- the decision this core exists to inform. */
TEST(mem_core_classes_do_not_bleed) {
size_t node_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE);
size_t sem_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC);
void *p = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, 8192);
ASSERT_TRUE(p != NULL);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_NODE) - node_before), 0);
ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_SEMANTIC) >= sem_before + 8192);
cbm_free(CBM_MEM_CLASS_SEMANTIC, p);
PASS();
}
TEST(mem_core_realloc_replaces_the_old_charge) {
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP);
void *p = cbm_alloc(CBM_MEM_CLASS_DUMP, 1024);
ASSERT_TRUE(p != NULL);
p = cbm_realloc(CBM_MEM_CLASS_DUMP, p, 65536);
ASSERT_TRUE(p != NULL);
size_t grown = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP);
/* The old 1024 must be gone, not stacked on top: exactly one block is live,
* so the delta is bounded by the new size plus rounding, not by the sum. */
ASSERT_TRUE(grown >= before + 65536);
ASSERT_TRUE(grown < before + 65536 + 65536);
cbm_free(CBM_MEM_CLASS_DUMP, p);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP) - before), 0);
PASS();
}
/* realloc(NULL) is alloc, and free(NULL) is a no-op: the core must match the C
* library exactly or adoption stops being a mechanical rename. */
TEST(mem_core_matches_libc_null_semantics) {
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE);
cbm_free(CBM_MEM_CLASS_STORE, NULL);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) - before), 0);
void *p = cbm_realloc(CBM_MEM_CLASS_STORE, NULL, 2048);
ASSERT_TRUE(p != NULL);
ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) >= before + 2048);
cbm_free(CBM_MEM_CLASS_STORE, p);
/* A zero-size request still yields a freeable pointer. */
void *z = cbm_alloc(CBM_MEM_CLASS_STORE, 0);
ASSERT_TRUE(z != NULL);
cbm_free(CBM_MEM_CLASS_STORE, z);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_STORE) - before), 0);
PASS();
}
TEST(mem_core_strdup_copies_and_accounts) {
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING);
ASSERT_TRUE(cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, NULL) == NULL);
const char *src = "qualified::name::example";
char *copy = cbm_mem_strdup(CBM_MEM_CLASS_GBUF_STRING, src);
ASSERT_TRUE(copy != NULL);
ASSERT_TRUE(strcmp(copy, src) == 0);
ASSERT_TRUE(copy != src);
ASSERT_TRUE(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING) > before);
cbm_free(CBM_MEM_CLASS_GBUF_STRING, copy);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_GBUF_STRING) - before), 0);
PASS();
}
/* A budget decision is about the PEAK, not about whatever was live when
* someone looked. Peak must survive the free that follows it. */
TEST(mem_core_peak_survives_the_free) {
cbm_mem_class_reset_peaks();
size_t base = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT);
void *p = cbm_alloc(CBM_MEM_CLASS_EXTRACT, 32768);
ASSERT_TRUE(p != NULL);
size_t peak_live = cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT);
ASSERT_TRUE(peak_live >= base + 32768);
cbm_free(CBM_MEM_CLASS_EXTRACT, p);
ASSERT_EQ((int)(cbm_mem_class_peak_bytes(CBM_MEM_CLASS_EXTRACT) - peak_live), 0);
PASS();
}
/* Arena-backed memory reports in bulk rather than per object: the extraction
* engine has 1301 arena call sites and rewriting them to per-object cbm_alloc
* would undo the batching that keeps its allocation count low. */
TEST(mem_core_external_bulk_accounting_is_symmetric) {
size_t before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT);
cbm_mem_class_add_external(CBM_MEM_CLASS_EXTRACT, 1024 * 1024);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT) - before), 1024 * 1024);
cbm_mem_class_remove_external(CBM_MEM_CLASS_EXTRACT, 1024 * 1024);
ASSERT_EQ((int)(cbm_mem_class_live_bytes(CBM_MEM_CLASS_EXTRACT) - before), 0);
PASS();
}
/* The one mistake a caller can actually make is freeing with the wrong class.
* That must never underflow the counter: an unsigned wrap would turn a small
* drift into a colossal bogus total that reads as a catastrophic leak and
* sends someone hunting a phantom. Clamp at zero instead. */
TEST(mem_core_mismatched_class_never_wraps) {
void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_EDGE, 4096);
ASSERT_TRUE(p != NULL);
/* Free against a class that was never charged for it. */
size_t other_before = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP);
cbm_free(CBM_MEM_CLASS_DUMP, p);
size_t other_after = cbm_mem_class_live_bytes(CBM_MEM_CLASS_DUMP);
ASSERT_TRUE(other_after <= other_before); /* clamped, never wrapped */
ASSERT_TRUE(other_after < (size_t)-1 / 2);
PASS();
}
TEST(mem_core_report_json_is_wellformed_or_empty) {
void *p = cbm_alloc(CBM_MEM_CLASS_GBUF_INDEX, 4096);
ASSERT_TRUE(p != NULL);
char buf[CBM_SZ_1K];
int n = cbm_mem_class_report_json(buf, sizeof(buf));
ASSERT_TRUE(n > 0);
ASSERT_TRUE(buf[0] == '[');
ASSERT_TRUE(buf[n - 1] == ']');
ASSERT_TRUE(strstr(buf, "gbuf_index") != NULL);
/* A buffer too small must yield NOTHING, never a truncated array that a
* JSON reader would reject or, worse, silently mis-parse. */
char tiny[8];
ASSERT_EQ(cbm_mem_class_report_json(tiny, sizeof(tiny)), 0);
cbm_free(CBM_MEM_CLASS_GBUF_INDEX, p);
PASS();
}
/* ── Pressure primitives and the charged reading ─────────────────────── */
TEST(mem_charged_is_positive_and_consistent_with_rss) {
size_t charged = cbm_mem_charged();
size_t rss = cbm_mem_rss();
ASSERT(charged > 0);
ASSERT(rss > 0);
/* Same order of magnitude as RSS on every platform: the charged value
* may sit below RSS (purged-but-resident pages) or slightly above it
* (compressed pages), never at zero or at a multiple. */
ASSERT(charged < rss * 4);
ASSERT(rss < charged * 4 + (size_t)64 * 1024 * 1024);
PASS();
}
/* The charged high-water mark never reads below a charge just taken, and a
* later, larger charge lifts it: it is a max over every reading. */
TEST(mem_peak_charged_is_the_high_water_of_charged) {
size_t charged = cbm_mem_charged();
ASSERT_TRUE(charged > 0);
ASSERT_TRUE(cbm_mem_peak_charged() >= charged);
size_t peak_before = cbm_mem_peak_charged();
(void)cbm_mem_charged();
ASSERT_TRUE(cbm_mem_peak_charged() >= peak_before);
PASS();
}
TEST(mem_footprint_zero_or_plausible) {
size_t fp = cbm_mem_footprint();
if (fp > 0) {
ASSERT(fp >= (size_t)1024 * 1024); /* a live test process is more than 1 MB */
}
PASS();
}
TEST(mem_system_available_ram_is_within_total) {
size_t avail = cbm_system_available_ram();
cbm_system_info_t info = cbm_system_info();
if (avail == 0 || info.total_ram == 0) {
PASS(); /* platform cannot answer; the caller treats that as unknown */
}
ASSERT(avail <= info.total_ram);
PASS();
}
TEST(mem_system_under_pressure_is_a_pure_threshold) {
size_t avail = cbm_system_available_ram();
cbm_system_info_t info = cbm_system_info();
bool under = cbm_mem_system_under_pressure();
if (avail == 0 || info.total_ram == 0) {
ASSERT_FALSE(under); /* never abort on a guess */
PASS();
}
ASSERT_EQ(under, avail < info.total_ram / 8);
PASS();
}
TEST(mem_over_budget_follows_the_charged_reading) {
size_t saved = cbm_mem_budget();
size_t charged = cbm_mem_charged();
ASSERT(charged > 0);
cbm_mem_set_budget_for_tests(charged * 4);
ASSERT_FALSE(cbm_mem_over_budget());
cbm_mem_set_budget_for_tests(charged / 4 + 1);
ASSERT_TRUE(cbm_mem_over_budget());
cbm_mem_set_budget_for_tests(saved);
PASS();
}
TEST(mem_core_class_names_are_total) {
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_SEMANTIC), "semantic") == 0);
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_ARENA), "arena") == 0);
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_TS_TREE), "ts_tree") == 0);
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_STORE), "store") == 0);
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_HASH_TABLE), "hash_table") == 0);
ASSERT_TRUE(strcmp(cbm_mem_class_name(CBM_MEM_CLASS_DYN_ARRAY), "dyn_array") == 0);
/* Out of range must still answer, so a log line never takes a NULL. */
ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)(CBM_MEM_CLASS_COUNT + 5)) != NULL);
ASSERT_TRUE(cbm_mem_class_name((cbm_mem_class_t)-1) != NULL);
PASS();
}
SUITE(mem) {
/* mem API */
RUN_TEST(mem_arena_eager_commit_follows_platform_commit_cost);
@@ -1402,4 +1635,20 @@ SUITE(mem) {
/* extraction scratch arena (#2010) */
RUN_TEST(extract_traversal_stacks_come_from_ctx_scratch_issue2010);
RUN_TEST(mem_core_accounts_alloc_and_free);
RUN_TEST(mem_core_classes_do_not_bleed);
RUN_TEST(mem_core_realloc_replaces_the_old_charge);
RUN_TEST(mem_core_matches_libc_null_semantics);
RUN_TEST(mem_core_strdup_copies_and_accounts);
RUN_TEST(mem_core_peak_survives_the_free);
RUN_TEST(mem_core_external_bulk_accounting_is_symmetric);
RUN_TEST(mem_core_mismatched_class_never_wraps);
RUN_TEST(mem_core_report_json_is_wellformed_or_empty);
RUN_TEST(mem_core_class_names_are_total);
RUN_TEST(mem_charged_is_positive_and_consistent_with_rss);
RUN_TEST(mem_peak_charged_is_the_high_water_of_charged);
RUN_TEST(mem_footprint_zero_or_plausible);
RUN_TEST(mem_system_available_ram_is_within_total);
RUN_TEST(mem_system_under_pressure_is_a_pure_threshold);
RUN_TEST(mem_over_budget_follows_the_charged_reading);
}
tests/test_parallel.c
+105 -43
View File
@@ -19,6 +19,7 @@
#include "foundation/platform.h"
#include "foundation/log.h"
#include "cbm.h"
#include "result_spill.h"
#include <stdlib.h>
#include <string.h>
@@ -271,6 +272,11 @@ static cbm_gbuf_t *run_sequential_with_lsp_cross(const char *project, const char
/* ── Run parallel pipeline on files, returning gbuf ───────────────── */
/* Spill mode for the harness: the run below opts its context in (the way
* run_parallel_pipeline does) and records how many results the store parked. */
static bool g_harness_spill = false;
static int64_t g_harness_parked = -1;
static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
const char *project, const char *repo_path, cbm_file_info_t *files, int file_count,
int worker_count, const cbm_parallel_extract_opts_t *extract_opts,
@@ -286,6 +292,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
.gbuf = gbuf,
.registry = reg,
.cancelled = &cancelled,
.spill_allowed = g_harness_spill,
};
if (seed_structure) {
@@ -306,6 +313,14 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
cbm_parallel_extract(&ctx, files, file_count, result_cache, &shared_ids, worker_count);
}
cbm_gbuf_set_next_id(gbuf, atomic_load(&shared_ids));
if (g_harness_spill) {
int64_t bytes = 0;
int64_t loads = 0;
g_harness_parked = -1;
if (ctx.spill) {
cbm_result_spill_stats(ctx.spill, &g_harness_parked, &bytes, &loads);
}
}
if (mutator) {
mutator(result_cache, file_count, mutator_ud);
@@ -323,8 +338,10 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
* cbm_pxc_run_one(_ts) per file BEFORE materializing CALLS edges. */
char **def_modules = (char **)calloc((size_t)file_count, sizeof(char *));
int def_count = 0;
CBMArena cross_arena;
cbm_arena_init(&cross_arena);
CBMLSPDef *all_defs =
def_modules ? cbm_pxc_collect_all_defs(&ctx, result_cache, files, file_count,
def_modules ? cbm_pxc_collect_all_defs(&ctx, &cross_arena, result_cache, files, file_count,
ctx.project_name, def_modules, &def_count, NULL)
: NULL;
CBMModuleDefIndex *module_def_index =
@@ -337,6 +354,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
cbm_pxc_free_module_def_index(module_def_index);
free(all_defs);
cbm_arena_destroy(&cross_arena);
if (def_modules) {
for (int i = 0; i < file_count; i++) {
free(def_modules[i]);
@@ -348,6 +366,7 @@ static cbm_gbuf_t *run_parallel_with_extract_opts_and_mutator(
if (result_cache[i])
cbm_free_result(result_cache[i]);
free(result_cache);
cbm_pipeline_spill_close(&ctx);
harness_ctx_free_tables(&ctx);
cbm_registry_free(reg);
@@ -528,6 +547,48 @@ TEST(parallel_total_edges) {
PASS();
}
/* ── Spill mode: the graph is the in-memory graph ─────────────────── */
/* CBM_MEM_SPILL=1 parks every compacted result on disk the moment it is
* extracted; registry build, def collection, surfaces, resolve and the infra
* passes read each one back only for the moment they need it. The graph must
* not be able to tell: same nodes, same edges per type as the in-memory run
* of the same repo -- and every file must actually have gone through the
* store (a store that failed to open would silently test nothing). */
TEST(parallel_spill_mode_builds_the_same_graph) {
if (ensure_parity_setup() != 0)
FAIL("setup failed");
cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL};
cbm_file_info_t *files = NULL;
int file_count = 0;
ASSERT_EQ(cbm_discover(g_par_tmpdir, &opts, &files, &file_count), 0);
ASSERT_GT(file_count, 0);
cbm_setenv("CBM_MEM_SPILL", "1", 1);
g_harness_spill = true;
cbm_gbuf_t *spilled = run_parallel("par-test", g_par_tmpdir, files, file_count, 2);
g_harness_spill = false;
cbm_unsetenv("CBM_MEM_SPILL");
cbm_discover_free(files, file_count);
ASSERT(spilled != NULL);
ASSERT_EQ((int)g_harness_parked, file_count);
ASSERT_EQ(cbm_gbuf_node_count(spilled), cbm_gbuf_node_count(g_par_gbuf));
ASSERT_EQ(cbm_gbuf_edge_count(spilled), cbm_gbuf_edge_count(g_par_gbuf));
static const char *const types[] = {"CALLS", "DEFINES", "DEFINES_METHOD", "IMPORTS",
"USES", "INHERITS", "IMPLEMENTS"};
for (size_t t = 0; t < sizeof(types) / sizeof(types[0]); t++) {
int in_memory = cbm_gbuf_edge_count_by_type(g_par_gbuf, types[t]);
int on_disk = cbm_gbuf_edge_count_by_type(spilled, types[t]);
if (in_memory != on_disk) {
printf(" FAIL: %s edges: in_memory=%d spilled=%d\n", types[t], in_memory, on_disk);
}
ASSERT_EQ(in_memory, on_disk);
}
cbm_gbuf_free(spilled);
PASS();
}
/* ── Empty file list ──────────────────────────────────────────────── */
TEST(parallel_empty_files) {
@@ -2583,10 +2644,9 @@ TEST(parallel_kotlin_nonbinary_operator_carriers_reach_graph) {
* the test green. The optional crate_b-local helper pins the second failure
* mode where a confident in-file resolution used to suppress cross-LSP. */
static cbm_gbuf_t *run_issue56_parallel_workspace(bool local_decoy) {
static const char workspace_toml[] =
"[workspace]\n"
"members = [\"crate_a\", \"crate_b\"]\n"
"resolver = \"2\"\n";
static const char workspace_toml[] = "[workspace]\n"
"members = [\"crate_a\", \"crate_b\"]\n"
"resolver = \"2\"\n";
static const char crate_a_toml[] =
"[package]\nname = \"crate_a\"\nversion = \"0.1.0\"\nedition = \"2021\"\n";
static const char crate_b_toml[] =
@@ -2594,13 +2654,11 @@ static cbm_gbuf_t *run_issue56_parallel_workspace(bool local_decoy) {
"\n[dependencies]\ncrate_a = { path = \"../crate_a\" }\n";
static const char crate_a_source[] = "pub fn helper() {}\n";
static const char unlisted_source[] = "pub fn helper() {}\n";
static const char caller_without_local[] =
"fn run() { crate_a::helper(); }\n"
"fn main() { run(); }\n";
static const char caller_with_local[] =
"fn helper() {}\n"
"fn run() { crate_a::helper(); }\n"
"fn main() { run(); }\n";
static const char caller_without_local[] = "fn run() { crate_a::helper(); }\n"
"fn main() { run(); }\n";
static const char caller_with_local[] = "fn helper() {}\n"
"fn run() { crate_a::helper(); }\n"
"fn main() { run(); }\n";
char tmpdir[256];
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_issue56_parallel_XXXXXX");
@@ -2669,13 +2727,12 @@ TEST(parallel_rust_cross_crate_worker_receives_workspace_manifest) {
const cbm_gbuf_edge_t *correct =
find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a.");
const bool correct_found = correct != NULL;
const bool unlisted =
callable_has_call_target_fragment(gbuf, "main.run", ".unlisted.");
const bool unlisted = callable_has_call_target_fragment(gbuf, "main.run", ".unlisted.");
const bool manifest_strategy =
correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate");
if (!correct || unlisted || !manifest_strategy) {
printf(" issue56 manifest diagnostic: correct=%d unlisted=%d strategy=%d\n",
correct_found, unlisted, manifest_strategy);
printf(" issue56 manifest diagnostic: correct=%d unlisted=%d strategy=%d\n", correct_found,
unlisted, manifest_strategy);
}
cbm_gbuf_free(gbuf);
@@ -2692,8 +2749,7 @@ TEST(parallel_rust_cross_crate_manifest_beats_confident_local_resolution) {
const cbm_gbuf_edge_t *correct =
find_call_edge_to_target_fragment(gbuf, "main.run", ".crate_a.");
const bool correct_found = correct != NULL;
const bool wrong_local =
callable_has_call_target_fragment(gbuf, "main.run", ".crate_b.");
const bool wrong_local = callable_has_call_target_fragment(gbuf, "main.run", ".crate_b.");
const bool manifest_strategy =
correct && correct->properties_json && strstr(correct->properties_json, "lsp_cross_crate");
if (!correct || wrong_local || !manifest_strategy) {
@@ -3259,7 +3315,8 @@ static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char
r->defs.count, r->imports.count, r->calls.count, r->resolved_calls.count);
for (int j = 0; j < r->resolved_calls.count; j++) {
const CBMResolvedCall *rc = &r->resolved_calls.items[j];
printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d span=[%u,%u)\n",
printf(" [diag] rc caller=%s callee=%s strategy=%s conf=%.2f kind=%d "
"span=[%u,%u)\n",
rc->caller_qn ? rc->caller_qn : "?", rc->callee_qn ? rc->callee_qn : "?",
rc->strategy ? rc->strategy : "?", rc->confidence, (int)rc->kind,
(unsigned)rc->site_start_byte, (unsigned)rc->site_end_byte);
@@ -3267,13 +3324,16 @@ static cbm_gbuf_t *run_go_field_chain_sequential(const char *project, const char
for (int j = 0; j < r->calls.count; j++) {
const CBMCall *c = &r->calls.items[j];
printf(" [diag] call callee=%s enclosing=%s span=[%u,%u) req=%d\n",
c->callee_name ? c->callee_name : "?", c->enclosing_func_qn ? c->enclosing_func_qn : "?",
(unsigned)c->site_start_byte, (unsigned)c->site_end_byte, (int)c->requires_lsp_resolution);
c->callee_name ? c->callee_name : "?",
c->enclosing_func_qn ? c->enclosing_func_qn : "?",
(unsigned)c->site_start_byte, (unsigned)c->site_end_byte,
(int)c->requires_lsp_resolution);
}
for (int j = 0; j < r->defs.count; j++) {
const CBMDefinition *d = &r->defs.items[j];
printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n", d->label ? d->label : "?",
d->qualified_name ? d->qualified_name : "?", d->parent_class ? d->parent_class : "?",
printf(" [diag] def label=%s qn=%s parent=%s ret=%s\n",
d->label ? d->label : "?", d->qualified_name ? d->qualified_name : "?",
d->parent_class ? d->parent_class : "?",
d->return_type ? d->return_type : "?");
}
}
@@ -3324,11 +3384,13 @@ TEST(parallel_go_cross_package_field_chain_resolves) {
"\n"
"type OrderService struct{}\n"
"\n"
"func (s *OrderService) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
"func (s *OrderService) PlaceOrder(ctx context.Context, req "
"*pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
" return nil, nil\n"
"}\n"
"\n"
"func (s *OrderService) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
"func (s *OrderService) ListOrders(ctx context.Context, req "
"*pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
" return nil, nil\n"
"}\n") != 0 ||
th_write_file(app_path, "package handler\n"
@@ -3353,15 +3415,18 @@ TEST(parallel_go_cross_package_field_chain_resolves) {
" searchSvc *service.SearchService\n"
"}\n"
"\n"
"func (h *OrderHandler) ListOrders(ctx context.Context, req *pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
"func (h *OrderHandler) ListOrders(ctx context.Context, req "
"*pb.ListOrdersReq) (*pb.ListOrdersRsp, error) {\n"
" return h.orderSvc.ListOrders(ctx, req)\n"
"}\n"
"\n"
"func (h *OrderHandler) PlaceOrder(ctx context.Context, req *pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
"func (h *OrderHandler) PlaceOrder(ctx context.Context, req "
"*pb.PlaceOrderReq) (*pb.PlaceOrderRsp, error) {\n"
" return h.orderSvc.PlaceOrder(ctx, req)\n"
"}\n"
"\n"
"func (h *OrderHandler) UpdateCart(ctx context.Context, req *pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n"
"func (h *OrderHandler) UpdateCart(ctx context.Context, req "
"*pb.UpdateCartReq) (*pb.UpdateCartRsp, error) {\n"
" return h.cartSvc.UpdateCart(ctx, req)\n"
"}\n") != 0) {
th_rmtree(tmpdir);
@@ -3379,11 +3444,11 @@ TEST(parallel_go_cross_package_field_chain_resolves) {
cbm_gbuf_t *gbuf = run_go_field_chain_sequential("go_field_fold", tmpdir, files, 2);
ASSERT_NOT_NULL(gbuf);
const cbm_gbuf_edge_t *edge = find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder");
const cbm_gbuf_edge_t *edge =
find_call_edge_to_target_fragment(gbuf, "handler.PlaceOrder", ".service.PlaceOrder");
const bool found = edge != NULL;
const bool dispatch =
edge && edge->properties_json &&
strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\"");
const bool dispatch = edge && edge->properties_json &&
strstr(edge->properties_json, "\"strategy\":\"lsp_type_dispatch\"");
if (!found || !dispatch) {
printf(" go field chain diagnostic: found=%d dispatch=%d\n", found, dispatch);
if (edge && edge->properties_json) {
@@ -3414,8 +3479,7 @@ TEST(parallel_go_cross_package_field_chain_resolves) {
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->source_id) : NULL;
const cbm_gbuf_node_t *dst =
all[i] ? cbm_gbuf_find_by_id(gbuf, all[i]->target_id) : NULL;
printf(" CALLS edge: %s -> %s props=%s\n",
src ? src->qualified_name : "?",
printf(" CALLS edge: %s -> %s props=%s\n", src ? src->qualified_name : "?",
dst ? dst->qualified_name : "?",
all[i]->properties_json ? all[i]->properties_json : "{}");
}
@@ -3797,8 +3861,8 @@ TEST(lsp_resolve_project_prefixed_duplicate_is_not_ambiguous) {
const char *project = "proj";
cbm_gbuf_t *gbuf = cbm_gbuf_new(project, "/tmp");
ASSERT_NOT_NULL(gbuf);
int64_t target_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler",
"proj.mod.Target.handler", "target.py", 1, 1, "{}");
int64_t target_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", "proj.mod.Target.handler",
"target.py", 1, 1, "{}");
ASSERT_GT(target_id, 0);
CBMCall call = make_call("proj.mod.Caller.run", "handler");
@@ -3807,8 +3871,7 @@ TEST(lsp_resolve_project_prefixed_duplicate_is_not_ambiguous) {
CBMResolvedCall raw = make_rc("proj.mod.Caller.run", "mod.Target.handler", 0.75f);
raw.site_start_byte = call.site_start_byte;
raw.site_end_byte = call.site_end_byte;
CBMResolvedCall prefixed =
make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f);
CBMResolvedCall prefixed = make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f);
prefixed.site_start_byte = call.site_start_byte;
prefixed.site_end_byte = call.site_end_byte;
CBMResolvedCall items[] = {raw, prefixed};
@@ -3827,9 +3890,8 @@ TEST(lsp_resolve_project_prefix_spellings_stay_ambiguous_when_both_nodes_exist)
ASSERT_NOT_NULL(gbuf);
int64_t raw_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler", "mod.Target.handler",
"raw.py", 1, 1, "{}");
int64_t prefixed_id = cbm_gbuf_upsert_node(gbuf, "Function", "handler",
"proj.mod.Target.handler", "prefixed.py", 1, 1,
"{}");
int64_t prefixed_id = cbm_gbuf_upsert_node(
gbuf, "Function", "handler", "proj.mod.Target.handler", "prefixed.py", 1, 1, "{}");
ASSERT_GT(raw_id, 0);
ASSERT_GT(prefixed_id, 0);
ASSERT(raw_id != prefixed_id);
@@ -3840,8 +3902,7 @@ TEST(lsp_resolve_project_prefix_spellings_stay_ambiguous_when_both_nodes_exist)
CBMResolvedCall raw = make_rc("proj.mod.Caller.run", "mod.Target.handler", 0.75f);
raw.site_start_byte = call.site_start_byte;
raw.site_end_byte = call.site_end_byte;
CBMResolvedCall prefixed =
make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f);
CBMResolvedCall prefixed = make_rc("proj.mod.Caller.run", "proj.mod.Target.handler", 0.90f);
prefixed.site_start_byte = call.site_start_byte;
prefixed.site_end_byte = call.site_end_byte;
CBMResolvedCall items[] = {raw, prefixed};
@@ -4348,6 +4409,7 @@ SUITE(parallel) {
RUN_TEST(parallel_implements_parity);
RUN_TEST(parallel_semantic_fixture_expected_counts);
RUN_TEST(parallel_total_edges);
RUN_TEST(parallel_spill_mode_builds_the_same_graph);
RUN_TEST(parallel_empty_files);
RUN_TEST(parallel_args_json_no_overflow);
tests/test_pipeline.c
+146
View File
@@ -14269,6 +14269,151 @@ TEST(pipeline_markdown_and_config_prose_reaches_fts_body) {
* attribute, so the sanitizer is silent there. The label counts pin the
* fixture to what it claims: at least one Struct and zero Function/Method,
* i.e. the scan genuinely finds nothing to sort. */
/* ── Semantic pass, batched == unbatched ─────────────────────────────── */
/* Under memory pressure the semantic pass tokenizes, counts and vectorizes
* in headroom-sized batches of functions (tokenizing twice) instead of
* holding every function's tokens at once. The graph must not be able to
* tell: the same repo indexed with CBM_SEM_BATCH=5 (forced batches, 30
* functions -> 6 batches) yields byte-identical node vectors, token vectors
* and SEMANTICALLY_RELATED edges. Rows are compared by qualified name, never
* by node id: parallel extraction hands out ids in worker order. */
static void write_sem_family(const char *base, const char *file, const char *subject) {
char path[512];
snprintf(path, sizeof(path), "%s/%s", base, file);
char body[4096];
snprintf(body, sizeof(body),
"package main\n\n"
"// Parse%sConfig reads the %s config file and returns the parsed %s config.\n"
"func Parse%sConfig(path string) (*%sConfig, error) {\n"
"\treturn Load%sConfig(path)\n}\n\n"
"// Load%sConfig loads the %s config from disk and validates it.\n"
"func Load%sConfig(path string) (*%sConfig, error) {\n"
"\tcfg := &%sConfig{}\n\tValidate%sConfig(cfg)\n\treturn cfg, nil\n}\n\n"
"// Validate%sConfig checks the %s config for missing fields.\n"
"func Validate%sConfig(cfg *%sConfig) bool {\n\treturn cfg != nil\n}\n\n"
"// Write%sConfig serializes the %s config back to disk.\n"
"func Write%sConfig(path string, cfg *%sConfig) error {\n"
"\tValidate%sConfig(cfg)\n\treturn nil\n}\n\n"
/* A near-duplicate of Load: same doc, same body, same calls -- the
* pair the pass must relate, in every venue. */
"// Load%sConfigFile loads the %s config from disk and validates it.\n"
"func Load%sConfigFile(path string) (*%sConfig, error) {\n"
"\tcfg := &%sConfig{}\n\tValidate%sConfig(cfg)\n\treturn cfg, nil\n}\n\n"
"type %sConfig struct{ Name string }\n",
subject, subject, subject, subject, subject, subject, subject, subject, subject,
subject, subject, subject, subject, subject, subject, subject, subject, subject,
subject, subject, subject, subject, subject, subject, subject, subject, subject,
subject);
th_write_file(path, body);
}
/* Step both statements in lockstep; every column of every row must match. */
static bool sem_same_rows(sqlite3 *a, sqlite3 *b, const char *sql, int *rows) {
sqlite3_stmt *sa = NULL;
sqlite3_stmt *sb = NULL;
bool same = sqlite3_prepare_v2(a, sql, -1, &sa, NULL) == SQLITE_OK &&
sqlite3_prepare_v2(b, sql, -1, &sb, NULL) == SQLITE_OK;
*rows = 0;
while (same) {
int ra = sqlite3_step(sa);
int rb = sqlite3_step(sb);
if (ra != rb) {
same = false;
break;
}
if (ra != SQLITE_ROW) {
break;
}
int cols = sqlite3_column_count(sa);
for (int c = 0; c < cols && same; c++) {
const unsigned char *va = sqlite3_column_text(sa, c);
const unsigned char *vb = sqlite3_column_text(sb, c);
same = (va == NULL) == (vb == NULL) &&
(!va || strcmp((const char *)va, (const char *)vb) == 0);
}
(*rows)++;
}
sqlite3_finalize(sa);
sqlite3_finalize(sb);
return same;
}
TEST(pipeline_semantic_batched_matches_unbatched) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_sembatch_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
static const char *const subjects[] = {"User", "Server", "Client", "Cache", "Queue", "Mail"};
for (size_t i = 0; i < sizeof(subjects) / sizeof(subjects[0]); i++) {
char file[64];
snprintf(file, sizeof(file), "%s_config.go", subjects[i]);
write_sem_family(tmp, file, subjects[i]);
}
char db_plain[512];
char db_batched[512];
snprintf(db_plain, sizeof(db_plain), "%s/plain.db", tmp);
snprintf(db_batched, sizeof(db_batched), "%s/batched.db", tmp);
/* The default threshold (0.75) admits no pair on a 30-function fixture;
* 0.3 admits the near-duplicates. The test is about equality, not the bar. */
cbm_setenv("CBM_SEMANTIC_THRESHOLD", "0.3", 1);
cbm_unsetenv("CBM_SEM_BATCH");
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_plain, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
cbm_pipeline_free(p);
p = cbm_pipeline_new(tmp, db_batched, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
cbm_setenv("CBM_SEM_BATCH", "5", 1); /* read by the pass, once per run */
int rc = cbm_pipeline_run(p);
cbm_unsetenv("CBM_SEM_BATCH");
cbm_unsetenv("CBM_SEMANTIC_THRESHOLD");
cbm_pipeline_free(p);
ASSERT_EQ(rc, 0);
cbm_store_t *sp = cbm_store_open_path(db_plain);
cbm_store_t *sb = cbm_store_open_path(db_batched);
ASSERT_NOT_NULL(sp);
ASSERT_NOT_NULL(sb);
sqlite3 *a = cbm_store_get_db(sp);
sqlite3 *b = cbm_store_get_db(sb);
/* The fixture must exercise batching: more functions than the forced
* batch, and a semantic pass that actually stored vectors and edges. */
sqlite3_stmt *st = NULL;
ASSERT_EQ(
sqlite3_prepare_v2(a, "SELECT COUNT(*) FROM nodes WHERE label = 'Function'", -1, &st, NULL),
SQLITE_OK);
ASSERT_EQ(sqlite3_step(st), SQLITE_ROW);
int functions = sqlite3_column_int(st, 0);
sqlite3_finalize(st);
ASSERT_GT(functions, 5);
int rows = 0;
ASSERT_TRUE(sem_same_rows(a, b,
"SELECT n.qualified_name, hex(v.vector) FROM node_vectors v "
"JOIN nodes n ON n.id = v.node_id ORDER BY n.qualified_name",
&rows));
ASSERT_EQ(rows, functions);
ASSERT_TRUE(sem_same_rows(
a, b, "SELECT token, hex(vector), idf FROM token_vectors ORDER BY token", &rows));
ASSERT_GT(rows, 0);
ASSERT_TRUE(
sem_same_rows(a, b,
"SELECT s.qualified_name, t.qualified_name, e.properties FROM edges e "
"JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id "
"WHERE e.type = 'SEMANTICALLY_RELATED' ORDER BY 1, 2",
&rows));
ASSERT_GT(rows, 0);
cbm_store_close(sp);
cbm_store_close(sb);
th_rmtree(tmp);
PASS();
}
TEST(pipeline_semantic_edges_no_functions) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_nofunc_XXXXXX");
@@ -14730,6 +14875,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_delta_patch_indexes_docstring_into_fts_body);
RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body);
RUN_TEST(pipeline_semantic_edges_no_functions);
RUN_TEST(pipeline_semantic_batched_matches_unbatched);
}
/* Focused semantic-manifest and publication contracts. Kept separate from the