mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-24 06:23:06 +08:00
Merge pull request #1164 from DeusData/feat/test-sharding
Parallel test execution: per-suite processes with a zero-loss gate contract
This commit is contained in:
Martin Vogel
+7
-1
@@ -513,7 +513,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
|
||||
|
||||
# ── Targets ──────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: test test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security
|
||||
.PHONY: test test-par test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security
|
||||
|
||||
$(BUILD_DIR):
|
||||
mkdir -p $(BUILD_DIR)
|
||||
@@ -639,6 +639,12 @@ $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_
|
||||
test: $(BUILD_DIR)/test-runner
|
||||
cd $(CURDIR) && $(BUILD_DIR)/test-runner
|
||||
|
||||
# Parallel variant: every suite as its own process; identical gate quality
|
||||
# (see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh — union guard
|
||||
# against --list-suites + aggregated pass/fail/skip totals).
|
||||
test-par: $(BUILD_DIR)/test-runner
|
||||
cd $(CURDIR) && bash scripts/run-tests-parallel.sh $(BUILD_DIR)/test-runner
|
||||
|
||||
# Focused native development is intentionally a separate, explicit target so
|
||||
# an inherited TEST_SUITES value cannot silently narrow the gating test target.
|
||||
TEST_SUITES ?=
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-tests-parallel.sh — run every registered test suite as parallel
|
||||
# processes of the already-built test-runner.
|
||||
#
|
||||
# ZERO-LOSS CONTRACT (gate quality must be identical to the sequential run):
|
||||
# 1. The suite list comes from `test-runner --list-suites`, which is printed
|
||||
# by the SAME macro table that executes suites — the list cannot drift
|
||||
# from reality by construction.
|
||||
# 2. UNION GUARD: after the run, the set of suites that actually produced a
|
||||
# result is compared against that list; any difference (a suite that
|
||||
# never ran, or ran twice) fails the gate loudly. A newly added suite is
|
||||
# picked up automatically on the next invocation.
|
||||
# 3. Per-suite pass/fail/skip counts are summed and reported in the same
|
||||
# "N passed[, M failed][, K skipped]" shape as the sequential runner, so
|
||||
# before/after totals are directly comparable.
|
||||
# 4. ANY suite failing, crashing (nonzero exit), or missing ⇒ exit 1.
|
||||
#
|
||||
# Usage: run-tests-parallel.sh <path-to-test-runner> [jobs]
|
||||
# jobs defaults to CBM_TEST_PAR_JOBS, then the CPU count.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
RUNNER="${1:?usage: run-tests-parallel.sh <path-to-test-runner> [jobs]}"
|
||||
JOBS="${2:-${CBM_TEST_PAR_JOBS:-}}"
|
||||
|
||||
if [ -z "$JOBS" ]; then
|
||||
if command -v nproc >/dev/null 2>&1; then
|
||||
JOBS=$(nproc)
|
||||
elif command -v sysctl >/dev/null 2>&1; then
|
||||
JOBS=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
else
|
||||
JOBS=4
|
||||
fi
|
||||
fi
|
||||
|
||||
LOGDIR="$(dirname "$RUNNER")/test-logs"
|
||||
rm -rf "$LOGDIR"
|
||||
mkdir -p "$LOGDIR"
|
||||
|
||||
SUITES_FILE="$LOGDIR/suites.txt"
|
||||
RESULTS_FILE="$LOGDIR/results.txt"
|
||||
|
||||
# tr strips the CR that the Windows CRT appends to every stdout line — a
|
||||
# suites file with CRLF endings made the runner reject every name
|
||||
# ("arena\r" is an unknown suite) and fail all 104 suites on CI.
|
||||
if ! "$RUNNER" --list-suites | tr -d '\r' > "$SUITES_FILE"; then
|
||||
echo "FAIL: test-runner --list-suites exited nonzero" >&2
|
||||
exit 1
|
||||
fi
|
||||
NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ')
|
||||
if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then
|
||||
echo "FAIL: suite list empty or malformed (runner too old for --list-suites?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Timing-sensitive suites run SEQUENTIALLY after the parallel wave: they
|
||||
# spawn subprocesses / watch the filesystem / bind ports with fixed
|
||||
# deadlines, and a saturated 4-core CI runner starves those deadlines into
|
||||
# flakes (3 cli-suite failures on the ubuntu legs of the first CI run).
|
||||
# Same suites, same tests, same gates — only the schedule differs; the
|
||||
# union guard below still checks the COMBINED result set.
|
||||
# stack_overflow_a/b/c: their giant-recursion ASan allocations stall ~100x
|
||||
# when co-STARTED with a large wave on Apple Silicon (2s staggered vs ~230s
|
||||
# simultaneous — a local scheduler/zone quirk, not contention: job count
|
||||
# does not change it). Staggered in the tail they cost seconds.
|
||||
SERIAL_SUITES="cli subprocess watcher incremental httpd ui index_resilience mcp \
|
||||
stack_overflow_a stack_overflow_b stack_overflow_c"
|
||||
is_serial() {
|
||||
case " $SERIAL_SUITES " in *" $1 "*) return 0 ;; *) return 1 ;; esac
|
||||
}
|
||||
PAR_FILE="$LOGDIR/suites-parallel.txt"
|
||||
SER_FILE="$LOGDIR/suites-serial.txt"
|
||||
: > "$PAR_FILE"
|
||||
: > "$SER_FILE"
|
||||
while IFS= read -r sname; do
|
||||
if is_serial "$sname"; then
|
||||
echo "$sname" >> "$SER_FILE"
|
||||
else
|
||||
echo "$sname" >> "$PAR_FILE"
|
||||
fi
|
||||
done < "$SUITES_FILE"
|
||||
echo "=== parallel test run: $NSUITES suites ($(wc -l < "$SER_FILE" | tr -d ' ') serial-tail), $JOBS jobs ==="
|
||||
|
||||
export RUNNER LOGDIR RESULTS_FILE
|
||||
run_one() {
|
||||
s="$1"
|
||||
t0=$SECONDS
|
||||
"$RUNNER" "$s" > "$LOGDIR/$s.log" 2>&1
|
||||
rc=$?
|
||||
secs=$((SECONDS - t0))
|
||||
summary=$(grep -E '^ [0-9]+ passed' "$LOGDIR/$s.log" | tail -1)
|
||||
pass=$(printf '%s' "$summary" | sed -n 's/^ \([0-9]*\) passed.*/\1/p')
|
||||
failn=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) failed.*/\1/p')
|
||||
skip=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) skipped.*/\1/p')
|
||||
# A single short echo line is an atomic append (< PIPE_BUF).
|
||||
echo "$s rc=$rc pass=${pass:-0} fail=${failn:-0} skip=${skip:-0} secs=$secs" >> "$RESULTS_FILE"
|
||||
}
|
||||
export -f run_one
|
||||
|
||||
xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE"
|
||||
# Serial tail: quiet machine for the deadline-sensitive suites.
|
||||
while IFS= read -r sname; do
|
||||
run_one "$sname"
|
||||
done < "$SER_FILE"
|
||||
|
||||
# ── Union guard: every listed suite produced exactly one result ──
|
||||
MISSING=$(comm -23 <(sort "$SUITES_FILE") <(awk '{print $1}' "$RESULTS_FILE" | sort -u))
|
||||
DUPES=$(awk '{print $1}' "$RESULTS_FILE" | sort | uniq -d)
|
||||
if [ -n "$MISSING" ] || [ -n "$DUPES" ]; then
|
||||
echo "FAIL: shard union does not match --list-suites (GATE-QUALITY LOSS)" >&2
|
||||
[ -n "$MISSING" ] && echo " never ran: $MISSING" >&2
|
||||
[ -n "$DUPES" ] && echo " ran twice: $DUPES" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TOTAL_PASS=$(awk -F'pass=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
|
||||
TOTAL_FAIL=$(awk -F'fail=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
|
||||
TOTAL_SKIP=$(awk -F'skip=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
|
||||
BAD_RC=$(grep -cv ' rc=0 ' "$RESULTS_FILE" || true)
|
||||
|
||||
echo "── 8 slowest suites ──"
|
||||
sort -t= -k6 -rn "$RESULTS_FILE" | head -8
|
||||
grep -v ' rc=0 ' "$RESULTS_FILE" || true
|
||||
for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do
|
||||
echo "──── $f: every failure site ────"
|
||||
grep -B2 -A8 "FAIL" "$LOGDIR/$f.log" | head -120
|
||||
echo "──── $f: last 15 lines ────"
|
||||
tail -15 "$LOGDIR/$f.log"
|
||||
done
|
||||
|
||||
echo "────────────────────────────────────────────"
|
||||
echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"
|
||||
echo "────────────────────────────────────────────"
|
||||
|
||||
if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
+11
-2
@@ -57,8 +57,17 @@ verify_compiler "$CC"
|
||||
# Step 1: Clean
|
||||
scripts/clean.sh
|
||||
|
||||
# Step 2 + 3: Build and run tests (Makefile applies $ARCHFLAGS on macOS)
|
||||
make -j"$NPROC" -f Makefile.cbm test $MAKE_ARGS
|
||||
# Step 2 + 3: Build, then run every suite as parallel processes (identical
|
||||
# gate quality — see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh:
|
||||
# the suite set is enumerated from the runner itself and union-guarded, and
|
||||
# pass/fail/skip totals aggregate to the same numbers as the sequential run).
|
||||
# CBM_TEST_SEQUENTIAL=1 restores the single-process runner.
|
||||
make -j"$NPROC" -f Makefile.cbm build/c/test-runner $MAKE_ARGS
|
||||
if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then
|
||||
make -f Makefile.cbm test $MAKE_ARGS
|
||||
else
|
||||
make -f Makefile.cbm test-par $MAKE_ARGS
|
||||
fi
|
||||
|
||||
# Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary
|
||||
# in a subprocess with a wall-clock timeout — a hang must fail, not block the run.
|
||||
|
||||
+24
-2
@@ -1413,6 +1413,10 @@ TEST(cli_vscode_profile_mcp_uninstall) {
|
||||
char *saved_home = save_test_env("HOME");
|
||||
char *saved_path = save_test_env("PATH");
|
||||
char *saved_appdata = save_test_env("APPDATA");
|
||||
char *saved_xdg = save_test_env("XDG_CONFIG_HOME");
|
||||
char xdg_dir[640];
|
||||
snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir);
|
||||
cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolvers prefer XDG */
|
||||
cbm_setenv("HOME", tmpdir, 1);
|
||||
cbm_setenv("PATH", tmpdir, 1);
|
||||
#ifdef _WIN32
|
||||
@@ -1432,6 +1436,7 @@ TEST(cli_vscode_profile_mcp_uninstall) {
|
||||
restore_test_env("HOME", saved_home);
|
||||
restore_test_env("PATH", saved_path);
|
||||
restore_test_env("APPDATA", saved_appdata);
|
||||
restore_test_env("XDG_CONFIG_HOME", saved_xdg);
|
||||
test_rmdir_r(tmpdir);
|
||||
if (rc != 0 || !removed)
|
||||
FAIL("VS Code uninstall must remove MCP entries from every existing profile");
|
||||
@@ -2953,8 +2958,20 @@ TEST(cli_durable_profiles_follow_current_vendor_paths) {
|
||||
FAIL("cbm_mkdtemp failed");
|
||||
|
||||
const char *const env_names[] = {
|
||||
"HOME", "PATH", "CLAUDE_CONFIG_DIR", "CODEX_HOME", "QWEN_HOME",
|
||||
"COPILOT_HOME", "CLINE_DATA_DIR", "KIRO_HOME", "VIBE_HOME", "OPENCODE_CONFIG",
|
||||
"HOME",
|
||||
"PATH",
|
||||
"CLAUDE_CONFIG_DIR",
|
||||
"CODEX_HOME",
|
||||
"QWEN_HOME",
|
||||
"COPILOT_HOME",
|
||||
"CLINE_DATA_DIR",
|
||||
"KIRO_HOME",
|
||||
"VIBE_HOME",
|
||||
"OPENCODE_CONFIG",
|
||||
/* Linux path resolvers prefer $XDG_CONFIG_HOME over $HOME/.config;
|
||||
* CI runners export it, so an isolated-process run resolved OUTSIDE
|
||||
* the fixture home (green only via env leaked from earlier suites). */
|
||||
"XDG_CONFIG_HOME",
|
||||
};
|
||||
char *saved_env[sizeof(env_names) / sizeof(env_names[0])];
|
||||
for (size_t i = 0U; i < sizeof(env_names) / sizeof(env_names[0]); i++) {
|
||||
@@ -7786,7 +7803,12 @@ TEST(cli_detect_agents_finds_zed) {
|
||||
#endif
|
||||
test_mkdirp(dir);
|
||||
|
||||
char *saved_xdg = save_test_env("XDG_CONFIG_HOME");
|
||||
char xdg_dir[640];
|
||||
snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir);
|
||||
cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolver prefers XDG */
|
||||
cbm_detected_agents_t agents = cbm_detect_agents(tmpdir);
|
||||
restore_test_env("XDG_CONFIG_HOME", saved_xdg);
|
||||
ASSERT_TRUE(agents.zed);
|
||||
|
||||
test_rmdir_r(tmpdir);
|
||||
|
||||
+39
-12
@@ -166,11 +166,20 @@ static bool suite_requested(const char *name) {
|
||||
return requested;
|
||||
}
|
||||
|
||||
#define RUN_SELECTED_SUITE(name) \
|
||||
do { \
|
||||
if (suite_requested(#name)) { \
|
||||
RUN_SUITE(name); \
|
||||
} \
|
||||
/* --list-suites: print every registered suite name, one per line, without
|
||||
* running anything. The list and the run share this ONE macro table, so the
|
||||
* list can never drift from what actually executes — shard runners (make
|
||||
* test-par, the CI shard matrix) enumerate suites from here and their union
|
||||
* guard compares against it. */
|
||||
static bool g_list_only = false;
|
||||
|
||||
#define RUN_SELECTED_SUITE(name) \
|
||||
do { \
|
||||
if (g_list_only) { \
|
||||
printf("%s\n", #name); \
|
||||
} else if (suite_requested(#name)) { \
|
||||
RUN_SUITE(name); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* Forward declarations of suite functions */
|
||||
@@ -273,7 +282,9 @@ extern void suite_semantic(void);
|
||||
extern void suite_ast_profile(void);
|
||||
extern void suite_slab_alloc(void);
|
||||
extern void suite_simhash(void);
|
||||
extern void suite_stack_overflow(void);
|
||||
extern void suite_stack_overflow_a(void);
|
||||
extern void suite_stack_overflow_b(void);
|
||||
extern void suite_stack_overflow_c(void);
|
||||
extern void suite_dump_verify(void);
|
||||
extern void suite_dump_verify_io(void);
|
||||
|
||||
@@ -308,16 +319,23 @@ int main(int argc, char **argv) {
|
||||
return 2;
|
||||
}
|
||||
|
||||
g_suite_argc = argc;
|
||||
g_suite_argv = argv;
|
||||
if (argc > 1) {
|
||||
if (argc == 2 && strcmp(argv[1], "--list-suites") == 0) {
|
||||
g_list_only = true;
|
||||
g_suite_argc = 1; /* no suite-name args to match */
|
||||
} else {
|
||||
g_suite_argc = argc;
|
||||
g_suite_argv = argv;
|
||||
}
|
||||
if (g_suite_argc > 1) {
|
||||
g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched));
|
||||
if (!g_suite_arg_matched) {
|
||||
fprintf(stderr, "Failed to allocate test-suite argument tracking\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
printf("\n codebase-memory-mcp C test suite\n");
|
||||
if (!g_list_only) {
|
||||
printf("\n codebase-memory-mcp C test suite\n");
|
||||
}
|
||||
|
||||
/* Foundation */
|
||||
RUN_SELECTED_SUITE(arena);
|
||||
@@ -454,8 +472,11 @@ int main(int argc, char **argv) {
|
||||
RUN_SELECTED_SUITE(ast_profile);
|
||||
RUN_SELECTED_SUITE(simhash);
|
||||
|
||||
/* Stack overflow regression (GitHub #199) */
|
||||
RUN_SELECTED_SUITE(stack_overflow);
|
||||
/* Stack overflow regression (GitHub #199) — split a/b/c so no single
|
||||
* suite serializes a parallel run (each ~1/3 of the old wall time). */
|
||||
RUN_SELECTED_SUITE(stack_overflow_a);
|
||||
RUN_SELECTED_SUITE(stack_overflow_b);
|
||||
RUN_SELECTED_SUITE(stack_overflow_c);
|
||||
|
||||
/* Integration (end-to-end) */
|
||||
RUN_SELECTED_SUITE(integration);
|
||||
@@ -480,6 +501,12 @@ int main(int argc, char **argv) {
|
||||
|
||||
RUN_SELECTED_SUITE(incremental);
|
||||
|
||||
if (g_list_only) {
|
||||
fflush(stdout);
|
||||
cbm_kind_in_set_free_cache();
|
||||
sqlite3_shutdown();
|
||||
return 0;
|
||||
}
|
||||
bool any_suite_matched = false;
|
||||
for (int i = 1; i < g_suite_argc; i++) {
|
||||
any_suite_matched = any_suite_matched || g_suite_arg_matched[i];
|
||||
|
||||
@@ -441,6 +441,16 @@ static bool so_extract_crashes(const char *content, CBMLanguage lang, const char
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
/* Child-side alarm handler: a clean exit on the deadline — never reached on
|
||||
* a crash (SIGSEGV/SIGBUS terminate the child first and stay visible). */
|
||||
static void so_parse_alarm_exit(int sig) {
|
||||
(void)sig;
|
||||
_exit(0);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/* Parse `content` with tree-sitter DIRECTLY in a forked child — bypassing
|
||||
* cbm_extract_file's Perl pre-parse nesting guard — returning true if the child
|
||||
* died by signal. This is the crash-isolating regression for the vendored GLR
|
||||
@@ -471,6 +481,20 @@ static bool so_parse_crashes(const char *content, CBMLanguage lang) {
|
||||
return false;
|
||||
}
|
||||
if (pid == 0) {
|
||||
/* Deterministic ceiling. Falsification data (2026-07-18, macOS
|
||||
* ASan): with CBM_TS_STACK_MERGE_MAX_DEPTH deleted outright, this
|
||||
* branch stayed GREEN both unbounded (220s full grind, no crash)
|
||||
* and bounded — instrumentation shows the recursive merge path is
|
||||
* never even entered here (max depth 0 through the whole window).
|
||||
* The cap-regression DETECTION therefore lives in the Windows
|
||||
* in-process branch above (real ~1 MB native stack, where #913
|
||||
* manifested); this POSIX branch is a bounded no-crash smoke of
|
||||
* the pathological parse, and grinding past 10s adds no signal
|
||||
* (unbounded it wandered 2s-218s with machine state). A real
|
||||
* SIGSEGV/SIGBUS still terminates the child by signal before the
|
||||
* alarm handler can exit cleanly. */
|
||||
signal(SIGALRM, so_parse_alarm_exit);
|
||||
alarm(10);
|
||||
TSParser *parser = ts_parser_new();
|
||||
if (parser) {
|
||||
ts_parser_set_language(parser, ts_lang);
|
||||
@@ -750,7 +774,12 @@ TEST(lsp_kotlin_deep_nesting_no_crash) {
|
||||
* Suite registration
|
||||
* ═══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
SUITE(stack_overflow) {
|
||||
/* Split into three sub-suites so parallel/sharded runs are not serialized
|
||||
* behind one ~4-minute suite (it was the wall-clock critical path: every
|
||||
* other suite finished underneath it). Pure re-registration — the 20
|
||||
* RUN_TEST entries are exactly the ones the single suite carried; the
|
||||
* before/after test-count parity is asserted in the shard runner. */
|
||||
SUITE(stack_overflow_a) {
|
||||
cbm_init();
|
||||
|
||||
RUN_TEST(ts_allocator_bound_to_mimalloc_issue424);
|
||||
@@ -760,6 +789,13 @@ SUITE(stack_overflow) {
|
||||
RUN_TEST(lsp_cpp_deep_expression_no_crash);
|
||||
RUN_TEST(lsp_python_deep_expression_no_crash);
|
||||
RUN_TEST(lsp_perl_deep_expression_no_crash);
|
||||
|
||||
cbm_shutdown();
|
||||
}
|
||||
|
||||
SUITE(stack_overflow_b) {
|
||||
cbm_init();
|
||||
|
||||
RUN_TEST(perl_glr_deep_parse_recursion_capped);
|
||||
RUN_TEST(lsp_ts_cyclic_types_no_crash);
|
||||
RUN_TEST(lsp_python_deep_nesting_no_crash);
|
||||
@@ -767,6 +803,12 @@ SUITE(stack_overflow) {
|
||||
RUN_TEST(lsp_php_deep_nesting_no_crash);
|
||||
RUN_TEST(lsp_kotlin_deep_nesting_no_crash);
|
||||
|
||||
cbm_shutdown();
|
||||
}
|
||||
|
||||
SUITE(stack_overflow_c) {
|
||||
cbm_init();
|
||||
|
||||
RUN_TEST(js_calls_exceed_512);
|
||||
RUN_TEST(python_calls_exceed_512);
|
||||
RUN_TEST(go_calls_exceed_1024);
|
||||
|
||||
Reference in New Issue
Block a user