Merge pull request #1465 from DeusData/feat/ci-run-isolation

fix(ci): isolate local-CI runs per run id, and tidy what each run creates
This commit is contained in:
Martin Vogel
2026-08-06 04:23:45 +02:00
committed by GitHub
5 changed files with 177 additions and 24 deletions
+8
View File
@@ -319,5 +319,13 @@ volumes:
# native filesystem: object writes over the virtiofs bind mount are the
# container legs' largest avoidable I/O cost, and the fixture cache makes
# the network clone once-per-volume instead of once-per-container.
# The build volume is the ONLY per-run-mutable one: concurrent legs writing
# the same build dir clobber each other's objects and test-logs (a scheduler
# then fails reading a suite log that another run replaced). run.sh gives each
# run its own volume via CBM_CI_BUILD_VOLUME and removes it afterwards; the
# default name preserves the single-run fast path. ccache and the fixture
# cache stay SHARED on purpose — ccache is concurrency-safe and
# content-verified, and sharing them is what keeps isolated runs fast.
cbm-build:
name: ${CBM_CI_BUILD_VOLUME:-cbm-build}
cbm-fixture-cache:
+43 -1
View File
@@ -56,6 +56,46 @@
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
# ── Per-run isolation (concurrent agents / multiple worktrees) ───────────────
# Container names are already unique per `compose run --rm`, but the BUILD
# volume was not: two legs running at once wrote the same /src/build, so one
# run's objects and test-logs replaced the other's — after which the parallel
# scheduler dies reading a suite log that another run had removed. Each run now
# gets its own build volume, keyed by a unique run id. ccache and the fixture
# cache stay SHARED on purpose: ccache is concurrency-safe and content-verified,
# and sharing them is what keeps an isolated run fast rather than cold.
#
# CBM_CI_RUN_ID=<id> name/reuse a run (default: pid + epoch, unique)
# CBM_CI_SHARED_BUILD=1 opt back into the single shared `cbm-build` volume
# CBM_CI_KEEP=1 keep this run's build volume even on success
RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}"
if [ "${CBM_CI_SHARED_BUILD:-0}" = "1" ]; then
CBM_CI_BUILD_VOLUME="cbm-build"
else
CBM_CI_BUILD_VOLUME="cbm-build-${RUN_ID}"
fi
export CBM_CI_BUILD_VOLUME
# Tidy what this run generated. A FAILED run KEEPS its volume — those artifacts
# are the post-mortem — and prints how to inspect and drop it. Successful runs
# leave nothing behind. Shared-volume mode never removes anything.
ci_cleanup() {
local rc=$?
if [ "${CBM_CI_BUILD_VOLUME}" = "cbm-build" ] || [ "${CBM_CI_KEEP:-0}" = "1" ]; then
return $rc
fi
if [ $rc -eq 0 ]; then
docker volume rm -f "${CBM_CI_BUILD_VOLUME}" >/dev/null 2>&1 || true
else
echo "run.sh: kept ${CBM_CI_BUILD_VOLUME} for post-mortem (run failed)" >&2
echo " inspect: docker run --rm -v ${CBM_CI_BUILD_VOLUME}:/b alpine ls -R /b" >&2
echo " drop: docker volume rm ${CBM_CI_BUILD_VOLUME}" >&2
fi
return $rc
}
trap ci_cleanup EXIT
COMPOSE="docker compose -f $ROOT/test-infrastructure/docker-compose.yml"
usage() {
@@ -229,7 +269,9 @@ case "${1:-full}" in
echo "=== Windows: binary version check (cross-compile + Wine) ==="
$COMPOSE run --rm smoke-windows
;;
amd64)
amd64 | test-amd64)
# `--help` advertises `amd64|test-amd64`, but only `amd64` dispatched:
# the documented spelling died with "unknown leg 'test-amd64'".
echo "=== Linux amd64: test + build ==="
$COMPOSE run --rm -e CBM_SKIP_PERF=1 test-amd64
$COMPOSE run --rm build-amd64
+33 -3
View File
@@ -46,7 +46,25 @@ case "${1:-}" in
esac
RUNNER="${CBM_VM_RUNNER:-}"
LOG="${CBM_VM_TEST_LOG:-/tmp/win-test.log}"
# Per-run identity. The VM holds ONE checkout (/c/cbm), so two concurrent runs
# shared a FIXED log path and the same build dir: the later run's output
# replaced the earlier one's, and -PruneStale could delete a live run's temp
# root out from under it. The run id namespaces the log and the build dir; the
# protected temp root is already unique per run.
CALLER_RUN_ID="${CBM_CI_RUN_ID:-}"
RUN_ID="${CBM_CI_RUN_ID:-$$-$(date +%s)}"
export CBM_CI_RUN_ID="$RUN_ID"
LOG="${CBM_VM_TEST_LOG:-/tmp/win-test-${RUN_ID}.log}"
# A caller that sets CBM_CI_RUN_ID is declaring concurrency, so give that run
# its own BUILD_DIR; the default single-run path keeps the shared one and its
# incremental reuse (a per-run build dir on the VM costs a full rebuild).
if [ -n "$CALLER_RUN_ID" ]; then
BUILD_ARGS=("BUILD_DIR=build/vm-${RUN_ID}")
else
BUILD_ARGS=()
fi
[ $# -ge 1 ] || { echo "vm-run-tests: missing arguments. Please consult --help." >&2; exit 2; }
if [ "$1" = "--soak" ]; then
@@ -130,13 +148,25 @@ if [ -n "$RUNNER" ]; then
rc="${PIPESTATUS[0]}"
fi
elif [ "$1" = "--par" ]; then
scripts/test.sh CC=clang CXX=clang++ 2>&1 | tee "$LOG"
scripts/test.sh CC=clang CXX=clang++ \
${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG"
rc="${PIPESTATUS[0]}"
else
scripts/test.sh --suites "$*" CC=clang CXX=clang++ 2>&1 | tee "$LOG"
scripts/test.sh --suites "$*" CC=clang CXX=clang++ \
${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} 2>&1 | tee "$LOG"
rc="${PIPESTATUS[0]}"
fi
# Tidy this run's own build dir on success; keep it on failure as the
# post-mortem. Only ever touches a dir this run created (concurrent mode).
if [ -n "$CALLER_RUN_ID" ] && [ "${CBM_CI_KEEP:-0}" != "1" ]; then
if [ "${rc:-1}" -eq 0 ]; then
rm -rf "build/vm-${RUN_ID}"
else
echo "vm-run-tests: kept build/vm-${RUN_ID} and $LOG for post-mortem" >&2
fi
fi
if ! grep -Eq '[0-9]+ passed' "$LOG"; then
echo "GUARD: test runner produced no completion summary — the suites did" \
"not validly run; treating as failure (runner rc=$rc)" >&2
+85 -18
View File
@@ -88,6 +88,24 @@ ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
[ -f "$CONFIG" ] && . "$CONFIG"
HOST="${CBM_VM_HOST:?set CBM_VM_HOST in ~/.claude/cbm-vm/config}"
USER_="${CBM_VM_USER:-test}"
# ── Per-run checkout (the VM's last shared-state hazard) ─────────────────────
# The VM holds ONE working tree at /c/cbm, and `update`/`sync` REPLACE it
# (`git reset --hard` + `clean -fdx`). A second session syncing its own branch
# therefore swaps the code under a running leg — observed 2026-08-06, where a
# gate run on main compiled another branch's files and died on a header that
# does not exist on Windows. That is worse than a red: it is a confident wrong
# answer about the wrong tree.
#
# Setting CBM_CI_RUN_ID gives this run its own checkout, cloned from /c/cbm.
# A local clone hardlinks .git/objects, so it costs seconds and little disk.
# Unset (the default) keeps the historical single-tree behaviour.
VM_BASE_REPO="/c/cbm"
if [ -n "${CBM_CI_RUN_ID:-}" ]; then
VM_REPO="/c/cbm-run-${CBM_CI_RUN_ID}"
else
VM_REPO="$VM_BASE_REPO"
fi
HOST_KEY="${CBM_VM_HOST_KEY_SHA256:?set CBM_VM_HOST_KEY_SHA256 in ~/.claude/cbm-vm/config}"
LOCAL_BRANCH="$(git -C "$ROOT" branch --show-current)"
BRANCH="${CBM_VM_BRANCH:-${LOCAL_BRANCH:-main}}"
@@ -117,6 +135,38 @@ SCP=(scp "${SSH_OPTIONS[@]}")
vm() { local env="$1"; shift
"${SSH[@]}" "C:\\msys64\\msys2_shell.cmd -defterm -no-start -${env} -c \"set -e -o pipefail; $*\""; }
# Create this run's own checkout if it does not exist yet. A LOCAL clone
# hardlinks .git/objects, so it is seconds and near-zero disk rather than a
# fresh network clone. No-op when running in the shared (default) tree.
vm_ensure_run_checkout() {
[ "$VM_REPO" = "$VM_BASE_REPO" ] && return 0
# `git clone --local` points origin at the base PATH, so a later
# `fetch origin <branch>` would resolve against whatever the base happens
# to hold (i.e. another session's state) instead of GitHub. Inherit the
# base's real origin URL so the isolated tree fetches independently.
# The substitution is single-quoted here so it expands in the REMOTE shell
# (same pattern as JOBS above); nested double quotes do not survive the
# cmd.exe -> msys2_shell wrapper.
local base_url_expr='$(git -C '"$VM_BASE_REPO"' remote get-url origin)'
vm clangarm64 "[ -d '$VM_REPO/.git' ] || git clone --local '$VM_BASE_REPO' '$VM_REPO'"
vm clangarm64 "git -C '$VM_REPO' remote set-url origin $base_url_expr"
}
# The VM tree must be the one the caller means. A concurrent session running
# `update`/`sync` in the SHARED tree swaps it mid-flight, and the leg then
# reports a confident verdict about someone else's branch. Callers assert the
# expected HEAD; a mismatch fails loudly instead of producing that verdict.
vm_assert_head() {
local want="$1" got
got="$(vm clangarm64 "cd $VM_REPO && git rev-parse --verify HEAD" | tr -d '\r')"
if [ "$got" != "$want" ]; then
echo "FATAL: VM tree $VM_REPO is at $got, expected $want." >&2
echo " Another session likely re-synced the shared checkout." >&2
echo " Re-run with CBM_CI_RUN_ID set to get an isolated tree." >&2
return 1
fi
}
vm_cmd() { "${SSH[@]}" "$@"; } # plain cmd.exe (CI-shaped environment)
# A GitHub runner is ephemeral: every job starts on a fresh image with a
@@ -157,13 +207,25 @@ esac
case "$cmd" in
status)
"${SSH[@]}" "echo VM_REACHABLE & ver"
vm clangarm64 "cd /c/cbm 2>/dev/null && git log --oneline -1 && ls -la build/c/codebase-memory-mcp.exe build/c/test-runner.exe 2>/dev/null || echo 'repo/build missing — run provision-windows.sh'"
vm clangarm64 "cd $VM_REPO 2>/dev/null && git log --oneline -1 && ls -la build/c/codebase-memory-mcp.exe build/c/test-runner.exe 2>/dev/null || echo 'repo/build missing — run provision-windows.sh'"
;;
update)
vm clangarm64 "cd /c/cbm && git fetch origin ${BRANCH} && git reset --hard FETCH_HEAD && git clean -fdx && git log --oneline -1"
vm_ensure_run_checkout
vm clangarm64 "cd $VM_REPO && git fetch origin ${BRANCH} && git reset --hard FETCH_HEAD && git clean -fdx && git log --oneline -1"
exec "$0" build
;;
drop-run-checkout)
# Explicit tidy-up of THIS run's checkout (no-op in shared mode). The run
# that created it removes it; nothing else may, since another session's
# tree must never be deleted out from under it.
if [ "$VM_REPO" = "$VM_BASE_REPO" ]; then
echo "win.sh: shared checkout ($VM_BASE_REPO) — nothing to drop" >&2
else
vm clangarm64 "rm -rf '$VM_REPO' && echo dropped $VM_REPO"
fi
;;
sync)
vm_ensure_run_checkout
local_head="$(git -C "$ROOT" rev-parse --verify HEAD)"
WIN_MANIFEST="$(mktemp "${TMPDIR:-/tmp}/cbm-vm-manifest.XXXXXX")"
WIN_ARCHIVE="$(mktemp "${TMPDIR:-/tmp}/cbm-vm-worktree.XXXXXX.tar")"
@@ -172,25 +234,25 @@ sync)
cbm_vm_write_untracked_manifest "$ROOT" "$WIN_MANIFEST"
COPYFILE_DISABLE=1 tar --no-xattrs --no-mac-metadata \
-C "$ROOT" --null -T "$WIN_MANIFEST" -cf "$WIN_ARCHIVE"
remote_head="$(vm clangarm64 "cd /c/cbm && git rev-parse --verify HEAD")"
remote_head="$(vm clangarm64 "cd $VM_REPO && git rev-parse --verify HEAD")"
remote_head="${remote_head//$'\r'/}"
if [ "$remote_head" != "$local_head" ]; then
echo "FATAL: Windows VM is at $remote_head, expected local HEAD $local_head; run win.sh update first." >&2
exit 1
fi
vm clangarm64 \
"cd /c/cbm && git reset --hard '$local_head' && git clean -fdx"
"cd $VM_REPO && git reset --hard '$local_head' && git clean -fdx"
if [ -s "$WIN_PATCH" ]; then
"${SSH[@]}" \
'C:\msys64\msys2_shell.cmd -defterm -no-start -clangarm64 -c "cd /c/cbm && git apply --binary --whitespace=nowarn -"' \
'C:\msys64\msys2_shell.cmd -defterm -no-start -clangarm64 -c "cd $VM_REPO && git apply --binary --whitespace=nowarn -"' \
<"$WIN_PATCH"
fi
if [ -s "$WIN_MANIFEST" ]; then
"${SSH[@]}" \
'C:\msys64\msys2_shell.cmd -defterm -no-start -clangarm64 -c "cd /c/cbm && tar -xf -"' \
'C:\msys64\msys2_shell.cmd -defterm -no-start -clangarm64 -c "cd $VM_REPO && tar -xf -"' \
<"$WIN_ARCHIVE"
fi
vm clangarm64 "cd /c/cbm && git status --short --branch"
vm clangarm64 "cd $VM_REPO && git status --short --branch"
win_cleanup
exec "$0" build
;;
@@ -201,7 +263,7 @@ build)
# NOT via CC='ccache clang', which bypassed the verified-cache env layer.
# The test-runner is no longer built here: the test leg (scripts/test.sh)
# builds its own runner, same as CI's test jobs.
vm clangarm64 "cd /c/cbm && scripts/build.sh CC=clang CXX=clang++ > /tmp/win-build.log 2>&1 && echo BUILD_OK || (echo BUILD_FAIL; tail -20 /tmp/win-build.log; exit 1)"
vm clangarm64 "cd $VM_REPO && scripts/build.sh CC=clang CXX=clang++ > /tmp/win-build.log 2>&1 && echo BUILD_OK || (echo BUILD_FAIL; tail -20 /tmp/win-build.log; exit 1)"
;;
test)
[ $# -ge 1 ] || { echo "usage: win.sh test <suite...>" >&2; exit 2; }
@@ -210,7 +272,12 @@ test)
# protected per-user temp root via vm-run-tests.sh, which streams FULL
# output and refuses to report success without the runner's completion
# summary. A `| tail -40` here once hid 40 real Windows failures.
vm clangarm64 "cd /c/cbm && bash test-infrastructure/vm/vm-run-tests.sh $*"
# CBM_VM_EXPECT_HEAD=<sha> asserts the tree is the one the caller means
# before spending a leg on it — cheap insurance against a concurrent
# session having re-synced the shared checkout.
[ -z "${CBM_VM_EXPECT_HEAD:-}" ] || vm_assert_head "$CBM_VM_EXPECT_HEAD"
vm clangarm64 "cd $VM_REPO && bash test-infrastructure/vm/vm-run-tests.sh $*"
[ -z "${CBM_VM_EXPECT_HEAD:-}" ] || vm_assert_head "$CBM_VM_EXPECT_HEAD"
;;
guards)
# Match the Windows CI product build: a clean, embedded-UI product binary.
@@ -222,7 +289,7 @@ guards)
# needs the official MSVC Node.js (/c/node, from provision-windows.sh):
# MSVC-built npm native modules cannot resolve Node-API symbols against
# MSYS2's mingw node.
vm clangarm64 "export PATH=/c/node:\$PATH && cd /c/cbm && scripts/build.sh --with-ui CC=clang CXX=clang++ SANITIZE= BUILD_DIR=build/guards"
vm clangarm64 "export PATH=/c/node:\$PATH && cd $VM_REPO && scripts/build.sh --with-ui CC=clang CXX=clang++ SANITIZE= BUILD_DIR=build/guards"
# The guards themselves run through plain cmd/PowerShell, exactly like the
# CI job: under the MSYS2 shell TMP is C:\msys64\tmp, whose ancestry
# grants mutation rights to Authenticated Users — the daemon's
@@ -238,14 +305,14 @@ smoke-install)
# build/c (including any test-runner) just as CI's separate jobs imply;
# ccache keeps the rebuild to minutes — parity was the explicit user call
# over incremental speed here.
vm clangarm64 "cd /c/cbm && scripts/build.sh CC=clang CXX=clang++ && bash test-infrastructure/vm/vm-smoke.sh"
vm clangarm64 "cd $VM_REPO && scripts/build.sh CC=clang CXX=clang++ && bash test-infrastructure/vm/vm-smoke.sh"
;;
smoke-artifact)
# The release-archive flow with local bytes: canonical build → the ONE
# package-release.sh → extract → vm-smoke.sh in artifact mode — the same
# sequence _build.yml + _smoke.yml run remotely, so archive-layout bugs
# surface here instead of in a release dry run.
vm clangarm64 "cd /c/cbm && bash scripts/ci/smoke-artifact.sh windows arm64 CC=clang CXX=clang++"
vm clangarm64 "cd $VM_REPO && bash scripts/ci/smoke-artifact.sh windows arm64 CC=clang CXX=clang++"
;;
soak)
duration="${1:-10}"
@@ -256,7 +323,7 @@ soak)
echo "usage: win.sh soak [positive-minutes]" >&2
exit 2
fi
vm clangarm64 "cd /c/cbm && CBM_VM_TEST_LOG=/tmp/win-soak.log bash \
vm clangarm64 "cd $VM_REPO && CBM_VM_TEST_LOG=/tmp/win-soak.log bash \
test-infrastructure/vm/vm-run-tests.sh --soak '$duration'"
;;
sh)
@@ -273,11 +340,11 @@ ubsan-build)
# Validated: UBSan needs no interceptors, so it builds, runs, AND reports
# correctly under emulation. (ASan does NOT: no aarch64 runtime exists and
# the x86_64 runtime faults in emulated process-init — ASan stays CI-only.)
vm clang64 "cd /c/cbm && make -j${JOBS} -f Makefile.cbm CC=clang CXX=clang++ SANITIZE='-fsanitize=undefined -fno-omit-frame-pointer' build/c/test-runner > /tmp/win-ubsan-build.log 2>&1 && echo UBSAN_BUILD_OK || (echo UBSAN_BUILD_FAIL; tail -20 /tmp/win-ubsan-build.log; exit 1)"
vm clang64 "cd $VM_REPO && make -j${JOBS} -f Makefile.cbm CC=clang CXX=clang++ SANITIZE='-fsanitize=undefined -fno-omit-frame-pointer' build/c/test-runner > /tmp/win-ubsan-build.log 2>&1 && echo UBSAN_BUILD_OK || (echo UBSAN_BUILD_FAIL; tail -20 /tmp/win-ubsan-build.log; exit 1)"
;;
ubsan-test)
[ $# -ge 1 ] || { echo "usage: win.sh ubsan-test <suite...>" >&2; exit 2; }
vm clang64 "cd /c/cbm && CBM_VM_TEST_LOG=/tmp/win-ubsan-test.log bash test-infrastructure/vm/vm-run-tests.sh $*"
vm clang64 "cd $VM_REPO && CBM_VM_TEST_LOG=/tmp/win-ubsan-test.log bash test-infrastructure/vm/vm-run-tests.sh $*"
;;
trap-ubsan-build)
# NATIVE ARM64 UBSan via trap mode. -fsanitize-trap=undefined needs NO
@@ -289,13 +356,13 @@ trap-ubsan-build)
# fired, reproduce under the emulated `win.sh ubsan-build`/`ubsan-test`,
# which carries the full runtime + message. BUILD_DIR isolated so it never
# clobbers the plain test-runner.
vm clangarm64 "cd /c/cbm && make -j${JOBS} -f Makefile.cbm CC='ccache clang' CXX='ccache clang++' SANITIZE='-fsanitize=undefined -fsanitize-trap=undefined -fstack-protector-strong -fno-omit-frame-pointer' BUILD_DIR=build/trap-ubsan build/trap-ubsan/test-runner > /tmp/win-trap-ubsan-build.log 2>&1 && echo TRAP_UBSAN_BUILD_OK || (echo TRAP_UBSAN_BUILD_FAIL; tail -20 /tmp/win-trap-ubsan-build.log; exit 1)"
vm clangarm64 "cd $VM_REPO && make -j${JOBS} -f Makefile.cbm CC='ccache clang' CXX='ccache clang++' SANITIZE='-fsanitize=undefined -fsanitize-trap=undefined -fstack-protector-strong -fno-omit-frame-pointer' BUILD_DIR=build/trap-ubsan build/trap-ubsan/test-runner > /tmp/win-trap-ubsan-build.log 2>&1 && echo TRAP_UBSAN_BUILD_OK || (echo TRAP_UBSAN_BUILD_FAIL; tail -20 /tmp/win-trap-ubsan-build.log; exit 1)"
;;
trap-ubsan-test)
[ $# -ge 1 ] || { echo "usage: win.sh trap-ubsan-test <suite...>" >&2; exit 2; }
# A UB trap crashes the runner with SIGILL (exit 132); the harness reports
# the failing suite so the emulated diagnosis loop can name the check.
vm clangarm64 "cd /c/cbm && CBM_VM_RUNNER=build/trap-ubsan/test-runner CBM_VM_TEST_LOG=/tmp/win-trap-ubsan-test.log bash test-infrastructure/vm/vm-run-tests.sh $*"
vm clangarm64 "cd $VM_REPO && CBM_VM_RUNNER=build/trap-ubsan/test-runner CBM_VM_TEST_LOG=/tmp/win-trap-ubsan-test.log bash test-infrastructure/vm/vm-run-tests.sh $*"
;;
pageheap)
# OS-level heap verification (page-granular overflow/UAF detection) for the
@@ -318,7 +385,7 @@ test-par)
# under the CI-shaped protected temp root with FULL output (this leg once
# ran under the MSYS-shared /tmp and piped through `tail -25` — the same
# truncated-blindness class that hid 40 Windows failures from `test`).
vm clangarm64 "cd /c/cbm && CBM_VM_TEST_LOG=/tmp/win-test-par.log bash test-infrastructure/vm/vm-run-tests.sh --par"
vm clangarm64 "cd $VM_REPO && CBM_VM_TEST_LOG=/tmp/win-test-par.log bash test-infrastructure/vm/vm-run-tests.sh --par"
;;
help | -h | --help)
print_help
+8 -2
View File
@@ -537,8 +537,14 @@ require(
)
require(
sync_case is not None
and 'remote_head="$(vm clangarm64 "cd /c/cbm && git rev-parse --verify HEAD")"'
in sync_case.group("body")
# Assert the PROPERTY, not one spelling of it: capture the remote HEAD into
# a local variable and compare it here. The checkout path became a variable
# (per-run isolation), so pinning the literal `/c/cbm` was asserting the
# implementation rather than the contract it exists to protect.
and re.search(
r'remote_head="\$\(vm clangarm64 "cd \S+ && git rev-parse --verify HEAD"\)"',
sync_case.group("body"),
)
and 'test \\"\\$(git rev-parse --verify HEAD)\\"' not in sync_case.group("body"),
"win.sh sync must compare the remote HEAD locally instead of nesting shell quotes through "
"cmd.exe",