diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index c7d623bd0..01d10150d 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -115,31 +115,60 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache + # ThreadSanitizer data-race gate. The threaded production surfaces (watcher, + # subprocess, httpd, pipeline, mcp, worker pool, diagnostics, the runnable + # daemon-coordination paths) plus allocator concurrency all run under TSan + # here — see TEST_TSAN_SUITES in Makefile.cbm. Runs on both Linux + # architectures AND native ARM64 macOS: the threading code is shared, so a + # race is usually caught on all three, but scheduler differences mean each + # platform can surface a race the others miss. Windows has no TSan runtime on + # any toolchain (documented irreducible gap); this shared-code coverage is + # its substitute. test-tsan: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, cc: clang, cxx: clang++ } + - { os: ubuntu-24.04-arm, cc: clang, cxx: clang++ } + - { os: macos-14, cc: cc, cxx: c++ } + runs-on: ${{ matrix.os }} timeout-minutes: 120 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Install deps (Ubuntu) + if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y clang zlib1g-dev ccache + - name: Install deps (macOS) + if: runner.os == 'macOS' + run: brew install ccache + - name: Compiler cache (content-verified) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.ccache - key: ccache-tsan-${{ github.ref }}-${{ github.sha }} + key: ccache-tsan-${{ matrix.os }}-${{ github.ref }}-${{ github.sha }} restore-keys: | - ccache-tsan-${{ github.ref }}- + ccache-tsan-${{ matrix.os }}-${{ github.ref }}- - name: ThreadSanitizer tests - # This job calls make directly (no env.sh), so route the compilers - # through ccache's Debian masquerade dir; update-ccache-symlinks in the - # ccache postinst has already linked the clang installed above. + # Calls make directly (no env.sh), so route the compilers through + # ccache's masquerade dir: /usr/lib/ccache on Debian, libexec under the + # Homebrew prefix on macOS. On Linux, reduce mmap ASLR entropy first: + # modern (esp. aarch64) kernels randomize high enough that TSan's + # shadow mapping aborts with "unexpected memory mapping" before any + # test runs. macOS TSan needs no such adjustment. run: | - export PATH=/usr/lib/ccache:$PATH + if [ "$RUNNER_OS" = "macOS" ]; then + export PATH="$(brew --prefix ccache)/libexec:$PATH" + else + export PATH=/usr/lib/ccache:$PATH + sudo sysctl -w vm.mmap_rnd_bits=28 || true + fi export CCACHE_COMPILERCHECK=content - make -f Makefile.cbm test-tsan CC=clang CXX=clang++ + make -f Makefile.cbm test-tsan CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M @@ -209,10 +238,15 @@ jobs: shell: msys2 {0} # AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no # libclang_rt.asan for aarch64-w64-windows-gnu) and cannot intercept the - # system DLLs under x86-64 emulation either, so windows-11-arm runs the - # native ARM64 build with SANITIZE= (no sanitizer) — still a real - # functional gate. ASan/UBSan coverage comes from the other 9 legs, - # including native-ARM Linux/macOS. x86-64 Windows keeps full sanitizers. + # system DLLs under x86-64 emulation either. Instead of running the + # ARM64 leg unsanitized, it runs UBSan in TRAP mode: -fsanitize-trap + # needs no runtime library (the very thing aarch64-w64-windows-gnu + # lacks), so it instruments natively and turns any undefined behavior + # into an illegal-instruction trap; -fstack-protector-strong adds + # stack-smash coverage the heap tools miss. To see WHICH check fired, + # reproduce under the emulated x86_64 UBSan (win.sh ubsan-build), which + # carries the full message. x86-64 Windows keeps full ASan+UBSan; the + # other native-ARM legs (Linux/macOS) carry ASan+LSan+TSan. run: | # Native tests resolve temp via TEMP/TMP (cbm_tmpdir); shell tools use # TMPDIR. Route both through the protected root created above so no @@ -220,7 +254,7 @@ jobs: export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")" export TMP="$TEMP" export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")" - scripts/test.sh CC=clang CXX=clang++ ${{ matrix.os == 'windows-11-arm' && 'SANITIZE=' || '' }} + scripts/test.sh CC=clang CXX=clang++ ${{ matrix.os == 'windows-11-arm' && 'SANITIZE=-fsanitize=undefined -fsanitize-trap=undefined -fstack-protector-strong -fno-omit-frame-pointer' || '' }} env: CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} CCACHE_DIR: ${{ github.workspace }}/.ccache diff --git a/Makefile.cbm b/Makefile.cbm index 7bb700fc4..65f73a394 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -716,7 +716,29 @@ test-repro: $(BUILD_DIR)/test-repro-runner # ── TSan full test ─────────────────────────────────────────────── -TEST_TSAN_SUITES ?= mem slab_alloc parallel +# Every threaded production surface that runs clean AND stable under TSan: +# allocator concurrency (mem, slab_alloc), the parallel extraction worker pool +# (parallel, worker_pool, pipeline), the filesystem watcher (watcher), the +# embedded HTTP server (httpd), diagnostics sampling (diagnostics), the MCP +# server + mutation guard (mcp, mcp_mutation_guard), subprocess supervision +# (subprocess), and the runnable daemon-coordination paths (daemon, +# daemon_application). Was `mem slab_alloc parallel` — a keyhole that ran TSan +# over no real threaded production code. +# +# Deliberately EXCLUDED (documented, not forgotten): +# daemon_runtime — deadlocks under TSan: it forks after going +# multi-threaded and the child re-enters the runtime, a +# combination TSan's runtime does not support (hangs). +# daemon_ipc — a TEST-harness data race (a helper thread writes a +# main-thread stack slot without synchronization, +# test_daemon_ipc.c:2088); production IPC is not implicated. +# daemon_frontend — TEST-harness thread leaks (fixture threads left unjoined, +# test_daemon_frontend.c:698) plus a fork-under-TSan fixture +# failure. Test-quality work, tracked separately. +# Re-enabling these three needs test-harness synchronization + a TSan-fork +# strategy; it is out of scope for the current sanitizer-coverage pass. +TEST_TSAN_SUITES ?= mem slab_alloc parallel worker_pool watcher httpd pipeline \ + diagnostics mcp mcp_mutation_guard subprocess daemon daemon_application TSAN_OPTIONS ?= halt_on_error=1 $(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR) diff --git a/internal/cbm/lsp/scope.h b/internal/cbm/lsp/scope.h index 5aa0498d7..7823f1c2a 100644 --- a/internal/cbm/lsp/scope.h +++ b/internal/cbm/lsp/scope.h @@ -3,7 +3,8 @@ #include "type_rep.h" #include "../arena.h" -#include /* getenv, atoi (cbm_lsp_max_walk_depth) */ +#include /* relaxed cache for cbm_lsp_max_walk_depth */ +#include /* getenv, atoi (cbm_lsp_max_walk_depth) */ typedef struct { const char* name; @@ -40,17 +41,22 @@ typedef struct CBMScope { // Resolved walk-depth cap: env override (CBM_LSP_MAX_WALK_DEPTH, if a positive // integer) else CBM_LSP_MAX_WALK_DEPTH. Read once and cached — the walkers call -// this per node, so it must not hit getenv on the hot path. The cache is a -// benign idempotent race under multi-threaded indexing (every thread computes -// the same value). +// this per node, so it must not hit getenv on the hot path. The cache is +// idempotent under multi-threaded indexing (every worker computes the same +// value), but a plain data race is undefined behavior even when the values +// agree, so the slot is a relaxed atomic: on the hot path this is a plain load +// with no fence, and a first-touch double-compute simply stores the same +// value. This keeps the parallel extractor TSan-clean. static inline int cbm_lsp_max_walk_depth(void) { - static int cached = -1; - if (cached < 0) { + static _Atomic int cached = -1; + int value = atomic_load_explicit(&cached, memory_order_relaxed); + if (value < 0) { const char* e = getenv("CBM_LSP_MAX_WALK_DEPTH"); int v = (e && *e) ? atoi(e) : 0; - cached = (v > 0) ? v : CBM_LSP_MAX_WALK_DEPTH; + value = (v > 0) ? v : CBM_LSP_MAX_WALK_DEPTH; + atomic_store_explicit(&cached, value, memory_order_relaxed); } - return cached; + return value; } CBMScope* cbm_scope_push(CBMArena* a, CBMScope* current); diff --git a/test-infrastructure/docker-compose.yml b/test-infrastructure/docker-compose.yml index 89080e616..342ead06d 100644 --- a/test-infrastructure/docker-compose.yml +++ b/test-infrastructure/docker-compose.yml @@ -39,6 +39,46 @@ services: CCACHE_MAXSIZE: 1500M command: ["CC=gcc", "CXX=g++", "BUILD_DIR=build/linux-amd64"] + # ── ThreadSanitizer (data-race gate) ─────────────────────────── + # Overrides the default test.sh entrypoint to build + run the widened TSan + # suite set. /usr/lib/ccache is first on PATH in the image, so CC=gcc is + # ccache-masqueraded automatically (same as the ASan legs). + test-tsan: + build: + context: .. + dockerfile: test-infrastructure/Dockerfile + platform: linux/arm64 + volumes: + - ..:/src + - cbm-ccache-arm64:/root/.ccache + environment: + CCACHE_DIR: /root/.ccache + CCACHE_MAXSIZE: 1500M + # TSan's shadow memory needs a predictable address space; modern aarch64 + # kernels' high-entropy mmap ASLR makes its runtime abort with "unexpected + # memory mapping" before any test runs. setarch -R (ADDR_NO_RANDOMIZE) + # fixes it, but the personality() syscall it uses is blocked by the default + # seccomp profile — unconfined is safe here (the container runs only our own + # test code). The no-randomize personality inherits to the make child. + security_opt: ["seccomp=unconfined"] + entrypoint: ["setarch", "-R", "make", "-f", "Makefile.cbm", "test-tsan"] + command: ["CC=gcc", "CXX=g++", "BUILD_DIR=build/linux-arm64-tsan"] + + test-tsan-amd64: + build: + context: .. + dockerfile: test-infrastructure/Dockerfile + platform: linux/amd64 + volumes: + - ..:/src + - cbm-ccache-amd64:/root/.ccache + environment: + CCACHE_DIR: /root/.ccache + CCACHE_MAXSIZE: 1500M + security_opt: ["seccomp=unconfined"] + entrypoint: ["setarch", "-R", "make", "-f", "Makefile.cbm", "test-tsan"] + command: ["CC=gcc", "CXX=g++", "BUILD_DIR=build/linux-amd64-tsan"] + # ── Linux production build (-O2 -Werror) ─────────────────── build: build: diff --git a/test-infrastructure/run.sh b/test-infrastructure/run.sh index 017f4ecce..815a3cce7 100755 --- a/test-infrastructure/run.sh +++ b/test-infrastructure/run.sh @@ -29,6 +29,8 @@ # ./test-infrastructure/run.sh windows # Windows cross-compile only # ./test-infrastructure/run.sh test # Linux arm64 test only (no perf) # ./test-infrastructure/run.sh perf # Linux arm64 perf/incremental only +# ./test-infrastructure/run.sh tsan # Linux arm64 ThreadSanitizer race gate +# ./test-infrastructure/run.sh tsan-amd64 # Linux amd64 ThreadSanitizer race gate # ./test-infrastructure/run.sh build # Linux arm64 build only # ./test-infrastructure/run.sh lint # clang-format + cppcheck # ./test-infrastructure/run.sh shell # debug shell @@ -76,6 +78,8 @@ case "${1:-full}" in echo "=== Linux arm64: test + build ===" $COMPOSE run --rm -e CBM_SKIP_PERF=1 test $COMPOSE run --rm build + echo "=== Linux arm64: ThreadSanitizer (data-race gate) ===" + $COMPOSE run --rm test-tsan echo "=== Linux arm64: smoke test ===" $COMPOSE run --rm smoke echo "=== Linux portable: Alpine static build + smoke ===" @@ -92,6 +96,19 @@ case "${1:-full}" in echo "=== Linux arm64: perf/incremental tests ===" $COMPOSE run --rm test ;; + tsan) + echo "=== Linux arm64: ThreadSanitizer (data-race gate) ===" + $COMPOSE run --rm test-tsan + ;; + tsan-amd64) + # NOTE: TSan's shadow memory is incompatible with x86_64-on-ARM + # translation (Rosetta/QEMU), so this FATALs ("unexpected memory + # mapping") on an Apple-Silicon host — it is a real-amd64-hardware / + # GitHub-CI gate, not a local-on-ARM one. ASan amd64 (test-amd64) runs + # fine under Rosetta; only TSan's mapping does not. + echo "=== Linux amd64: ThreadSanitizer (data-race gate; native amd64 only) ===" + $COMPOSE run --rm test-tsan-amd64 + ;; build) echo "=== Linux arm64: production build (-O2 -Werror) ===" $COMPOSE run --rm build @@ -130,6 +147,10 @@ case "${1:-full}" in $COMPOSE run --rm -e CBM_SKIP_PERF=1 test $COMPOSE run --rm build $COMPOSE run --rm smoke + echo "=== Linux arm64: ThreadSanitizer (data-race gate) ===" + $COMPOSE run --rm test-tsan + # amd64 TSan is CI/native-amd64 only (Rosetta can't map TSan shadow); + # run it with `run.sh tsan-amd64` on real amd64. GitHub CI gates it. echo "=== Linux portable: Alpine static build + smoke ===" $COMPOSE run --rm smoke-portable echo "=== Linux amd64: test + build + smoke ===" @@ -153,7 +174,7 @@ case "${1:-full}" in $COMPOSE run --rm --entrypoint bash test-portable ;; *) - echo "Usage: $0 {full|test|perf|build|smoke|portable|portable-test|windows|smoke-windows|soak-windows|amd64|all|lint|shell|shell-alpine}" + echo "Usage: $0 {full|test|perf|tsan|tsan-amd64|build|smoke|portable|portable-test|windows|smoke-windows|soak-windows|amd64|all|lint|shell|shell-alpine}" exit 1 ;; esac diff --git a/test-infrastructure/vm/win.sh b/test-infrastructure/vm/win.sh index f64cf85ed..279cbaa34 100755 --- a/test-infrastructure/vm/win.sh +++ b/test-infrastructure/vm/win.sh @@ -17,6 +17,7 @@ # win.sh push-file # scp one file into the VM (WIP iteration) # win.sh test-par # full suite, parallel on all VM cores # win.sh ubsan-build|ubsan-test # UBSan at CI's x86_64 arch (emulated; works) +# win.sh trap-ubsan-build|-test # NATIVE ARM64 UBSan (trap mode, no runtime) # win.sh pageheap on|off # OS heap verification for native runs set -euo pipefail @@ -153,6 +154,24 @@ ubsan-test) [ $# -ge 1 ] || { echo "usage: win.sh ubsan-test " >&2; exit 2; } vm clang64 "cd /c/cbm && ./build/c/test-runner $* 2>&1 | tail -40" ;; +trap-ubsan-build) + # NATIVE ARM64 UBSan via trap mode. -fsanitize-trap=undefined needs NO + # runtime library (which is exactly what aarch64-w64-windows-gnu lacks), so + # unlike ASan this instruments and runs on native ARM64: a UB hit becomes a + # bare illegal-instruction trap (SIGILL) instead of a diagnostic. Paired + # with -fstack-protector-strong for stack-smash coverage the heap tools + # (PageHeap) miss. This is the native-arch UBSan gate; to see WHICH check + # fired, reproduce under the emulated `win.sh ubsan-build`/`ubsan-test`, + # which carries the full runtime + message. BUILD_DIR isolated so it never + # clobbers the plain test-runner. + vm clangarm64 "cd /c/cbm && make -j${JOBS} -f Makefile.cbm CC='ccache clang' CXX='ccache clang++' SANITIZE='-fsanitize=undefined -fsanitize-trap=undefined -fstack-protector-strong -fno-omit-frame-pointer' BUILD_DIR=build/trap-ubsan build/trap-ubsan/test-runner > /tmp/win-trap-ubsan-build.log 2>&1 && echo TRAP_UBSAN_BUILD_OK || (echo TRAP_UBSAN_BUILD_FAIL; tail -20 /tmp/win-trap-ubsan-build.log; exit 1)" + ;; +trap-ubsan-test) + [ $# -ge 1 ] || { echo "usage: win.sh trap-ubsan-test " >&2; exit 2; } + # A UB trap crashes the runner with SIGILL (exit 132); the harness reports + # the failing suite so the emulated diagnosis loop can name the check. + vm clangarm64 "cd /c/cbm && ./build/trap-ubsan/test-runner $* 2>&1 | tail -40" + ;; pageheap) # OS-level heap verification (page-granular overflow/UAF detection) for the # native ARM64 test-runner — toolchain-agnostic partial ASan substitute.