From 68a9dea3fa0f23154c6850b6acc5c5d29ce6cc09 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 00:42:45 -0500 Subject: [PATCH 1/2] Give the capped Linux test workers the heap they had before the cap --- .github/workflows/ci.yml | 7 +++++-- tests/harness/ci-test-plan.test.ts | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26a38482a..2d3ecbf18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -221,13 +221,16 @@ jobs: # the whole runner did before (a sanitized shard peaks near 16 GB), and # when RAM and swap run out the kernel kills the scope's largest process. # OOMPolicy=continue leaves the rest of the scope running, so vitest - # reports the killed worker and the job log names the test. + # reports the killed worker and the job log names the test. Node sizes + # its default heap from the memory it may use, which the scope lowers + # to about 3.5 GB; the workers keep the 4 GB heap they had on the bare + # runner. - name: Tests (corpus and integration suites) run: | ulimit -c 0 if [ -d /run/systemd/system ]; then exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=14G -p OOMPolicy=continue \ - --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" node scripts/ci-test.mjs + --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" NODE_OPTIONS="--max-old-space-size=4096" node scripts/ci-test.mjs fi node scripts/ci-test.mjs diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index dd77039ca..6cee99370 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -299,7 +299,9 @@ test("pull requests run the full suite on Linux and the darwin suite on macOS", const linuxRun = linux.steps.at(-1)!.run as string; expect(linuxRun.startsWith("ulimit -c 0\n")).toBe(true); expect(linuxRun).toContain("systemd-run --scope --quiet -p MemoryMax=14G -p OOMPolicy=continue"); - expect(linuxRun).toMatch(/-- env PATH="\$PATH" HOME="\$HOME" node scripts\/ci-test\.mjs\n/); + expect(linuxRun).toMatch( + /-- env PATH="\$PATH" HOME="\$HOME" NODE_OPTIONS="--max-old-space-size=4096" node scripts\/ci-test\.mjs\n/, + ); expect(linuxRun.endsWith("fi\nnode scripts/ci-test.mjs\n")).toBe(true); expect(macos.steps.at(-1)!.run).toBe("node scripts/ci-test.mjs"); for (const job of [linux, macos]) expect(job.strategy.matrix.flavor).toEqual(["plain", "san"]); From 937db546561ca427cc85b027f3033239d8164084 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 01:42:23 -0500 Subject: [PATCH 2/2] Raise the capped Linux test scope to 15 GB of RAM The sanitized self-hosting native-frontend build needs about 15 GB by itself: its clang compile of the whole frontend under ASan peaks at 10.5 GB beside a 3.8 GB compile in Node. In the 14 GB scope, with the shard's other work running beside it, the kernel killed that clang intermittently, so SC5004 reported clang's warnings and no error (SAN 16/20 and 19/20). The scope now caps RAM at 15 GB, still below the runner's 16 GB, and may still swap past it. --- .github/workflows/ci.yml | 12 +++++++----- tests/harness/ci-test-plan.test.ts | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d3ecbf18..edb2b5731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,19 +217,21 @@ jobs: # worker budget covers both processes; file and case shard axes stay # separate so every fixture still runs exactly once in each lane. # The tests run in a transient scope capped below the runner's 16 GB, so - # the runner itself keeps memory: past 14 GB of RAM the scope swaps, as - # the whole runner did before (a sanitized shard peaks near 16 GB), and - # when RAM and swap run out the kernel kills the scope's largest process. + # the runner itself keeps memory: past 15 GB of RAM the scope swaps, as + # the whole runner did before, and when RAM and swap run out the kernel + # kills the scope's largest process. A sanitized shard peaks near 16 GB, + # and the sanitized self-hosting native-frontend build alone needs 15 GB + # (its clang compile of the frontend takes 10.5 GB). # OOMPolicy=continue leaves the rest of the scope running, so vitest # reports the killed worker and the job log names the test. Node sizes # its default heap from the memory it may use, which the scope lowers - # to about 3.5 GB; the workers keep the 4 GB heap they had on the bare + # below 4 GB; the workers keep the 4 GB heap they had on the bare # runner. - name: Tests (corpus and integration suites) run: | ulimit -c 0 if [ -d /run/systemd/system ]; then - exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=14G -p OOMPolicy=continue \ + exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=15G -p OOMPolicy=continue \ --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" NODE_OPTIONS="--max-old-space-size=4096" node scripts/ci-test.mjs fi node scripts/ci-test.mjs diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 6cee99370..29268c841 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -298,7 +298,7 @@ test("pull requests run the full suite on Linux and the darwin suite on macOS", // runner, and directly elsewhere. const linuxRun = linux.steps.at(-1)!.run as string; expect(linuxRun.startsWith("ulimit -c 0\n")).toBe(true); - expect(linuxRun).toContain("systemd-run --scope --quiet -p MemoryMax=14G -p OOMPolicy=continue"); + expect(linuxRun).toContain("systemd-run --scope --quiet -p MemoryMax=15G -p OOMPolicy=continue"); expect(linuxRun).toMatch( /-- env PATH="\$PATH" HOME="\$HOME" NODE_OPTIONS="--max-old-space-size=4096" node scripts\/ci-test\.mjs\n/, );