diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3bf0e770..5ab1d9331 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,9 +216,16 @@ jobs: # Start the corpus immediately beside a serial side queue. One shared # worker budget covers both processes; file and case shard axes stay # separate so every fixture still runs exactly once in each lane. + # The tests run in a transient scope capped below the runner's 16 GB: + # past 13 GB the kernel kills the scope's largest process, so the job + # log names the test instead of the runner itself running out of memory. - name: Tests (corpus and integration suites) run: | ulimit -c 0 + if [ -d /run/systemd/system ]; then + exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=13G -p MemorySwapMax=0 \ + --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" node scripts/ci-test.mjs + fi node scripts/ci-test.mjs tests_macos: diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 2b76b52d6..184f04d1c 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -42,22 +42,38 @@ export const testCosts = JSON.parse( readFileSync(new URL("./ci-test-costs.json", import.meta.url), "utf8"), ); +// Side-queue files whose process tree, run alone, peaks far above a plain +// vitest run (about 1 GB): megabytes at the peak, measured with CI's settings. +// They run beside the corpus task's three workers, so no two of them share a +// shard while there are shards without one. +export const memoryHeavyFiles = { + "tests/harness/decl-boundary.test.ts": 1838, +}; + // The shard (1-based) of each measured side-queue file and isolated suite: // longest first to the shard with the least work so far, counting about a // second per file for vitest to import it. Ties go to the highest shard, so the // longest files start from the last shard while the cost-balanced corpus and // three.js cases (tests/harness/shard.ts) place their longest from the first. -// An isolated suite without a measurement weighs as much as the median -// measured file. Files without a measurement are left to vitest's --shard, -// which splits them by count. +// An isolated suite or memory-heavy file without a measurement weighs as much +// as the median measured file. Memory-heavy files are placed after the others, +// each on the shard with the least memory-heavy work so far and, among those, +// the least work. Files without a +// measurement are left to vitest's --shard, which splits them by count. export function sideOwners(flavor, count) { const costs = testCosts[flavor]; const known = Object.values(costs).sort((a, b) => a - b); const median = known.length === 0 ? 0 : known[Math.floor(known.length / 2)]; const weight = (file) => (costs[file] ?? median) + 1000; - const files = [...new Set([...Object.keys(costs), ...isolatedFiles.map(([, file]) => file)])]; + const heavy = Object.keys(memoryHeavyFiles).sort( + (a, b) => memoryHeavyFiles[b] - memoryHeavyFiles[a] || (a < b ? -1 : a > b ? 1 : 0), + ); + const files = [ + ...new Set([...Object.keys(costs), ...isolatedFiles.map(([, file]) => file)]), + ].filter((file) => !(file in memoryHeavyFiles)); files.sort((a, b) => weight(b) - weight(a) || (a < b ? -1 : a > b ? 1 : 0)); const loads = new Array(count).fill(0); + const memory = new Array(count).fill(0); const owners = new Map(); for (const file of files) { let lightest = count - 1; @@ -66,6 +82,18 @@ export function sideOwners(flavor, count) { loads[lightest] += weight(file); owners.set(file, lightest + 1); } + for (const file of heavy) { + let lightest = count - 1; + for (let shard = count - 2; shard >= 0; shard--) + if ( + memory[shard] < memory[lightest] || + (memory[shard] === memory[lightest] && loads[shard] < loads[lightest]) + ) + lightest = shard; + memory[lightest] += memoryHeavyFiles[file]; + loads[lightest] += weight(file); + owners.set(file, lightest + 1); + } return owners; } diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 34d61346f..9ff730ddf 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -11,6 +11,8 @@ import { caseShardedSideFiles, isolatedFiles, leakSuppressions, + memoryHeavyFiles, + sideOwners, testCosts, } from "../../scripts/ci-test-plan.mjs"; import { runCiTestPlan } from "../../scripts/ci-test.mjs"; @@ -40,20 +42,27 @@ test.for([1, 3, 5, 10, 20])( file, ).toHaveLength(everyShard ? count : 1); } - // Each measured file runs on exactly one shard, outside vitest's file shard. + // Each measured or memory-heavy file runs on exactly one shard, outside + // vitest's file shard. const isolated = new Set(isolatedFiles.map(([, file]) => file)); const balanced = plans.flatMap((plan) => plan.side .filter((task) => task.name === "cost-balanced tests") .flatMap((task) => task.args.slice(2)), ); - const measured = Object.keys(testCosts[flavor]).filter((file) => !isolated.has(file)); + const measured = [ + ...new Set([...Object.keys(testCosts[flavor]), ...Object.keys(memoryHeavyFiles)]), + ].filter((file) => !isolated.has(file)); expect(measured.length).toBeGreaterThan(0); expect(balanced.toSorted()).toEqual(measured.toSorted()); for (const plan of plans) { expect(plan.corpus.args).toEqual(["test", ...corpusFiles]); expect(plan.side[0]!.args).toContain(`--shard=${plan.corpus.env.SCRIPTC_TEST_SHARD}`); - for (const file of [...fileExclusions, ...Object.keys(testCosts[flavor])]) + for (const file of [ + ...fileExclusions, + ...Object.keys(testCosts[flavor]), + ...Object.keys(memoryHeavyFiles), + ]) expect(plan.side[0]!.args).toContain(`--exclude=${file}`); const cache = plan.side.find((task) => task.name === "native-cache correctness"); expect(Boolean(cache)).toBe(flavor === "plain"); @@ -103,6 +112,26 @@ test("the side queue balances measured files across the twenty Linux shards", () } }); +test("memory-heavy side files run whole in the cost-balanced task, on shards of their own", () => { + for (const flavor of ["plain", "san"] as const) { + const owners = sideOwners(flavor, 20); + const heavy = Object.keys(memoryHeavyFiles); + const shards = heavy.map((file) => owners.get(file)); + expect(shards.every((shard) => shard !== undefined)).toBe(true); + expect(new Set(shards).size).toBe(Math.min(heavy.length, 20)); + for (const file of heavy) { + const plan = ciTestPlan({ shard: `${owners.get(file)}/20`, flavor }); + const balanced = plan.side.find((task) => task.name === "cost-balanced tests")!; + expect(balanced.args).toContain(file); + // The file-sharded task leaves it out on every shard. + for (let shard = 1; shard <= 20; shard++) + expect(ciTestPlan({ shard: `${shard}/20`, flavor }).side[0]!.args).toContain( + `--exclude=${file}`, + ); + } + } +}); + test.for([1, 2, 4, 8])( "CI overlaps reserved lanes within a budget of %i workers", async (workers) => { @@ -265,7 +294,13 @@ test("pull requests run the full suite on Linux and the darwin suite on macOS", ); expect(coreStep).toBeGreaterThan(-1); expect(coreStep).toBeLessThan(linux.steps.length - 1); - expect(linux.steps.at(-1)!.run).toBe("ulimit -c 0\nnode scripts/ci-test.mjs\n"); + // Linux runs the suites in a memory-capped scope where systemd manages the + // runner, and directly elsewhere. + const linuxRun = linux.steps.at(-1)!.run as string; + expect(linuxRun.startsWith("ulimit -c 0\n")).toBe(true); + expect(linuxRun).toContain("systemd-run --scope --quiet -p MemoryMax=13G -p MemorySwapMax=0"); + expect(linuxRun).toMatch(/-- env PATH="\$PATH" HOME="\$HOME" node scripts\/ci-test\.mjs\n/); + expect(linuxRun.endsWith("fi\nnode scripts/ci-test.mjs\n")).toBe(true); expect(macos.steps.at(-1)!.run).toBe("node scripts/ci-test.mjs"); for (const job of [linux, macos]) expect(job.strategy.matrix.flavor).toEqual(["plain", "san"]); });