From bbd24beba3a90709fc94da6dca0dbf8a7b8fef03 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 15 Oct 2026 22:43:42 -0500 Subject: [PATCH] Split npm-static.test.ts across the Linux shards by measured test cost npm-static.test.ts ran whole on one shard as an isolated suite: 406 s plain and 519 s sanitized in the cost table, and 672 s on san 2/20 in run 38835202089, the second-longest job of that run. skipUnownedTest (tests/harness/shard.ts), called from the suite's beforeEach, balances the suite's tests across the shards of SCRIPTC_TEST_SHARD by their measured milliseconds (tests/harness/npm-static-costs.json; an unmeasured test weighs as much as the median) and skips the tests this shard does not own. Repeated test.each titles get an occurrence suffix as their key. Without a shard spec every test runs. The CI plan runs the suite on every shard in its own vitest process (caseShardedSideFiles) instead of on one owner. At 20 shards the largest shares are 113 and 106 s plain and 181 and 137 s sanitized (the two published Effect generator tests), and about 10 s elsewhere, plus about 11 s per shard to collect the file. scripts/ci-test-costs.mjs also refreshes the per-test table and drops side-queue files that no longer run whole. --- scripts/ci-test-costs.json | 2 - scripts/ci-test-costs.mjs | 51 ++++---- scripts/ci-test-plan.mjs | 8 +- tests/harness/ci-test-plan.test.ts | 5 +- tests/harness/npm-static-costs.json | 176 ++++++++++++++++++++++++++++ tests/harness/npm-static.test.ts | 13 +- tests/harness/shard.test.ts | 36 ++++++ tests/harness/shard.ts | 42 +++++++ 8 files changed, 308 insertions(+), 25 deletions(-) create mode 100644 tests/harness/npm-static-costs.json diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 5ea6000a0..d1c76edae 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -361,7 +361,6 @@ "tests/harness/node-compatibility.test.ts": 3, "tests/harness/node-test.test.ts": 14918, "tests/harness/npm-fetch-defaults.test.ts": 2062, - "tests/harness/npm-static.test.ts": 406491, "tests/harness/oci-manifest.test.ts": 3, "tests/harness/opaque-class-view.test.ts": 1743, "tests/harness/process-argv.test.ts": 1923, @@ -777,7 +776,6 @@ "tests/harness/node-compatibility.test.ts": 5, "tests/harness/node-test.test.ts": 17953, "tests/harness/npm-fetch-defaults.test.ts": 5674, - "tests/harness/npm-static.test.ts": 518221, "tests/harness/oci-manifest.test.ts": 3, "tests/harness/opaque-class-view.test.ts": 2545, "tests/harness/process-argv.test.ts": 2583, diff --git a/scripts/ci-test-costs.mjs b/scripts/ci-test-costs.mjs index 661d0a859..c227424d0 100644 --- a/scripts/ci-test-costs.mjs +++ b/scripts/ci-test-costs.mjs @@ -1,21 +1,27 @@ #!/usr/bin/env node -// Refreshes scripts/ci-test-costs.json from the logs of the Linux test shards of green CI runs on main. +// Refreshes scripts/ci-test-costs.json and tests/harness/npm-static-costs.json from the logs of the Linux test +// shards of green CI runs on main. // // Usage: node scripts/ci-test-costs.mjs ... // // Each log is a downloaded "test (Linux, plain|san, i/n)" job log. vitest prints every test file it ran with its -// duration; for each file that runs in the side queue (not the case-sharded corpus, Effect, coverage and cache -// suites) this stores the median over the logs of its flavor. Files without a measurement keep their current -// value. +// duration, and every test slower than its threshold with its duration. ci-test-costs.json stores, for each file +// that runs whole in the side queue (not the case-sharded corpus, Effect, coverage, cache and npm-static suites), +// the median over the logs of its flavor; npm-static-costs.json stores the median of each npm-static test. Entries +// without a new measurement keep their value; files the side queue no longer runs whole are removed. import { readFileSync, writeFileSync } from "node:fs"; import { fileExclusions, isolatedFiles } from "./ci-test-plan.mjs"; -const path = new URL("./ci-test-costs.json", import.meta.url); -const table = JSON.parse(readFileSync(path, "utf8")); +const filePath = new URL("./ci-test-costs.json", import.meta.url); +const npmStaticPath = new URL("../tests/harness/npm-static-costs.json", import.meta.url); +const tables = { + files: JSON.parse(readFileSync(filePath, "utf8")), + npmStatic: JSON.parse(readFileSync(npmStaticPath, "utf8")), +}; const skipped = new Set( fileExclusions.filter((file) => !isolatedFiles.some(([, isolated]) => isolated === file)), ); -const samples = { plain: {}, san: {} }; +const samples = { files: { plain: {}, san: {} }, npmStatic: { plain: {}, san: {} } }; for (const log of process.argv.slice(2)) { // Logs keep the terminal color codes (ESC [ ... m); drop them before matching. const text = readFileSync(log, "utf8") @@ -23,21 +29,26 @@ for (const log of process.argv.slice(2)) { .replace(/\[[0-9;]*m/g, ""); const flavor = /SCRIPTC_SAN: 1\b/.test(text) ? "san" : "plain"; for (const line of text.split("\n")) { - const m = /^\S+ {2}✓ (\S+\.test\.ts) \(\d+ tests?[^)]*\) (\d+)ms$/.exec(line); - if (!m || skipped.has(m[1])) continue; - (samples[flavor][m[1]] ??= []).push(Number(m[2])); + let m = /^\S+ {2}✓ (\S+\.test\.ts) \(\d+ tests?[^)]*\) (\d+)ms$/.exec(line); + if (m && !skipped.has(m[1])) (samples.files[flavor][m[1]] ??= []).push(Number(m[2])); + m = /^\S+ {4}✓ npm-static pilots(?: \(sanitized\))? > (.*?) {2}(\d+)ms$/.exec(line); + if (m) (samples.npmStatic[flavor][m[1]] ??= []).push(Number(m[2])); } } const median = (values) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; let updated = 0; -for (const flavor of ["plain", "san"]) - for (const [file, values] of Object.entries(samples[flavor])) { - table[flavor][file] = median(values); - updated++; +for (const [name, table] of Object.entries(tables)) + for (const flavor of ["plain", "san"]) { + for (const [key, values] of Object.entries(samples[name][flavor])) { + table[flavor][key] = median(values); + updated++; + } + table[flavor] = Object.fromEntries( + Object.entries(table[flavor]) + .filter(([key]) => name !== "files" || !skipped.has(key)) + .sort(([a], [b]) => a.localeCompare(b)), + ); } -for (const flavor of ["plain", "san"]) - table[flavor] = Object.fromEntries( - Object.entries(table[flavor]).sort(([a], [b]) => a.localeCompare(b)), - ); -writeFileSync(path, JSON.stringify(table, null, 2) + "\n"); -console.log(`updated ${updated} file costs`); +writeFileSync(filePath, JSON.stringify(tables.files, null, 2) + "\n"); +writeFileSync(npmStaticPath, JSON.stringify(tables.npmStatic, null, 2) + "\n"); +console.log(`updated ${updated} costs`); diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index af716c3c5..d922b53db 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -19,15 +19,19 @@ export const isolatedFiles = [ ["self-hosting", "tests/harness/self-hosting.test.ts"], ["self-hosting LLVM emitter", "tests/harness/self-hosting-llvm-emitter.test.ts"], ["fetch", "tests/harness/fetch.test.ts"], - ["npm static", "tests/harness/npm-static.test.ts"], ["surface manifest", "tests/harness/surface-manifest.test.ts"], ]; +// Suites that split their own tests across the shards (SCRIPTC_TEST_SHARD, +// skipUnownedTest in tests/harness/shard.ts): every shard runs its share in +// its own vitest process. +export const caseShardedSideFiles = [["npm static", "tests/harness/npm-static.test.ts"]]; export const fileExclusions = [ ...corpusFiles, ...effectFiles, coverageFile, cacheFile, ...isolatedFiles.map(([, file]) => file), + ...caseShardedSideFiles.map(([, file]) => file), "tests/harness/self-hosting-native-driver.test.ts", ]; @@ -152,6 +156,8 @@ export function ciTestPlan({ shard, flavor, workers = 4, suite = "full" }) { if (owners.get(file) === index) side.push({ name, args: ["test", file], env: { ...env, SCRIPTC_TEST_WORKERS: "1" } }); } + for (const [name, file] of caseShardedSideFiles) + side.push({ name, args: ["test", file], env: { ...env, SCRIPTC_TEST_WORKERS: "1" } }); // Frontend coverage and cache invalidation do not consume sanitizer state. if (flavor === "plain") { side.push({ diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 7925d799b..76f7f0bce 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -8,6 +8,7 @@ import { darwinFiles, effectFiles, fileExclusions, + caseShardedSideFiles, isolatedFiles, testCosts, } from "../../scripts/ci-test-plan.mjs"; @@ -31,10 +32,12 @@ test.for([1, 3, 5, 10, 20])( !file.includes("coverage") && !file.includes("native-toolchain"), )) { + // A suite that splits its own tests runs on every shard; the others on one. + const everyShard = caseShardedSideFiles.some(([, sharded]) => sharded === file); expect( sideFiles.filter((selected) => selected === file), file, - ).toHaveLength(1); + ).toHaveLength(everyShard ? count : 1); } // Each measured file runs on exactly one shard, outside vitest's file shard. const isolated = new Set(isolatedFiles.map(([, file]) => file)); diff --git a/tests/harness/npm-static-costs.json b/tests/harness/npm-static-costs.json new file mode 100644 index 000000000..2f1cdde1e --- /dev/null +++ b/tests/harness/npm-static-costs.json @@ -0,0 +1,176 @@ +{ + "plain": { + "__toESM dependencies compile recursively without explicit selection": 626, + "--npm-static=auto degrades a deviant __toESM package with the same note": 496, + "--npm-static=auto opts a getter-table bundle in": 442, + "--npm-static=auto opts the eligible pilot in": 476, + "--npm-static=auto probes runtime JS from the importing file's realm": 1819, + "--npm-static=auto refuses a package with no own .d.ts": 314, + "a cyclic package tree retains an impure dependency through an unused namespace re-export": 1517, + "a deviant __toESM helper refuses the package with a construct-naming note": 532, + "a package's computed specifier preserves createRequire fallback": 708, + "a package's destructured reassignment preserves createRequire fallback": 769, + "a package's early declarator call preserves createRequire fallback": 722, + "a package's early hoisted call preserves createRequire fallback": 754, + "a package's early let call preserves createRequire fallback": 733, + "a package's early shorthand call preserves createRequire fallback": 713, + "a package's early var call preserves createRequire fallback": 768, + "a package's escaped mutable shorthand preserves createRequire fallback": 701, + "a package's escaped require preserves createRequire fallback": 698, + "a package's exported mutable alias preserves createRequire fallback": 659, + "a package's exported mutable require preserves createRequire fallback": 756, + "a package's foreign base preserves createRequire fallback": 718, + "a package's literal createRequire calls compile without an engine (llvm)": 2137, + "a package's optional resolution preserves createRequire fallback": 738, + "a package's reassigned require preserves createRequire fallback": 718, + "a package's redeclared require preserves createRequire fallback": 729, + "a package's shadowed filename preserves createRequire fallback": 730, + "a preflight-refused package reports its native refusal": 608, + "a workspace-linked package classifies as an npm import without flags": 342, + "a workspace-linked package compiles statically under --npm-static": 1490, + "an ineligible untyped package is refused alongside a compiled dependency": 566, + "auto preserves safe declaration overloads while compiling the JavaScript body": 2164, + "bundled class aliases preserve declared methods and callback fields (llvm)": 5433, + "bundled CommonJS function imports compile without an engine (llvm)": 2234, + "bundler-emitted CJS 2465-getter-table compiles statically and byte-matches Node": 1713, + "bundler-emitted CJS 2466-getter-star compiles statically and byte-matches Node": 1570, + "bundler-emitted CJS 2467-star-barrel compiles statically and byte-matches Node": 1587, + "bundler-emitted CJS 2468-defineprop-exports compiles statically and byte-matches Node": 1484, + "bundler-emitted CJS 2556-toesm-external compiles statically and byte-matches Node": 1645, + "commander compiles statically and byte-matches Node": 23670, + "commander computed option listener registration compiles statically and byte-matches Node": 7303, + "conditional exports keep distinct import and require behavior": 4421, + "conflicting public class aliases keep the JavaScript return inference": 598, + "createRequire loads an opted-in package through its require entry": 3154, + "dualist compiles statically and byte-matches Node": 1896, + "effect4-shape-match-metadata.ts preserves native object layout and module initialization": 1990, + "effect4-shape-prototype-factory.ts preserves native object layout and module initialization": 1790, + "effect4-shape-semantics.ts preserves native object layout and module initialization": 9277, + "effect4-shape-string-constructor.ts preserves native object layout and module initialization": 2844, + "escape-string-regexp compiles statically and byte-matches Node": 1619, + "hoisted CommonJS function exports stay static (auto)": 2139, + "hoisted CommonJS function exports stay static (explicit)": 2187, + "inferred class methods dispatch through inherited checked receivers (dev)": 1752, + "inferred class methods dispatch through inherited checked receivers (release)": 2240, + "inferred helpers preserve native array storage (dev)": 1171, + "inferred helpers preserve native array storage (release)": 1217, + "inferred variadic factories preserve checked callable results": 1231, + "ms compiles statically and byte-matches Node": 2306, + "ms supports invalid duration strings without runtime fences": 691, + "opted-in npm packages retain imports deeper than four JavaScript modules": 1799, + "picocolors compiles fully statically and byte-matches both color branches": 3698, + "picocolors inherited hasOwnProperty matches Node": 3008, + "picocolors inherited valueOf retains the prototype-method fence": 708, + "published type-only declarations coexist with a static runtime bundle": 1829, + "purebarrel namespace re-exports byte-match Node": 1449, + "recursive inferred JavaScript results retain their promised ABI": 1369, + "renderer startup 'async-backend-factory.mjs' compiles from shipped JavaScript ('llvm')": 2230, + "renderer startup 'checked-renderer-defaults.js' compiles from shipped JavaScript ('llvm')": 2359, + "renderer startup 'destructure-inferred-row-defaults.js' compiles from shipped JavaScript ('llvm')": 2313, + "renderer startup 'fresh-array-union-layout.js' compiles from shipped JavaScript ('llvm')": 2260, + "renderer startup 'frozen-checked-dictionaries.mjs' compiles from shipped JavaScript ('llvm')": 2579, + "renderer startup 'object-from-checked-entries.js' compiles from shipped JavaScript ('llvm')": 3447, + "renderer startup 'regex-checked-storage.js' compiles from shipped JavaScript ('llvm')": 2648, + "renderer startup 'renderer-export-dictionaries.js' compiles from shipped JavaScript ('llvm')": 2405, + "renderer startup 'renderer-specializations.js' compiles from shipped JavaScript ('llvm')": 2578, + "renderer startup 'set-checked-methods.js' compiles from shipped JavaScript ('llvm')": 2375, + "renderer startup 'stored-native-builtins.js' compiles from shipped JavaScript ('llvm')": 2495, + "renderer startup 'stored-object-helpers.js' compiles from shipped JavaScript ('llvm')": 2509, + "renderer values and callbacks compile from shipped JavaScript (llvm)": 5773, + "selectivebarrel-cli.ts preserves native object layout and module initialization": 1772, + "slash compiles statically and byte-matches Node": 1646, + "statefulbarrel namespace re-exports byte-match Node": 1427, + "the crypto utility package compiles fully statically and byte-matches Node": 2554, + "the published Effect Function subpath initializes and runs statically": 9097, + "the published Effect generator has deterministic native results": 105577, + "the published Effect generator logs and completes statically": 112814, + "untyped package methods preserve virtual overrides (llvm)": 3065 + }, + "san": { + "__toESM dependencies compile recursively without explicit selection": 458, + "--npm-static=auto degrades a deviant __toESM package with the same note": 370, + "--npm-static=auto opts a getter-table bundle in": 317, + "--npm-static=auto opts the eligible pilot in": 524, + "--npm-static=auto probes runtime JS from the importing file's realm": 2349, + "--npm-static=auto refuses a package with no own .d.ts": 424, + "a cyclic package tree retains an impure dependency through an unused namespace re-export": 1505, + "a deviant __toESM helper refuses the package with a construct-naming note": 427, + "a package's computed specifier preserves createRequire fallback": 623, + "a package's destructured reassignment preserves createRequire fallback": 585, + "a package's early declarator call preserves createRequire fallback": 559, + "a package's early hoisted call preserves createRequire fallback": 531, + "a package's early let call preserves createRequire fallback": 599, + "a package's early shorthand call preserves createRequire fallback": 545, + "a package's early var call preserves createRequire fallback": 586, + "a package's escaped mutable shorthand preserves createRequire fallback": 596, + "a package's escaped require preserves createRequire fallback": 566, + "a package's exported mutable alias preserves createRequire fallback": 565, + "a package's exported mutable require preserves createRequire fallback": 515, + "a package's foreign base preserves createRequire fallback": 551, + "a package's literal createRequire calls compile without an engine (llvm)": 2051, + "a package's optional resolution preserves createRequire fallback": 605, + "a package's reassigned require preserves createRequire fallback": 535, + "a package's redeclared require preserves createRequire fallback": 644, + "a package's shadowed filename preserves createRequire fallback": 626, + "a preflight-refused package reports its native refusal": 659, + "a workspace-linked package classifies as an npm import without flags": 340, + "a workspace-linked package compiles statically under --npm-static": 1446, + "an ineligible untyped package is refused alongside a compiled dependency": 552, + "auto preserves safe declaration overloads while compiling the JavaScript body": 2370, + "bundled class aliases preserve declared methods and callback fields (llvm)": 4728, + "bundled CommonJS function imports compile without an engine (llvm)": 2088, + "bundler-emitted CJS 2465-getter-table compiles statically and byte-matches Node": 1490, + "bundler-emitted CJS 2466-getter-star compiles statically and byte-matches Node": 1325, + "bundler-emitted CJS 2467-star-barrel compiles statically and byte-matches Node": 1370, + "bundler-emitted CJS 2468-defineprop-exports compiles statically and byte-matches Node": 1361, + "bundler-emitted CJS 2556-toesm-external compiles statically and byte-matches Node": 1608, + "commander compiles statically and byte-matches Node": 32614, + "commander computed option listener registration compiles statically and byte-matches Node": 7719, + "conditional exports keep distinct import and require behavior": 5017, + "conflicting public class aliases keep the JavaScript return inference": 482, + "createRequire loads an opted-in package through its require entry": 2581, + "dualist compiles statically and byte-matches Node": 1864, + "effect4-shape-match-metadata.ts preserves native object layout and module initialization": 2106, + "effect4-shape-prototype-factory.ts preserves native object layout and module initialization": 1612, + "effect4-shape-semantics.ts preserves native object layout and module initialization": 23336, + "effect4-shape-string-constructor.ts preserves native object layout and module initialization": 2806, + "escape-string-regexp compiles statically and byte-matches Node": 1816, + "hoisted CommonJS function exports stay static (auto)": 1955, + "hoisted CommonJS function exports stay static (explicit)": 1983, + "inferred class methods dispatch through inherited checked receivers (dev)": 1954, + "inferred class methods dispatch through inherited checked receivers (release)": 2111, + "inferred helpers preserve native array storage (dev)": 1189, + "inferred helpers preserve native array storage (release)": 1079, + "inferred variadic factories preserve checked callable results": 1157, + "ms compiles statically and byte-matches Node": 2387, + "ms supports invalid duration strings without runtime fences": 669, + "opted-in npm packages retain imports deeper than four JavaScript modules": 2301, + "picocolors compiles fully statically and byte-matches both color branches": 2954, + "picocolors inherited hasOwnProperty matches Node": 2682, + "picocolors inherited valueOf retains the prototype-method fence": 535, + "published type-only declarations coexist with a static runtime bundle": 1684, + "purebarrel namespace re-exports byte-match Node": 1308, + "recursive inferred JavaScript results retain their promised ABI": 1628, + "renderer startup 'async-backend-factory.mjs' compiles from shipped JavaScript ('llvm')": 2063, + "renderer startup 'checked-renderer-defaults.js' compiles from shipped JavaScript ('llvm')": 2539, + "renderer startup 'destructure-inferred-row-defaults.js' compiles from shipped JavaScript ('llvm')": 2301, + "renderer startup 'fresh-array-union-layout.js' compiles from shipped JavaScript ('llvm')": 2117, + "renderer startup 'frozen-checked-dictionaries.mjs' compiles from shipped JavaScript ('llvm')": 2414, + "renderer startup 'object-from-checked-entries.js' compiles from shipped JavaScript ('llvm')": 2968, + "renderer startup 'regex-checked-storage.js' compiles from shipped JavaScript ('llvm')": 2997, + "renderer startup 'renderer-export-dictionaries.js' compiles from shipped JavaScript ('llvm')": 2440, + "renderer startup 'renderer-specializations.js' compiles from shipped JavaScript ('llvm')": 2435, + "renderer startup 'set-checked-methods.js' compiles from shipped JavaScript ('llvm')": 2176, + "renderer startup 'stored-native-builtins.js' compiles from shipped JavaScript ('llvm')": 2283, + "renderer startup 'stored-object-helpers.js' compiles from shipped JavaScript ('llvm')": 2164, + "renderer values and callbacks compile from shipped JavaScript (llvm)": 4668, + "selectivebarrel-cli.ts preserves native object layout and module initialization": 1725, + "slash compiles statically and byte-matches Node": 2173, + "statefulbarrel namespace re-exports byte-match Node": 1246, + "the crypto utility package compiles fully statically and byte-matches Node": 3064, + "the published Effect Function subpath initializes and runs statically": 8280, + "the published Effect generator has deterministic native results": 136502, + "the published Effect generator logs and completes statically": 181378, + "untyped package methods preserve virtual overrides (llvm)": 2730 + } +} diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index b6e451314..8de3886c8 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -12,8 +12,9 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; -import { describe, expect, test } from "vitest"; +import { beforeEach, describe, expect, test } from "vitest"; import { analyze, compile } from "@scriptc/compiler"; +import { skipUnownedTest } from "./shard.js"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -21,6 +22,13 @@ const fixturesRoot = join(repoRoot, "tests/fixtures"); const pilotRoot = join(fixturesRoot, "npm-static"); const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; +// Measured milliseconds per test on a CI runner (scripts/ci-test-costs.mjs). +const testCosts = ( + JSON.parse(readFileSync(new URL("./npm-static-costs.json", import.meta.url), "utf8")) as { + plain: Record; + san: Record; + } +)[sanitize ? "san" : "plain"]; interface RunResult { stdout: Buffer; @@ -99,6 +107,9 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise } describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { + // CI runs this file on every Linux shard; each runs its share of the tests. + beforeEach((context) => skipUnownedTest(context, testCosts)); + test.each([ ["effect4-shape-string-constructor.ts", "effect4-shapes"], ["effect4-shape-prototype-factory.ts", "effect4-shapes"], diff --git a/tests/harness/shard.test.ts b/tests/harness/shard.test.ts index ea1c10e4d..e645f2cac 100644 --- a/tests/harness/shard.test.ts +++ b/tests/harness/shard.test.ts @@ -13,6 +13,7 @@ import { shardOf, shardSelect, shardSuffix, + skipUnownedTest, } from "./shard.js"; import corpusCosts from "./corpus-costs.json"; import fixtureCosts from "./effect4-costs.json"; @@ -208,6 +209,41 @@ describe("balancedShardSelect", () => { }); }); +test("skipUnownedTest runs each test of a suite on exactly one shard, longest apart", () => { + // Two tests share a title, as test.each titles without parameters do. + const names = [...Array.from({ length: 39 }, (_, index) => `case ${index}`), "case 0"]; + // Most tests measured at a second, two long ones, and the last five unmeasured. + const costs: Record = Object.fromEntries( + names.slice(0, 35).map((name) => [name, 1_000]), + ); + costs["case 3"] = 200_000; + costs["case 7"] = 150_000; + // One suite per process, as in a CI shard: the owners are computed once per suite. + const runs = (spec: { index: number; count: number } | undefined) => { + const suite = { tasks: [] as { name: string }[] }; + suite.tasks = names.map((name) => ({ type: "test", name, suite })); + return suite.tasks + .filter((task) => { + let skipped = false; + skipUnownedTest( + { task, skip: () => (skipped = true) } as unknown as Parameters< + typeof skipUnownedTest + >[0], + costs, + spec, + ); + return !skipped; + }) + .map((task) => task.name); + }; + expect(runs(undefined)).toEqual(names); + const shards = Array.from({ length: 5 }, (_, index) => runs({ index: index + 1, count: 5 })); + expect(shards.flat().toSorted()).toEqual(names.toSorted()); + // The two long tests hold shards of their own. + expect(shards.find((shard) => shard.includes("case 3"))).toEqual(["case 3"]); + expect(shards.find((shard) => shard.includes("case 7"))).toEqual(["case 7"]); +}); + describe("corpusSelect", () => { const programs = ["ts", "js", "mjs", "cjs"] .flatMap((ext) => [ diff --git a/tests/harness/shard.ts b/tests/harness/shard.ts index 460a78df3..4c3b9c244 100644 --- a/tests/harness/shard.ts +++ b/tests/harness/shard.ts @@ -17,6 +17,7 @@ */ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; +import type { Suite, Test, TestContext } from "vitest"; export interface ShardSpec { /** 1-based shard index. */ @@ -122,3 +123,44 @@ export function corpusSelect( export function shardSuffix(spec: ShardSpec | undefined = parseShardSpec()): string { return spec === undefined ? "" : `, shard ${spec.index}/${spec.count}`; } + +const ownedTests = new WeakMap>(); + +/** For a suite of independent tests, called from beforeEach: skips the tests + * this shard does not own. The suite's tests are balanced across the shards + * by their measured cost (`costs`: milliseconds by test name; a test without + * a measurement weighs as much as the median measured one), so every shard + * runs a similar share and the union of the shards runs each test once. */ +export function skipUnownedTest( + context: TestContext, + costs: Readonly>, + spec: ShardSpec | undefined = parseShardSpec(), +): void { + const suite = context.task.suite; + if (spec === undefined || spec.count === 1 || suite === undefined) return; + let owned = ownedTests.get(suite); + if (owned === undefined) { + const known = Object.values(costs).sort((a, b) => a - b); + const median = known.length === 0 ? 1 : known[Math.floor(known.length / 2)]!; + const tests = suite.tasks.filter((task): task is Test => task.type === "test"); + // test.each titles without their parameters repeat; later repeats get "#2", "#3"... as their key. + const seen = new Map(); + const keys = new Map( + tests.map((test) => { + const count = (seen.get(test.name) ?? 0) + 1; + seen.set(test.name, count); + return [test, count === 1 ? test.name : `${test.name} #${count}`]; + }), + ); + owned = new Set( + balancedShardSelect( + tests, + (test) => keys.get(test)!, + (test) => Math.max(1, costs[test.name] ?? median), + spec, + ), + ); + ownedTests.set(suite, owned); + } + if (!owned.has(context.task)) context.skip(); +}