diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 5ea6000a0..d1c76edae 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -361,7 +361,6 @@ "tests/harness/node-compatibility.test.ts": 3, "tests/harness/node-test.test.ts": 14918, "tests/harness/npm-fetch-defaults.test.ts": 2062, - "tests/harness/npm-static.test.ts": 406491, "tests/harness/oci-manifest.test.ts": 3, "tests/harness/opaque-class-view.test.ts": 1743, "tests/harness/process-argv.test.ts": 1923, @@ -777,7 +776,6 @@ "tests/harness/node-compatibility.test.ts": 5, "tests/harness/node-test.test.ts": 17953, "tests/harness/npm-fetch-defaults.test.ts": 5674, - "tests/harness/npm-static.test.ts": 518221, "tests/harness/oci-manifest.test.ts": 3, "tests/harness/opaque-class-view.test.ts": 2545, "tests/harness/process-argv.test.ts": 2583, diff --git a/scripts/ci-test-costs.mjs b/scripts/ci-test-costs.mjs index 661d0a859..c227424d0 100644 --- a/scripts/ci-test-costs.mjs +++ b/scripts/ci-test-costs.mjs @@ -1,21 +1,27 @@ #!/usr/bin/env node -// Refreshes scripts/ci-test-costs.json from the logs of the Linux test shards of green CI runs on main. +// Refreshes scripts/ci-test-costs.json and tests/harness/npm-static-costs.json from the logs of the Linux test +// shards of green CI runs on main. // // Usage: node scripts/ci-test-costs.mjs ... // // Each log is a downloaded "test (Linux, plain|san, i/n)" job log. vitest prints every test file it ran with its -// duration; for each file that runs in the side queue (not the case-sharded corpus, Effect, coverage and cache -// suites) this stores the median over the logs of its flavor. Files without a measurement keep their current -// value. +// duration, and every test slower than its threshold with its duration. ci-test-costs.json stores, for each file +// that runs whole in the side queue (not the case-sharded corpus, Effect, coverage, cache and npm-static suites), +// the median over the logs of its flavor; npm-static-costs.json stores the median of each npm-static test. Entries +// without a new measurement keep their value; files the side queue no longer runs whole are removed. import { readFileSync, writeFileSync } from "node:fs"; import { fileExclusions, isolatedFiles } from "./ci-test-plan.mjs"; -const path = new URL("./ci-test-costs.json", import.meta.url); -const table = JSON.parse(readFileSync(path, "utf8")); +const filePath = new URL("./ci-test-costs.json", import.meta.url); +const npmStaticPath = new URL("../tests/harness/npm-static-costs.json", import.meta.url); +const tables = { + files: JSON.parse(readFileSync(filePath, "utf8")), + npmStatic: JSON.parse(readFileSync(npmStaticPath, "utf8")), +}; const skipped = new Set( fileExclusions.filter((file) => !isolatedFiles.some(([, isolated]) => isolated === file)), ); -const samples = { plain: {}, san: {} }; +const samples = { files: { plain: {}, san: {} }, npmStatic: { plain: {}, san: {} } }; for (const log of process.argv.slice(2)) { // Logs keep the terminal color codes (ESC [ ... m); drop them before matching. const text = readFileSync(log, "utf8") @@ -23,21 +29,26 @@ for (const log of process.argv.slice(2)) { .replace(/\[[0-9;]*m/g, ""); const flavor = /SCRIPTC_SAN: 1\b/.test(text) ? "san" : "plain"; for (const line of text.split("\n")) { - const m = /^\S+ {2}✓ (\S+\.test\.ts) \(\d+ tests?[^)]*\) (\d+)ms$/.exec(line); - if (!m || skipped.has(m[1])) continue; - (samples[flavor][m[1]] ??= []).push(Number(m[2])); + let m = /^\S+ {2}✓ (\S+\.test\.ts) \(\d+ tests?[^)]*\) (\d+)ms$/.exec(line); + if (m && !skipped.has(m[1])) (samples.files[flavor][m[1]] ??= []).push(Number(m[2])); + m = /^\S+ {4}✓ npm-static pilots(?: \(sanitized\))? > (.*?) {2}(\d+)ms$/.exec(line); + if (m) (samples.npmStatic[flavor][m[1]] ??= []).push(Number(m[2])); } } const median = (values) => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; let updated = 0; -for (const flavor of ["plain", "san"]) - for (const [file, values] of Object.entries(samples[flavor])) { - table[flavor][file] = median(values); - updated++; +for (const [name, table] of Object.entries(tables)) + for (const flavor of ["plain", "san"]) { + for (const [key, values] of Object.entries(samples[name][flavor])) { + table[flavor][key] = median(values); + updated++; + } + table[flavor] = Object.fromEntries( + Object.entries(table[flavor]) + .filter(([key]) => name !== "files" || !skipped.has(key)) + .sort(([a], [b]) => a.localeCompare(b)), + ); } -for (const flavor of ["plain", "san"]) - table[flavor] = Object.fromEntries( - Object.entries(table[flavor]).sort(([a], [b]) => a.localeCompare(b)), - ); -writeFileSync(path, JSON.stringify(table, null, 2) + "\n"); -console.log(`updated ${updated} file costs`); +writeFileSync(filePath, JSON.stringify(tables.files, null, 2) + "\n"); +writeFileSync(npmStaticPath, JSON.stringify(tables.npmStatic, null, 2) + "\n"); +console.log(`updated ${updated} costs`); diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index af716c3c5..d922b53db 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -19,15 +19,19 @@ export const isolatedFiles = [ ["self-hosting", "tests/harness/self-hosting.test.ts"], ["self-hosting LLVM emitter", "tests/harness/self-hosting-llvm-emitter.test.ts"], ["fetch", "tests/harness/fetch.test.ts"], - ["npm static", "tests/harness/npm-static.test.ts"], ["surface manifest", "tests/harness/surface-manifest.test.ts"], ]; +// Suites that split their own tests across the shards (SCRIPTC_TEST_SHARD, +// skipUnownedTest in tests/harness/shard.ts): every shard runs its share in +// its own vitest process. +export const caseShardedSideFiles = [["npm static", "tests/harness/npm-static.test.ts"]]; export const fileExclusions = [ ...corpusFiles, ...effectFiles, coverageFile, cacheFile, ...isolatedFiles.map(([, file]) => file), + ...caseShardedSideFiles.map(([, file]) => file), "tests/harness/self-hosting-native-driver.test.ts", ]; @@ -152,6 +156,8 @@ export function ciTestPlan({ shard, flavor, workers = 4, suite = "full" }) { if (owners.get(file) === index) side.push({ name, args: ["test", file], env: { ...env, SCRIPTC_TEST_WORKERS: "1" } }); } + for (const [name, file] of caseShardedSideFiles) + side.push({ name, args: ["test", file], env: { ...env, SCRIPTC_TEST_WORKERS: "1" } }); // Frontend coverage and cache invalidation do not consume sanitizer state. if (flavor === "plain") { side.push({ diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 7925d799b..76f7f0bce 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -8,6 +8,7 @@ import { darwinFiles, effectFiles, fileExclusions, + caseShardedSideFiles, isolatedFiles, testCosts, } from "../../scripts/ci-test-plan.mjs"; @@ -31,10 +32,12 @@ test.for([1, 3, 5, 10, 20])( !file.includes("coverage") && !file.includes("native-toolchain"), )) { + // A suite that splits its own tests runs on every shard; the others on one. + const everyShard = caseShardedSideFiles.some(([, sharded]) => sharded === file); expect( sideFiles.filter((selected) => selected === file), file, - ).toHaveLength(1); + ).toHaveLength(everyShard ? count : 1); } // Each measured file runs on exactly one shard, outside vitest's file shard. const isolated = new Set(isolatedFiles.map(([, file]) => file)); diff --git a/tests/harness/npm-static-costs.json b/tests/harness/npm-static-costs.json new file mode 100644 index 000000000..2f1cdde1e --- /dev/null +++ b/tests/harness/npm-static-costs.json @@ -0,0 +1,176 @@ +{ + "plain": { + "__toESM dependencies compile recursively without explicit selection": 626, + "--npm-static=auto degrades a deviant __toESM package with the same note": 496, + "--npm-static=auto opts a getter-table bundle in": 442, + "--npm-static=auto opts the eligible pilot in": 476, + "--npm-static=auto probes runtime JS from the importing file's realm": 1819, + "--npm-static=auto refuses a package with no own .d.ts": 314, + "a cyclic package tree retains an impure dependency through an unused namespace re-export": 1517, + "a deviant __toESM helper refuses the package with a construct-naming note": 532, + "a package's computed specifier preserves createRequire fallback": 708, + "a package's destructured reassignment preserves createRequire fallback": 769, + "a package's early declarator call preserves createRequire fallback": 722, + "a package's early hoisted call preserves createRequire fallback": 754, + "a package's early let call preserves createRequire fallback": 733, + "a package's early shorthand call preserves createRequire fallback": 713, + "a package's early var call preserves createRequire fallback": 768, + "a package's escaped mutable shorthand preserves createRequire fallback": 701, + "a package's escaped require preserves createRequire fallback": 698, + "a package's exported mutable alias preserves createRequire fallback": 659, + "a package's exported mutable require preserves createRequire fallback": 756, + "a package's foreign base preserves createRequire fallback": 718, + "a package's literal createRequire calls compile without an engine (llvm)": 2137, + "a package's optional resolution preserves createRequire fallback": 738, + "a package's reassigned require preserves createRequire fallback": 718, + "a package's redeclared require preserves createRequire fallback": 729, + "a package's shadowed filename preserves createRequire fallback": 730, + "a preflight-refused package reports its native refusal": 608, + "a workspace-linked package classifies as an npm import without flags": 342, + "a workspace-linked package compiles statically under --npm-static": 1490, + "an ineligible untyped package is refused alongside a compiled dependency": 566, + "auto preserves safe declaration overloads while compiling the JavaScript body": 2164, + "bundled class aliases preserve declared methods and callback fields (llvm)": 5433, + "bundled CommonJS function imports compile without an engine (llvm)": 2234, + "bundler-emitted CJS 2465-getter-table compiles statically and byte-matches Node": 1713, + "bundler-emitted CJS 2466-getter-star compiles statically and byte-matches Node": 1570, + "bundler-emitted CJS 2467-star-barrel compiles statically and byte-matches Node": 1587, + "bundler-emitted CJS 2468-defineprop-exports compiles statically and byte-matches Node": 1484, + "bundler-emitted CJS 2556-toesm-external compiles statically and byte-matches Node": 1645, + "commander compiles statically and byte-matches Node": 23670, + "commander computed option listener registration compiles statically and byte-matches Node": 7303, + "conditional exports keep distinct import and require behavior": 4421, + "conflicting public class aliases keep the JavaScript return inference": 598, + "createRequire loads an opted-in package through its require entry": 3154, + "dualist compiles statically and byte-matches Node": 1896, + "effect4-shape-match-metadata.ts preserves native object layout and module initialization": 1990, + "effect4-shape-prototype-factory.ts preserves native object layout and module initialization": 1790, + "effect4-shape-semantics.ts preserves native object layout and module initialization": 9277, + "effect4-shape-string-constructor.ts preserves native object layout and module initialization": 2844, + "escape-string-regexp compiles statically and byte-matches Node": 1619, + "hoisted CommonJS function exports stay static (auto)": 2139, + "hoisted CommonJS function exports stay static (explicit)": 2187, + "inferred class methods dispatch through inherited checked receivers (dev)": 1752, + "inferred class methods dispatch through inherited checked receivers (release)": 2240, + "inferred helpers preserve native array storage (dev)": 1171, + "inferred helpers preserve native array storage (release)": 1217, + "inferred variadic factories preserve checked callable results": 1231, + "ms compiles statically and byte-matches Node": 2306, + "ms supports invalid duration strings without runtime fences": 691, + "opted-in npm packages retain imports deeper than four JavaScript modules": 1799, + "picocolors compiles fully statically and byte-matches both color branches": 3698, + "picocolors inherited hasOwnProperty matches Node": 3008, + "picocolors inherited valueOf retains the prototype-method fence": 708, + "published type-only declarations coexist with a static runtime bundle": 1829, + "purebarrel namespace re-exports byte-match Node": 1449, + "recursive inferred JavaScript results retain their promised ABI": 1369, + "renderer startup 'async-backend-factory.mjs' compiles from shipped JavaScript ('llvm')": 2230, + "renderer startup 'checked-renderer-defaults.js' compiles from shipped JavaScript ('llvm')": 2359, + "renderer startup 'destructure-inferred-row-defaults.js' compiles from shipped JavaScript ('llvm')": 2313, + "renderer startup 'fresh-array-union-layout.js' compiles from shipped JavaScript ('llvm')": 2260, + "renderer startup 'frozen-checked-dictionaries.mjs' compiles from shipped JavaScript ('llvm')": 2579, + "renderer startup 'object-from-checked-entries.js' compiles from shipped JavaScript ('llvm')": 3447, + "renderer startup 'regex-checked-storage.js' compiles from shipped JavaScript ('llvm')": 2648, + "renderer startup 'renderer-export-dictionaries.js' compiles from shipped JavaScript ('llvm')": 2405, + "renderer startup 'renderer-specializations.js' compiles from shipped JavaScript ('llvm')": 2578, + "renderer startup 'set-checked-methods.js' compiles from shipped JavaScript ('llvm')": 2375, + "renderer startup 'stored-native-builtins.js' compiles from shipped JavaScript ('llvm')": 2495, + "renderer startup 'stored-object-helpers.js' compiles from shipped JavaScript ('llvm')": 2509, + "renderer values and callbacks compile from shipped JavaScript (llvm)": 5773, + "selectivebarrel-cli.ts preserves native object layout and module initialization": 1772, + "slash compiles statically and byte-matches Node": 1646, + "statefulbarrel namespace re-exports byte-match Node": 1427, + "the crypto utility package compiles fully statically and byte-matches Node": 2554, + "the published Effect Function subpath initializes and runs statically": 9097, + "the published Effect generator has deterministic native results": 105577, + "the published Effect generator logs and completes statically": 112814, + "untyped package methods preserve virtual overrides (llvm)": 3065 + }, + "san": { + "__toESM dependencies compile recursively without explicit selection": 458, + "--npm-static=auto degrades a deviant __toESM package with the same note": 370, + "--npm-static=auto opts a getter-table bundle in": 317, + "--npm-static=auto opts the eligible pilot in": 524, + "--npm-static=auto probes runtime JS from the importing file's realm": 2349, + "--npm-static=auto refuses a package with no own .d.ts": 424, + "a cyclic package tree retains an impure dependency through an unused namespace re-export": 1505, + "a deviant __toESM helper refuses the package with a construct-naming note": 427, + "a package's computed specifier preserves createRequire fallback": 623, + "a package's destructured reassignment preserves createRequire fallback": 585, + "a package's early declarator call preserves createRequire fallback": 559, + "a package's early hoisted call preserves createRequire fallback": 531, + "a package's early let call preserves createRequire fallback": 599, + "a package's early shorthand call preserves createRequire fallback": 545, + "a package's early var call preserves createRequire fallback": 586, + "a package's escaped mutable shorthand preserves createRequire fallback": 596, + "a package's escaped require preserves createRequire fallback": 566, + "a package's exported mutable alias preserves createRequire fallback": 565, + "a package's exported mutable require preserves createRequire fallback": 515, + "a package's foreign base preserves createRequire fallback": 551, + "a package's literal createRequire calls compile without an engine (llvm)": 2051, + "a package's optional resolution preserves createRequire fallback": 605, + "a package's reassigned require preserves createRequire fallback": 535, + "a package's redeclared require preserves createRequire fallback": 644, + "a package's shadowed filename preserves createRequire fallback": 626, + "a preflight-refused package reports its native refusal": 659, + "a workspace-linked package classifies as an npm import without flags": 340, + "a workspace-linked package compiles statically under --npm-static": 1446, + "an ineligible untyped package is refused alongside a compiled dependency": 552, + "auto preserves safe declaration overloads while compiling the JavaScript body": 2370, + "bundled class aliases preserve declared methods and callback fields (llvm)": 4728, + "bundled CommonJS function imports compile without an engine (llvm)": 2088, + "bundler-emitted CJS 2465-getter-table compiles statically and byte-matches Node": 1490, + "bundler-emitted CJS 2466-getter-star compiles statically and byte-matches Node": 1325, + "bundler-emitted CJS 2467-star-barrel compiles statically and byte-matches Node": 1370, + "bundler-emitted CJS 2468-defineprop-exports compiles statically and byte-matches Node": 1361, + "bundler-emitted CJS 2556-toesm-external compiles statically and byte-matches Node": 1608, + "commander compiles statically and byte-matches Node": 32614, + "commander computed option listener registration compiles statically and byte-matches Node": 7719, + "conditional exports keep distinct import and require behavior": 5017, + "conflicting public class aliases keep the JavaScript return inference": 482, + "createRequire loads an opted-in package through its require entry": 2581, + "dualist compiles statically and byte-matches Node": 1864, + "effect4-shape-match-metadata.ts preserves native object layout and module initialization": 2106, + "effect4-shape-prototype-factory.ts preserves native object layout and module initialization": 1612, + "effect4-shape-semantics.ts preserves native object layout and module initialization": 23336, + "effect4-shape-string-constructor.ts preserves native object layout and module initialization": 2806, + "escape-string-regexp compiles statically and byte-matches Node": 1816, + "hoisted CommonJS function exports stay static (auto)": 1955, + "hoisted CommonJS function exports stay static (explicit)": 1983, + "inferred class methods dispatch through inherited checked receivers (dev)": 1954, + "inferred class methods dispatch through inherited checked receivers (release)": 2111, + "inferred helpers preserve native array storage (dev)": 1189, + "inferred helpers preserve native array storage (release)": 1079, + "inferred variadic factories preserve checked callable results": 1157, + "ms compiles statically and byte-matches Node": 2387, + "ms supports invalid duration strings without runtime fences": 669, + "opted-in npm packages retain imports deeper than four JavaScript modules": 2301, + "picocolors compiles fully statically and byte-matches both color branches": 2954, + "picocolors inherited hasOwnProperty matches Node": 2682, + "picocolors inherited valueOf retains the prototype-method fence": 535, + "published type-only declarations coexist with a static runtime bundle": 1684, + "purebarrel namespace re-exports byte-match Node": 1308, + "recursive inferred JavaScript results retain their promised ABI": 1628, + "renderer startup 'async-backend-factory.mjs' compiles from shipped JavaScript ('llvm')": 2063, + "renderer startup 'checked-renderer-defaults.js' compiles from shipped JavaScript ('llvm')": 2539, + "renderer startup 'destructure-inferred-row-defaults.js' compiles from shipped JavaScript ('llvm')": 2301, + "renderer startup 'fresh-array-union-layout.js' compiles from shipped JavaScript ('llvm')": 2117, + "renderer startup 'frozen-checked-dictionaries.mjs' compiles from shipped JavaScript ('llvm')": 2414, + "renderer startup 'object-from-checked-entries.js' compiles from shipped JavaScript ('llvm')": 2968, + "renderer startup 'regex-checked-storage.js' compiles from shipped JavaScript ('llvm')": 2997, + "renderer startup 'renderer-export-dictionaries.js' compiles from shipped JavaScript ('llvm')": 2440, + "renderer startup 'renderer-specializations.js' compiles from shipped JavaScript ('llvm')": 2435, + "renderer startup 'set-checked-methods.js' compiles from shipped JavaScript ('llvm')": 2176, + "renderer startup 'stored-native-builtins.js' compiles from shipped JavaScript ('llvm')": 2283, + "renderer startup 'stored-object-helpers.js' compiles from shipped JavaScript ('llvm')": 2164, + "renderer values and callbacks compile from shipped JavaScript (llvm)": 4668, + "selectivebarrel-cli.ts preserves native object layout and module initialization": 1725, + "slash compiles statically and byte-matches Node": 2173, + "statefulbarrel namespace re-exports byte-match Node": 1246, + "the crypto utility package compiles fully statically and byte-matches Node": 3064, + "the published Effect Function subpath initializes and runs statically": 8280, + "the published Effect generator has deterministic native results": 136502, + "the published Effect generator logs and completes statically": 181378, + "untyped package methods preserve virtual overrides (llvm)": 2730 + } +} diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index b6e451314..8de3886c8 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -12,8 +12,9 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; -import { describe, expect, test } from "vitest"; +import { beforeEach, describe, expect, test } from "vitest"; import { analyze, compile } from "@scriptc/compiler"; +import { skipUnownedTest } from "./shard.js"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -21,6 +22,13 @@ const fixturesRoot = join(repoRoot, "tests/fixtures"); const pilotRoot = join(fixturesRoot, "npm-static"); const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; +// Measured milliseconds per test on a CI runner (scripts/ci-test-costs.mjs). +const testCosts = ( + JSON.parse(readFileSync(new URL("./npm-static-costs.json", import.meta.url), "utf8")) as { + plain: Record; + san: Record; + } +)[sanitize ? "san" : "plain"]; interface RunResult { stdout: Buffer; @@ -99,6 +107,9 @@ async function buildStatic(entry: string, npmStatic: string[] | "auto"): Promise } describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { + // CI runs this file on every Linux shard; each runs its share of the tests. + beforeEach((context) => skipUnownedTest(context, testCosts)); + test.each([ ["effect4-shape-string-constructor.ts", "effect4-shapes"], ["effect4-shape-prototype-factory.ts", "effect4-shapes"], diff --git a/tests/harness/shard.test.ts b/tests/harness/shard.test.ts index ea1c10e4d..e645f2cac 100644 --- a/tests/harness/shard.test.ts +++ b/tests/harness/shard.test.ts @@ -13,6 +13,7 @@ import { shardOf, shardSelect, shardSuffix, + skipUnownedTest, } from "./shard.js"; import corpusCosts from "./corpus-costs.json"; import fixtureCosts from "./effect4-costs.json"; @@ -208,6 +209,41 @@ describe("balancedShardSelect", () => { }); }); +test("skipUnownedTest runs each test of a suite on exactly one shard, longest apart", () => { + // Two tests share a title, as test.each titles without parameters do. + const names = [...Array.from({ length: 39 }, (_, index) => `case ${index}`), "case 0"]; + // Most tests measured at a second, two long ones, and the last five unmeasured. + const costs: Record = Object.fromEntries( + names.slice(0, 35).map((name) => [name, 1_000]), + ); + costs["case 3"] = 200_000; + costs["case 7"] = 150_000; + // One suite per process, as in a CI shard: the owners are computed once per suite. + const runs = (spec: { index: number; count: number } | undefined) => { + const suite = { tasks: [] as { name: string }[] }; + suite.tasks = names.map((name) => ({ type: "test", name, suite })); + return suite.tasks + .filter((task) => { + let skipped = false; + skipUnownedTest( + { task, skip: () => (skipped = true) } as unknown as Parameters< + typeof skipUnownedTest + >[0], + costs, + spec, + ); + return !skipped; + }) + .map((task) => task.name); + }; + expect(runs(undefined)).toEqual(names); + const shards = Array.from({ length: 5 }, (_, index) => runs({ index: index + 1, count: 5 })); + expect(shards.flat().toSorted()).toEqual(names.toSorted()); + // The two long tests hold shards of their own. + expect(shards.find((shard) => shard.includes("case 3"))).toEqual(["case 3"]); + expect(shards.find((shard) => shard.includes("case 7"))).toEqual(["case 7"]); +}); + describe("corpusSelect", () => { const programs = ["ts", "js", "mjs", "cjs"] .flatMap((ext) => [ diff --git a/tests/harness/shard.ts b/tests/harness/shard.ts index 460a78df3..4c3b9c244 100644 --- a/tests/harness/shard.ts +++ b/tests/harness/shard.ts @@ -17,6 +17,7 @@ */ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; +import type { Suite, Test, TestContext } from "vitest"; export interface ShardSpec { /** 1-based shard index. */ @@ -122,3 +123,44 @@ export function corpusSelect( export function shardSuffix(spec: ShardSpec | undefined = parseShardSpec()): string { return spec === undefined ? "" : `, shard ${spec.index}/${spec.count}`; } + +const ownedTests = new WeakMap>(); + +/** For a suite of independent tests, called from beforeEach: skips the tests + * this shard does not own. The suite's tests are balanced across the shards + * by their measured cost (`costs`: milliseconds by test name; a test without + * a measurement weighs as much as the median measured one), so every shard + * runs a similar share and the union of the shards runs each test once. */ +export function skipUnownedTest( + context: TestContext, + costs: Readonly>, + spec: ShardSpec | undefined = parseShardSpec(), +): void { + const suite = context.task.suite; + if (spec === undefined || spec.count === 1 || suite === undefined) return; + let owned = ownedTests.get(suite); + if (owned === undefined) { + const known = Object.values(costs).sort((a, b) => a - b); + const median = known.length === 0 ? 1 : known[Math.floor(known.length / 2)]!; + const tests = suite.tasks.filter((task): task is Test => task.type === "test"); + // test.each titles without their parameters repeat; later repeats get "#2", "#3"... as their key. + const seen = new Map(); + const keys = new Map( + tests.map((test) => { + const count = (seen.get(test.name) ?? 0) + 1; + seen.set(test.name, count); + return [test, count === 1 ? test.name : `${test.name} #${count}`]; + }), + ); + owned = new Set( + balancedShardSelect( + tests, + (test) => keys.get(test)!, + (test) => Math.max(1, costs[test.name] ?? median), + spec, + ), + ); + ownedTests.set(suite, owned); + } + if (!owned.has(context.task)) context.skip(); +}