Shard the zero-cost gate and reuse baseline work across runs (#2365)

AGENTS.md

@@ -25,6 +25,8 @@ their supported native-clang contracts locally; other hosts retain those
 checks in the Sandboxes. Both lanes green is the bar before shipping any
 change.
 
+`SCRIPTC_SANDBOX_NATIVE_REFERENCE` names a built checkout inside a prepared `SCRIPTC_SANDBOX_SNAPSHOT` or image: each lane copies that checkout's LLVM helper and runtime pack instead of rebuilding them when their source inputs are identical. Sandbox creation is rate limited per team in vCPUs per minute, so the runner retries a rate-limited creation with backoff. Larger Sandboxes (`SCRIPTC_SANDBOX_VCPUS` up to 32, with `SCRIPTC_TEST_WORKERS` at half the vCPUs) shorten the lanes most.
+
 `VERCEL_OIDC_TOKEN` is the preferred credential. `VERCEL_TOKEN` remains
 supported with explicit `VERCEL_TEAM_ID` and `VERCEL_PROJECT_ID`. The custom
 `SCRIPTC_SANDBOX_IMAGE` is optional: without it, the gate starts from
@@ -58,6 +60,8 @@ A change that must not cost good code anything, such as a compatibility fix that
 node scripts/zero-cost-gate.mjs --baseline=/vercel/sandbox/prev --candidate=. --allow='\barguments\b'
 ```
 
+The whole gate takes 30 to 75 minutes on one 8-vCPU Sandbox, so shard it across Sandboxes. `--shard=i/n` splits the bench and corpus programs; the self-hosted compiler runs in shard 1, or alone with `--set=self`, and its candidate-source build overlaps its comparison build on hosts with 40 GB or more. Each shard writes `--json`, and `node scripts/zero-cost-merge.mjs <shard.json>...` merges them into one verdict, recomputing the compile-time intervals over every shard. `--record-baseline-cache=<file>` builds only the baseline and records each program's artifact hashes; `--baseline-cache=<file>` then skips the baseline build of every program whose candidate artifacts hash the same, which halves the work for a change that leaves most programs identical. A record applies only to the built baseline checkout, toolchain, and paths it was made with, and is ignored otherwise; a program whose entry, relative imports, or optimization level changed since the record builds its baseline.
+
 Test location follows scope:
 
 - Co-locate white-box unit tests with implementation files under

scripts/sandbox-config.mjs

@@ -291,3 +291,22 @@ export function sandboxTestSourceConfig(env) {
     createArgs: ["--image", image.sandboxImage],
   };
 }
+
+/** A built checkout inside a prepared snapshot whose native artifacts a
+ * lane may reuse when their inputs match (SCRIPTC_SANDBOX_NATIVE_REFERENCE).
+ * It needs the snapshot's tools, so it is ignored for the managed image. */
+export function sandboxNativeReference(env) {
+  loadLocalEnv();
+  const source = env ?? process.env;
+  const reference = source.SCRIPTC_SANDBOX_NATIVE_REFERENCE?.trim();
+  if (!reference) return undefined;
+  if (!sandboxTestSourceConfig(source).prepared) {
+    throw new Error(
+      "SCRIPTC_SANDBOX_NATIVE_REFERENCE needs SCRIPTC_SANDBOX_SNAPSHOT or SCRIPTC_SANDBOX_IMAGE",
+    );
+  }
+  if (!/^\/[\w./-]+$/.test(reference) || reference.split("/").includes("..")) {
+    throw new Error("SCRIPTC_SANDBOX_NATIVE_REFERENCE must be an absolute path without '..'");
+  }
+  return reference.replace(/\/+$/, "");
+}

scripts/sandbox-test.mjs

@@ -9,6 +9,7 @@ import { parseArgs } from "node:util";
 import { fileURLToPath } from "node:url";
 import {
   sandboxBootstrapCommand,
+  sandboxNativeReference,
   sandboxTestSourceConfig,
   sandboxRunnerConfig,
   sandboxEffectWorkerAllocation,
@@ -112,6 +113,38 @@ const hostInvariantFiles = [
   "packages/runtime/test/tonumber.test.ts",
   "packages/runtime/test/url.test.ts",
 ];
+// Native artifacts a lane needs and every source path their builds read.
+// Output paths are excluded from the input comparison by these names.
+const NATIVE_OUTPUT_NAMES = [
+  "bin",
+  "artifacts",
+  "runtime-pack.json",
+  ".runtime-pack-*",
+  "node_modules",
+  "dist",
+  "build",
+  ".cache",
+];
+const LLVM_HELPER_INPUTS = ["native/llvm-codegen", "packages/llvm-linux-x64-gnu"];
+const NATIVE_ARTIFACTS = {
+  llvm: {
+    inputs: LLVM_HELPER_INPUTS,
+    outputs: ["packages/llvm-linux-x64-gnu/bin/scriptc-llvm-codegen"],
+  },
+  runtime: {
+    inputs: [
+      ...LLVM_HELPER_INPUTS,
+      "packages/runtime",
+      "packages/runtime-pack-common",
+      "packages/runtime-linux-x64-gnu",
+      "packages/runtime-darwin-arm64/scripts",
+    ],
+    outputs: [
+      "packages/runtime-linux-x64-gnu/artifacts",
+      "packages/runtime-linux-x64-gnu/runtime-pack.json",
+    ],
+  },
+};
 const hostSchedule = sandboxHostSchedule(process.platform, hostInvariantFiles);
 const nativeHostInvariantFiles = hostSchedule.localInvariantFiles;
 const remoteWorkspaceReset = workspaceResetCommand("/workspace");
@@ -188,6 +221,8 @@ Environment:
   VERCEL_TOKEN              Access-token fallback; also set VERCEL_TEAM_ID + VERCEL_PROJECT_ID
   SCRIPTC_SANDBOX_IMAGE     Optional fully qualified VCR image (default: vercel/sandbox/universal)
   SCRIPTC_SANDBOX_SNAPSHOT  Optional prepared snapshot with the pinned toolchain (exclusive with IMAGE)
+  SCRIPTC_SANDBOX_NATIVE_REFERENCE  Optional built checkout inside the snapshot; its LLVM helper
+                            and runtime pack are reused when their inputs are identical
   SCRIPTC_SANDBOX_VCPUS     vCPUs per sandbox (default: 8)
   SCRIPTC_SANDBOX_TIMEOUT   sandbox and command timeout (default: 45m)
   SCRIPTC_TEST_WORKERS      Vitest workers per sandbox (default: 4)
@@ -197,6 +232,7 @@ Environment:
 }
 
 const sourceConfig = sandboxTestSourceConfig();
+const nativeReference = sandboxNativeReference();
 const bootstrapCommand = sandboxBootstrapCommand(sourceConfig.prepared);
 const vercelConfig = sandboxVercelConfig();
 const vercelProcessEnv = sandboxVercelEnvironment(vercelConfig);
@@ -496,6 +532,42 @@ const execIn = async (
   }
 };
 
+/** The command that provides one native artifact (`llvm`: the Linux LLVM
+ * helper, `runtime`: the Linux runtime pack). It builds the artifact unless
+ * SCRIPTC_SANDBOX_NATIVE_REFERENCE names a built checkout inside the Sandbox
+ * (a prepared snapshot's) whose inputs for that artifact are identical; then
+ * it copies the reference's artifact. The runtime build runs the helper, so
+ * its inputs include the helper's. */
+function nativeArtifactCommand(artifact) {
+  const pkg =
+    artifact === "llvm" ? "@scriptc/llvm-linux-x64-gnu" : "@scriptc/runtime-linux-x64-gnu";
+  if (nativeReference === undefined) return ["pnpm", ["--filter", pkg, "build:native"]];
+  const build = `pnpm --filter ${pkg} build:native`;
+  const { inputs, outputs } = NATIVE_ARTIFACTS[artifact];
+  const excludes = NATIVE_OUTPUT_NAMES.map((name) => `-x '${name}'`).join(" ");
+  const ref = nativeReference;
+  const script = [
+    "set -eu",
+    `ref='${ref}'`,
+    "same=1",
+    ...outputs.map((output) => `[ -e "$ref/${output}" ] || same=0`),
+    ...inputs.map(
+      (input) =>
+        `[ "$same" = 0 ] || diff -rq ${excludes} "$ref/${input}" "${input}" >/dev/null 2>&1 || same=0`,
+    ),
+    'if [ "$same" = 1 ]; then',
+    ...outputs.map(
+      (output) =>
+        `  rm -rf "${output}" && mkdir -p "$(dirname "${output}")" && cp -R "$ref/${output}" "${output}"`,
+    ),
+    `  echo "reused the ${artifact} artifact from $ref: identical inputs"`,
+    "else",
+    `  ${build}`,
+    "fi",
+  ].join("\n");
+  return ["sh", ["-c", script]];
+}
+
 async function preflight() {
   console.log("Sandbox preflight:");
   console.log(`  auth:  ${vercelConfig.authSource}`);
@@ -582,7 +654,10 @@ async function createWorker(worker) {
     vcpus,
     "--non-persistent",
   ];
-  for (let attempt = 1; attempt <= 2; attempt++) {
+  // Sandbox creation is rate limited per team in vCPUs per minute, so a
+  // large fan-out (many shards, large Sandboxes) retries with backoff.
+  const attempts = 6;
+  for (let attempt = 1; attempt <= attempts; attempt++) {
     try {
       await vercel(args, {
         idleTimeoutMs: 60_000,
@@ -598,8 +673,12 @@ async function createWorker(worker) {
         await execIn(worker, "true", [], {}, "create status", 60_000, "/");
         return;
       } catch (error) {
-        if (attempt === 2) throw error;
-        console.warn(`[${worker.label}] Sandbox is not reachable; retrying creation once...`);
+        if (attempt === attempts) throw error;
+        const delay = 15_000 * attempt;
+        console.warn(
+          `[${worker.label}] Sandbox is not reachable; retrying creation in ${delay / 1000}s...`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, delay));
       }
     }
   }
@@ -781,19 +860,13 @@ try {
         // including TypeScript's incremental metadata. Rebuild both together.
         await execIn(worker, "pnpm", ["build:fresh"], {}, "", 2 * 60_000);
         // Workspace builds deliberately do not rebuild packaged native artifacts.
-        // Every remote lane needs the Linux helper and runtime from this worktree.
+        // Every remote lane needs the Linux helper and runtime from this worktree,
+        // built from its sources or reused from a reference checkout whose
+        // native inputs are identical.
+        await execIn(worker, ...nativeArtifactCommand("llvm"), {}, "LLVM helper", 5 * 60_000);
         await execIn(
           worker,
-          "pnpm",
-          ["--filter", "@scriptc/llvm-linux-x64-gnu", "build:native"],
-          {},
-          "LLVM helper",
-          5 * 60_000,
-        );
-        await execIn(
-          worker,
-          "pnpm",
-          ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"],
+          ...nativeArtifactCommand("runtime"),
           { CC: "zig", AR: "zig" },
           "runtime pack",
           5 * 60_000,

scripts/zero-cost-compare.mjs

@@ -6,6 +6,7 @@
  * the first unit that differs. The self-hosted compiler's IR and LLVM exceed
  * V8's maximum string length, so splitting works on Buffers and hashes each
  * unit instead of decoding the artifact. */
+import assert from "node:assert/strict";
 import { createHash } from "node:crypto";
 
 const hash = (bytes) => createHash("sha1").update(bytes).digest("hex");
@@ -487,6 +488,57 @@ export function compareCounts(baseline, candidate) {
   };
 }
 
+/** The report lines the gate prints for one program. */
+export function programLines(entry) {
+  const lines = [];
+  const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`;
+  const label = {
+    identical: "identical",
+    allowed: "allowed  ",
+    differs: "DIFFERS  ",
+    "build-error": "ERROR    ",
+    "both-fail": "both-fail",
+  }[entry.verdict];
+  const notes = [];
+  for (const [kind, result] of Object.entries(entry.artifacts))
+    if (!result.identical) {
+      const counts = result.changed
+        ? ` (${result.changed.length} changed, ${result.added.length} added, ${result.removed.length} removed)`
+        : "";
+      notes.push(`${kind}: ${result.first}${counts}`);
+    }
+  const size = entry.artifacts.exe?.bytes;
+  if (size && size.baseline !== size.candidate)
+    notes.push(
+      `size ${size.baseline} -> ${size.candidate} B (${pct(size.candidate / size.baseline)})`,
+    );
+  if (entry.cachegrind?.total) {
+    const { total, deltas } = entry.cachegrind;
+    notes.push(
+      `Ir ${total.baseline} -> ${total.candidate} (${pct(total.candidate / Math.max(1, total.baseline))})` +
+        (deltas[0]
+          ? `, largest ${deltas[0].name} ${deltas[0].delta > 0 ? "+" : ""}${deltas[0].delta}`
+          : ""),
+    );
+  } else if (entry.cachegrind?.skipped)
+    notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`);
+  for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`);
+  lines.push(`${label}  ${entry.id}${notes.length ? `  ${notes.join("; ")}` : ""}`);
+  // A build error's first line is often a warning; print all of it.
+  const errors =
+    entry.baselineError !== undefined && entry.baselineError === entry.candidateError
+      ? [["baseline and candidate", entry.baselineError]]
+      : [
+          ["baseline", entry.baselineError],
+          ["candidate", entry.candidateError],
+        ].filter(([, error]) => error !== undefined);
+  for (const [side, error] of errors) {
+    lines.push(`           ${side}:`);
+    for (const line of error.split("\n")) lines.push(`             ${line}`);
+  }
+  return lines;
+}
+
 /* ── verdicts ──────────────────────────────────────────────────────────── */
 
 /** One program's verdict from its build results and artifact comparisons.
@@ -539,3 +591,142 @@ export function timeRatio(rows, resamples = 2000) {
     high: ratios[Math.floor(resamples * 0.975)],
   };
 }
+
+/** The compile-time verdict for one group of timing rows (`bench` or
+ * `self`): it fails when the 95% interval lies entirely above the allowed
+ * growth, and is inconclusive when the interval straddles it. */
+export function timingVerdict(name, rows, maxGrowth) {
+  const result = timeRatio(rows);
+  const fails = result.low > 1 + maxGrowth;
+  const verdict = fails
+    ? "SLOWER"
+    : result.high > 1 + maxGrowth
+      ? "inconclusive (rerun with more runs)"
+      : "within tolerance";
+  const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`;
+  return {
+    result,
+    fails,
+    line:
+      `time       ${name} ${rows.length > 1 ? "geomean " : ""}${pct(result.ratio)} ` +
+      `[95% CI ${pct(result.low)}, ${pct(result.high)}]: ${verdict}`,
+    failure: fails
+      ? `${name} compile time grew ${pct(result.ratio)} (CI low ${pct(result.low)})`
+      : null,
+  };
+}
+
+/** Timing rows grouped as the gate judges them. */
+export function timingGroups(rows) {
+  return {
+    bench: rows.filter((r) => r.id.startsWith("bench/")),
+    self: rows.filter((r) => r.id === "self/cli"),
+  };
+}
+
+/* ── baseline cache ────────────────────────────────────────────────────── */
+
+/** A content hash of one artifact file, as the baseline cache records it. */
+export const artifactHash = hash;
+
+/** The hash of what a program's build reads, as the baseline cache records
+ * it: `files` (the entry and its relative imports) by path relative to
+ * `root` and content, read through `read`, and the optimization level. */
+export function programInputsHash(root, files, optimization, read) {
+  const h = createHash("sha1").update(`${optimization}\0`);
+  for (const file of [...files].sort())
+    h.update(`${file.startsWith(root + "/") ? file.slice(root.length + 1) : file}\0`)
+      .update(read(file))
+      .update("\0");
+  return h.digest("hex");
+}
+
+/** The baseline cache entry recorded for program `id`, or undefined when the
+ * cache has none or recorded it for other inputs (`programInputsHash`): a
+ * program or an import that changed since the record must build its
+ * baseline, or a recorded build error or artifact hash of the old sources
+ * would settle the new sources' comparison. */
+export function recordedBaseline(cache, id, inputs) {
+  const entry = cache?.programs[id];
+  return entry !== undefined && entry.inputs === inputs ? entry : undefined;
+}
+
+/** The comparison a cached baseline entry settles without building the
+ * baseline, or null when the baseline must be built: identical artifacts
+ * (every recorded hash equals the candidate's) or the same build error on
+ * both sides. `candidate` holds the candidate's build result and `hashes`
+ * its artifact hashes. */
+export function cachedComparison(entry, candidate, hashes) {
+  if (entry === undefined) return null;
+  if (!entry.ok || !candidate.ok)
+    return !entry.ok && !candidate.ok && entry.error === candidate.error
+      ? {
+          baseline: { ok: false, error: entry.error, ms: entry.ms ?? null, cached: true },
+          artifacts: {},
+        }
+      : null;
+  const kinds = Object.keys(entry.hashes);
+  const differs = (kind) => entry.hashes[kind] === null || entry.hashes[kind] !== hashes[kind];
+  if (kinds.length === 0 || kinds.some(differs)) return null;
+  const artifacts = {};
+  for (const kind of kinds) artifacts[kind] = { identical: true };
+  if (artifacts.exe) artifacts.exe.bytes = { baseline: entry.exeBytes, candidate: entry.exeBytes };
+  return { baseline: { ok: true, ms: entry.ms ?? null }, artifacts };
+}
+
+/* ── sharded runs ──────────────────────────────────────────────────────── */
+
+/** Merge the --json reports of gate shards that split one good-code set
+ * (--shard, --set) into a single report with one verdict. Program entries
+ * and their failures are concatenated; the compile-time groups and the
+ * "nothing was compared" check are recomputed over all shards. */
+export function mergeReports(reports) {
+  assert(reports.length > 0, "no gate reports to merge");
+  const [first] = reports;
+  for (const report of reports) {
+    assert(report.schema === first.schema, "gate reports have different schemas");
+    for (const label of ["baseline", "candidate"])
+      assert(
+        report.trees[label].revision === first.trees[label].revision,
+        `gate reports compare different ${label} revisions`,
+      );
+  }
+  const globals = new Set(reports.flatMap((r) => r.globalFailures ?? []));
+  const merged = {
+    ...first,
+    shards: reports.length,
+    programs: reports.flatMap((r) => r.programs),
+    failures: reports.flatMap((r) => r.failures.filter((f) => !globals.has(f))),
+    globalFailures: [],
+    timing: null,
+    candidateSelf: reports.find((r) => r.candidateSelf)?.candidateSelf,
+    elapsedMs: Math.max(...reports.map((r) => r.elapsedMs ?? 0)),
+  };
+  const ids = new Set();
+  for (const p of merged.programs) {
+    assert(!ids.has(p.id), `program ${p.id} appears in more than one gate report`);
+    ids.add(p.id);
+  }
+  const timed = reports.filter((r) => r.timing !== null);
+  const lines = [];
+  if (timed.length > 0) {
+    const rows = timed.flatMap((r) => r.timing.rows);
+    const maxGrowth = timed[0].timing.maxGrowth;
+    merged.timing = { runs: timed[0].timing.runs, maxGrowth, rows, groups: {} };
+    for (const [name, group] of Object.entries(timingGroups(rows))) {
+      if (group.length === 0) continue;
+      const { result, line, failure } = timingVerdict(name, group, maxGrowth);
+      merged.timing.groups[name] = result;
+      lines.push(line);
+      if (failure !== null) merged.globalFailures.push(failure);
+    }
+  }
+  const counts = {};
+  for (const p of merged.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1;
+  if (!merged.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict)))
+    merged.globalFailures.push("no program built with both checkouts, so nothing was compared");
+  merged.failures.push(...merged.globalFailures);
+  merged.counts = counts;
+  merged.verdict = merged.failures.length > 0 ? "fail" : "pass";
+  return { report: merged, timingLines: lines };
+}

scripts/zero-cost-gate.mjs

@@ -13,6 +13,7 @@
  * pattern signature) may differ; any other difference fails the gate. */
 import assert from "node:assert/strict";
 import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import {
   existsSync,
   mkdirSync,
@@ -23,12 +24,14 @@ import {
   statSync,
   writeFileSync,
 } from "node:fs";
-import { availableParallelism } from "node:os";
+import { availableParallelism, totalmem } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { parseArgs } from "node:util";
 import { readWorkloads } from "../benchmarks/runtime/workloads.mjs";
 import {
+  artifactHash,
+  cachedComparison,
   cachegrindCounts,
   compareArtifact,
   compareCounts,
@@ -38,8 +41,12 @@ import {
   dynamicTierSymbols,
   median,
   nmOutput,
+  programInputsHash,
+  programLines,
   programVerdict,
-  timeRatio,
+  recordedBaseline,
+  timingGroups,
+  timingVerdict,
 } from "./zero-cost-compare.mjs";
 
 const repoRoot = fileURLToPath(new URL("..", import.meta.url));
@@ -61,6 +68,8 @@ const { values } = parseArgs({
     cachegrind: { type: "boolean", default: false },
     jobs: { type: "string", default: String(Math.max(1, availableParallelism())) },
     json: { type: "string" },
+    "baseline-cache": { type: "string" },
+    "record-baseline-cache": { type: "string" },
     keep: { type: "boolean", default: false },
     help: { type: "boolean", default: false },
   },
@@ -72,6 +81,7 @@ if (values.help || values.baseline === undefined) {
        [--timing-runs=N] [--self-timing-runs=N] [--max-time-growth=2]
        [--max-size-growth=0]
        [--cachegrind] [--jobs=N] [--json=<file>] [--keep]
+       [--baseline-cache=<file>] [--record-baseline-cache=<file>]
 
 Each checkout must be built (packages/cli/dist/bootstrap.js and this host's
 runtime pack). Both compile the same program files, taken from --programs
@@ -82,7 +92,9 @@ benchmarks/builds applications), self (packages/compiler/src/native/cli.ts,
 LLVM only; the candidate's own compiler source must also build with the
 candidate), corpus (tests/corpus TypeScript programs; --corpus=all adds
 JavaScript entries). --quick is --set=bench --timing-runs=0, for CI.
---shard=i/n keeps every n-th corpus program; --only filters program ids.
+--shard=i/n keeps every n-th bench and corpus program; the self program runs
+in shard 1. scripts/zero-cost-merge.mjs merges the shards' --json reports
+into one verdict. --only filters program ids.
 
 --allow=<regex> is the change's pattern signature: programs whose source
 files match it may differ (reported as "allowed"). Any other difference in
@@ -106,6 +118,18 @@ cachegrind (bench workloads with their arguments, corpus programs without)
 and compares instruction counts per function of the executable's own
 symbols, excluding the allocator's prefault thread.
 
+--record-baseline-cache=<file> builds only the baseline and records each
+program's artifact hashes (or build error) in <file>, with a hash of the
+program's inputs (its entry, its relative imports and its optimization
+level), then exits.
+--baseline-cache=<file> reads such a record: a program whose candidate
+artifacts hash the same as the recorded baseline's is identical without a
+baseline build, and the same build error on both sides is both-fail; any
+other program builds the baseline as usual, and so does a program whose
+inputs changed since the record. The record applies only to the
+baseline revision and toolchain it was made with; otherwise it is ignored.
+The self program always builds both sides, since its builds are timed.
+
 Exit status: 0 when the gate passes, 1 when it fails.`);
   process.exit(values.help ? 0 : 2);
 }
@@ -200,13 +224,7 @@ function goodCodeSet() {
         if (main !== undefined) entries.push(main);
       } else if (exts.some((ext) => name.endsWith(`.${ext}`))) entries.push(path);
     }
-    let selected = entries;
-    if (values.shard) {
-      const [index, count] = values.shard.split("/").map(Number);
-      assert.ok(index >= 1 && index <= count, "--shard must be i/n with 1 <= i <= n");
-      selected = entries.filter((_, k) => k % count === index - 1);
-    }
-    for (const entry of selected) {
+    for (const entry of entries) {
       const head = readFileSync(entry, "utf8").split("\n", 2);
       const speed = head.some((line) => /^\/\/ @optimization:\s*speed\s*$/.test(line));
       programs.push({
@@ -217,7 +235,17 @@ function goodCodeSet() {
       });
     }
   }
-  return programs.filter((p) => only === null || only.test(p.id));
+  const selected = programs.filter((p) => only === null || only.test(p.id));
+  if (!values.shard) return selected;
+  const [index, count] = values.shard.split("/").map(Number);
+  assert.ok(
+    Number.isInteger(index) && Number.isInteger(count) && index >= 1 && index <= count,
+    "--shard must be i/n with 1 <= i <= n",
+  );
+  // The self program takes minutes and pairs its own timing: shard 1 owns it.
+  const self = selected.filter((p) => p.kind === "source");
+  const rest = selected.filter((p) => p.kind !== "source");
+  return [...(index === 1 ? self : []), ...rest.filter((_, k) => k % count === index - 1)];
 }
 
 /* ── builds ────────────────────────────────────────────────────────────── */
@@ -415,7 +443,74 @@ function revision(root) {
 const fmtMs = (ms) => `${(ms / 1000).toFixed(2)}s`;
 const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2)}%`;
 
-const programs = goodCodeSet();
+/* ── baseline cache ────────────────────────────────────────────────────── */
+
+/** Every file under `dir`, recursively, in a stable order. */
+function filesUnder(dir) {
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir, { recursive: true, withFileTypes: true })
+    .filter((d) => d.isFile())
+    .map((d) => join(d.parentPath, d.name))
+    .sort();
+}
+
+/** What a baseline build depends on besides the program: the checkout's
+ * built compiler and CLI, its runtime packs and LLVM helpers, Node, the
+ * host and the program and checkout paths that reach the artifacts. */
+function toolchainFingerprint(root) {
+  const h = createHash("sha1");
+  h.update(JSON.stringify([process.platform, process.arch, process.version, root, programsRoot]));
+  const clang = spawnSync("clang", ["--version"], { encoding: "utf8" });
+  h.update(clang.stdout ?? "");
+  const packages = join(root, "packages");
+  for (const name of readdirSync(packages).sort()) {
+    const dir = join(packages, name);
+    const files = [
+      ...filesUnder(join(dir, "dist")),
+      ...(name.startsWith("runtime-")
+        ? [join(dir, "runtime-pack.json"), ...filesUnder(join(dir, "artifacts"))]
+        : []),
+      ...(name.startsWith("llvm-") ? filesUnder(join(dir, "bin")) : []),
+    ];
+    for (const file of files)
+      if (existsSync(file)) h.update(relative(root, file)).update(readFileSync(file));
+  }
+  return h.digest("hex");
+}
+
+/** The hash of `program`'s inputs, which its baseline cache entry records. */
+function inputsHash(program) {
+  return programInputsHash(
+    programsRoot,
+    programInputs(program.entry),
+    program.kind === "source" ? "source" : (program.optimization ?? "release"),
+    (file) => readFileSync(file),
+  );
+}
+
+/** The content hashes of one build's artifacts (null for a missing one). */
+function buildHashes(build) {
+  const hashes = {};
+  for (const kind of ["ir", "llvm", "exe"])
+    if (kind in build)
+      hashes[kind] = build[kind] === null ? null : artifactHash(readFileSync(build[kind]));
+  return hashes;
+}
+
+const recordPath = values["record-baseline-cache"];
+let baselineCache = null;
+if (recordPath === undefined && values["baseline-cache"] !== undefined) {
+  const file = resolve(values["baseline-cache"]);
+  const cache = existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : null;
+  const fingerprint = cache === null ? null : toolchainFingerprint(trees.baseline);
+  if (cache?.schema === 2 && cache.fingerprint === fingerprint) baselineCache = cache;
+  else
+    console.log(
+      `baseline cache ${file} ${cache === null ? "is missing" : "was made with another baseline or toolchain"}; building every baseline`,
+    );
+}
+
+const programs = goodCodeSet().filter((p) => recordPath === undefined || p.kind !== "source");
 assert.ok(programs.length > 0, "the selected good-code set is empty");
 const report = {
   schema: 1,
@@ -429,6 +524,13 @@ const report = {
   programs: [],
   timing: null,
   failures: [],
+  // Failures judged over the whole good-code set (compile time, nothing
+  // compared); scripts/zero-cost-merge.mjs recomputes them across shards.
+  globalFailures: [],
+  baselineCache:
+    baselineCache === null
+      ? null
+      : { programs: Object.keys(baselineCache.programs).length, used: 0 },
 };
 console.log(
   `zero-cost gate: ${programs.length} programs (${[...sets].join(", ")}), ` +
@@ -437,18 +539,95 @@ console.log(
 
 let failed = false;
 const started = Date.now();
+
+if (recordPath !== undefined) {
+  const record = {
+    schema: 2,
+    date: report.date,
+    fingerprint: toolchainFingerprint(trees.baseline),
+    baseline: report.trees.baseline,
+    programs: {},
+  };
+  await pool(
+    programs.map((program) => ({
+      run: async () => {
+        const build = await buildProgram(program, "baseline");
+        const inputs = inputsHash(program);
+        record.programs[program.id] = build.ok
+          ? {
+              inputs,
+              ok: true,
+              ms: build.ms,
+              hashes: buildHashes(build),
+              ...(build.exe ? { exeBytes: statSync(build.exe).size } : {}),
+            }
+          : { inputs, ok: false, ms: build.ms ?? null, error: build.error };
+        if (build.dir) rmSync(build.dir, { recursive: true, force: true });
+      },
+    })),
+  );
+  mkdirSync(dirname(resolve(recordPath)), { recursive: true });
+  writeFileSync(recordPath, JSON.stringify(record) + "\n");
+  rmSync(work, { recursive: true, force: true });
+  const failures = Object.values(record.programs).filter((p) => !p.ok).length;
+  console.log(
+    `zero-cost gate: recorded ${programs.length} baseline programs (${failures} build errors) ` +
+      `in ${recordPath} in ${fmtMs(Date.now() - started)}`,
+  );
+  process.exit(0);
+}
+
+/** The candidate's own compiler source must still build with the candidate
+ * (see below). On a host with room for a third self-hosted build it runs
+ * alongside the comparison builds instead of after them. */
+const checkCandidateSelf =
+  sets.has("self") && programsRoot !== trees.candidate && programs.some((p) => p.kind === "source");
+const buildCandidateSelf = () =>
+  buildProgram(
+    {
+      id: "self/candidate-source",
+      entry: join(trees.candidate, "packages/compiler/src/native/cli.ts"),
+      kind: "source",
+    },
+    "candidate",
+  );
+const earlyCandidateSelf =
+  checkCandidateSelf && totalmem() >= 40 * 1024 ** 3 ? buildCandidateSelf() : null;
+
 await pool(
   programs.map((program) => ({
     heavy: program.kind === "source",
     run: async () => {
-      const [baseline, candidate] = await Promise.all([
-        buildProgram(program, "baseline"),
-        buildProgram(program, "candidate"),
-      ]);
+      const recorded =
+        program.kind === "source" || baselineCache === null
+          ? undefined
+          : recordedBaseline(baselineCache, program.id, inputsHash(program));
+      let baseline;
+      let candidate;
+      let settled = null;
+      if (recorded === undefined)
+        [baseline, candidate] = await Promise.all([
+          buildProgram(program, "baseline"),
+          buildProgram(program, "candidate"),
+        ]);
+      else {
+        candidate = await buildProgram(program, "candidate");
+        settled = cachedComparison(recorded, candidate, candidate.ok ? buildHashes(candidate) : {});
+        // A cached identical baseline has the candidate's artifacts.
+        baseline =
+          settled === null
+            ? await buildProgram(program, "baseline")
+            : settled.baseline.ok
+              ? { ...candidate, ms: settled.baseline.ms, cached: true }
+              : settled.baseline;
+        if (settled !== null) report.baselineCache.used++;
+      }
       const allowed =
         allow !== null &&
         programInputs(program.entry).some((f) => allow.test(readFileSync(f, "utf8")));
-      const artifacts = baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {};
+      const artifacts =
+        settled?.artifacts ??
+        (baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {});
       // The benchmark workloads and the self-hosted compiler always build.
       const required = !program.id.startsWith("corpus/");
       const { verdict, fails } = programVerdict({
@@ -465,6 +644,7 @@ await pool(
         buildMs: { baseline: baseline.ms ?? null, candidate: candidate.ms ?? null },
         artifacts,
       };
+      if (settled !== null) entry.cachedBaseline = true;
       if (!baseline.ok) entry.baselineError = baseline.error;
       if (!candidate.ok) entry.candidateError = candidate.error;
       const size = artifacts.exe?.bytes;
@@ -506,7 +686,8 @@ await pool(
       nmCache.delete(baseline.exe);
       nmCache.delete(candidate.exe);
       if (!values.keep) {
-        if (baseline.dir) rmSync(baseline.dir, { recursive: true, force: true });
+        if (baseline.dir && !baseline.cached)
+          rmSync(baseline.dir, { recursive: true, force: true });
         if (candidate.dir) rmSync(candidate.dir, { recursive: true, force: true });
       }
     },
@@ -514,50 +695,7 @@ await pool(
 );
 
 function printProgram(entry) {
-  const label = {
-    identical: "identical",
-    allowed: "allowed  ",
-    differs: "DIFFERS  ",
-    "build-error": "ERROR    ",
-    "both-fail": "both-fail",
-  }[entry.verdict];
-  const notes = [];
-  for (const [kind, result] of Object.entries(entry.artifacts))
-    if (!result.identical) {
-      const counts = result.changed
-        ? ` (${result.changed.length} changed, ${result.added.length} added, ${result.removed.length} removed)`
-        : "";
-      notes.push(`${kind}: ${result.first}${counts}`);
-    }
-  const size = entry.artifacts.exe?.bytes;
-  if (size && size.baseline !== size.candidate)
-    notes.push(
-      `size ${size.baseline} -> ${size.candidate} B (${pct(size.candidate / size.baseline)})`,
-    );
-  if (entry.cachegrind?.total) {
-    const { total, deltas } = entry.cachegrind;
-    notes.push(
-      `Ir ${total.baseline} -> ${total.candidate} (${pct(total.candidate / Math.max(1, total.baseline))})` +
-        (deltas[0]
-          ? `, largest ${deltas[0].name} ${deltas[0].delta > 0 ? "+" : ""}${deltas[0].delta}`
-          : ""),
-    );
-  } else if (entry.cachegrind?.skipped)
-    notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`);
-  for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`);
-  console.log(`${label}  ${entry.id}${notes.length ? `  ${notes.join("; ")}` : ""}`);
-  // A build error's first line is often a warning; print all of it.
-  const errors =
-    entry.baselineError !== undefined && entry.baselineError === entry.candidateError
-      ? [["baseline and candidate", entry.baselineError]]
-      : [
-          ["baseline", entry.baselineError],
-          ["candidate", entry.candidateError],
-        ].filter(([, error]) => error !== undefined);
-  for (const [side, error] of errors) {
-    console.log(`           ${side}:`);
-    for (const line of error.split("\n")) console.log(`             ${line}`);
-  }
+  for (const line of programLines(entry)) console.log(line);
 }
 
 /* Compile time: paired --emit=llvm builds, the baseline and the candidate
@@ -603,29 +741,15 @@ if (timingRuns > 0) {
     );
   }
   report.timing = { runs: timingRuns, maxGrowth: maxTimeGrowth, rows, groups: {} };
-  const groups = {
-    bench: rows.filter((r) => r.id.startsWith("bench/")),
-    self: rows.filter((r) => r.id === "self/cli"),
-  };
-  for (const [name, group] of Object.entries(groups)) {
+  for (const [name, group] of Object.entries(timingGroups(rows))) {
     if (group.length === 0) continue;
-    const result = timeRatio(group);
+    const { result, line, failure } = timingVerdict(name, group, maxTimeGrowth);
     report.timing.groups[name] = result;
-    const verdict =
-      result.low > 1 + maxTimeGrowth
-        ? "SLOWER"
-        : result.high > 1 + maxTimeGrowth
-          ? "inconclusive (rerun with more runs)"
-          : "within tolerance";
-    console.log(
-      `time       ${name} ${group.length > 1 ? "geomean " : ""}${pct(result.ratio)} ` +
-        `[95% CI ${pct(result.low)}, ${pct(result.high)}]: ${verdict}`,
-    );
-    if (result.low > 1 + maxTimeGrowth) {
+    console.log(line);
+    if (failure !== null) {
       failed = true;
-      report.failures.push(
-        `${name} compile time grew ${pct(result.ratio)} (CI low ${pct(result.low)})`,
-      );
+      report.failures.push(failure);
+      report.globalFailures.push(failure);
     }
   }
 }
@@ -634,12 +758,8 @@ if (timingRuns > 0) {
 // from one checkout), so a change that adds compiler code the self-hosted
 // frontend cannot lower would pass unnoticed. The candidate's own compiler
 // source must still build with the candidate (LLVM only, not compared).
-if (sets.has("self") && programsRoot !== trees.candidate) {
-  const entry = join(trees.candidate, "packages/compiler/src/native/cli.ts");
-  const result = await buildProgram(
-    { id: "self/candidate-source", entry, kind: "source" },
-    "candidate",
-  );
+if (checkCandidateSelf) {
+  const result = await (earlyCandidateSelf ?? buildCandidateSelf());
   report.candidateSelf = result.ok
     ? { ok: true, buildMs: result.ms }
     : { ok: false, error: result.error };
@@ -658,7 +778,9 @@ const counts = {};
 for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1;
 if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) {
   failed = true;
-  report.failures.push("no program built with both checkouts, so nothing was compared");
+  const failure = "no program built with both checkouts, so nothing was compared";
+  report.failures.push(failure);
+  report.globalFailures.push(failure);
 }
 report.counts = counts;
 report.verdict = failed ? "fail" : "pass";
@@ -672,7 +794,8 @@ else console.log(`artifacts kept in ${work}`);
 console.log(
   `zero-cost gate: ${failed ? "FAIL" : "PASS"} — ${Object.entries(counts)
     .map(([k, v]) => `${v} ${k}`)
-    .join(", ")} in ${fmtMs(report.elapsedMs)}`,
+    .join(", ")} in ${fmtMs(report.elapsedMs)}` +
+    (report.baselineCache ? ` (${report.baselineCache.used} baselines from the cache)` : ""),
 );
 for (const failure of report.failures) console.log(`  ${failure}`);
 process.exit(failed ? 1 : 0);

scripts/zero-cost-merge.mjs

@@ -0,0 +1,52 @@
+/* Merge zero-cost gate shards into one verdict.
+ *
+ * A sharded gate runs scripts/zero-cost-gate.mjs several times over one
+ * good-code set (--shard=i/n for the bench and corpus programs, the self
+ * program in shard 1 or its own --set=self run), each with --json. This
+ * script merges those reports: program verdicts and failures are
+ * concatenated, and the compile-time verdicts and the "nothing was
+ * compared" check are recomputed over every shard's rows. */
+import { readFileSync, writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+import { mergeReports, programLines } from "./zero-cost-compare.mjs";
+
+const { values, positionals } = parseArgs({
+  allowPositionals: true,
+  options: {
+    json: { type: "string" },
+    quiet: { type: "boolean", default: false },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help || positionals.length === 0) {
+  console.log(`Usage: node scripts/zero-cost-merge.mjs [--json=<file>] [--quiet] <shard.json>...
+
+Merges the --json reports of zero-cost gate shards and prints one summary:
+every program that is not identical, the compile-time verdicts and the
+failures. --quiet prints only the summary and failures. --json writes the
+merged report. Exit status: 0 when the merged gate passes, 1 when it fails.`);
+  process.exit(values.help ? 0 : 2);
+}
+
+const reports = positionals.map((file) => JSON.parse(readFileSync(file, "utf8")));
+const { report, timingLines } = mergeReports(reports);
+if (!values.quiet)
+  for (const entry of report.programs)
+    if (entry.verdict !== "identical") for (const line of programLines(entry)) console.log(line);
+for (const line of timingLines) console.log(line);
+if (report.candidateSelf)
+  console.log(
+    report.candidateSelf.ok
+      ? `self       candidate compiler source builds with the candidate (${(report.candidateSelf.buildMs / 1000).toFixed(2)}s)`
+      : "self       candidate compiler source FAILS to build with the candidate",
+  );
+const cached = report.programs.filter((p) => p.cachedBaseline).length;
+console.log(
+  `zero-cost gate: ${report.verdict === "pass" ? "PASS" : "FAIL"} — ${Object.entries(report.counts)
+    .map(([k, v]) => `${v} ${k}`)
+    .join(", ")} over ${report.shards} shards (longest ${(report.elapsedMs / 1000).toFixed(2)}s` +
+    `${cached ? `, ${cached} baselines from the cache` : ""})`,
+);
+for (const failure of report.failures) console.log(`  ${failure}`);
+if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n");
+process.exit(report.verdict === "pass" ? 0 : 1);

tests/harness/sandbox-config.test.ts

@@ -5,6 +5,7 @@ import {
   sandboxBootstrapCommand,
   sandboxEffectWorkerAllocation,
   sandboxImageConfig,
+  sandboxNativeReference,
   sandboxRunnerConfig,
   sandboxTestSourceConfig,
   sandboxTestWorkerAllocation,
@@ -154,6 +155,24 @@ test("ambiguous or malformed snapshot sources fail before creating workers", ()
   );
 });
 
+test("a native reference checkout needs a prepared source and an absolute path", () => {
+  const snapshot = { SCRIPTC_SANDBOX_SNAPSHOT: "snap_ABC123" };
+  expect(sandboxNativeReference(snapshot)).toBeUndefined();
+  expect(
+    sandboxNativeReference({
+      ...snapshot,
+      SCRIPTC_SANDBOX_NATIVE_REFERENCE: " /vercel/sandbox/prev/ ",
+    }),
+  ).toBe("/vercel/sandbox/prev");
+  expect(() =>
+    sandboxNativeReference({ SCRIPTC_SANDBOX_NATIVE_REFERENCE: "/vercel/sandbox/prev" }),
+  ).toThrow("needs SCRIPTC_SANDBOX_SNAPSHOT");
+  for (const path of ["prev", "/a/../b", "/a b", "/a;rm"])
+    expect(() =>
+      sandboxNativeReference({ ...snapshot, SCRIPTC_SANDBOX_NATIVE_REFERENCE: path }),
+    ).toThrow("absolute path");
+});
+
 test("OIDC authentication wins and supplies its own Sandbox scope", () => {
   const config = sandboxVercelConfig({
     VERCEL_OIDC_TOKEN: "header.payload.signature",

tests/harness/zero-cost-gate.test.ts

@@ -1,14 +1,20 @@
 import { spawnSync } from "node:child_process";
 import { expect, test } from "vitest";
 import {
+  cachedComparison,
   cachegrindCounts,
   compareArtifact,
   compareCounts,
   compareExecutables,
   definedFunctionNames,
   dynamicTierSymbols,
+  mergeReports,
   nmOutput,
+  programInputsHash,
+  programLines,
   programVerdict,
+  recordedBaseline,
+  timingVerdict,
 } from "../../scripts/zero-cost-compare.mjs";
 
 const ir = (functions: object[], globals: object[]) =>
@@ -191,3 +197,105 @@ test("the zero-cost gate fails differing programs unless they match the signatur
   expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true });
   expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true });
 });
+
+test("a recorded baseline settles identical programs and shared build errors", () => {
+  const hashes = { ir: "a", llvm: "b", exe: "c" };
+  const entry = { ok: true, ms: 12, hashes, exeBytes: 100 };
+  expect(cachedComparison(entry, { ok: true }, { ...hashes })).toEqual({
+    baseline: { ok: true, ms: 12 },
+    artifacts: {
+      ir: { identical: true },
+      llvm: { identical: true },
+      exe: { identical: true, bytes: { baseline: 100, candidate: 100 } },
+    },
+  });
+  // Any other hash, a missing artifact or no record builds the baseline.
+  expect(cachedComparison(entry, { ok: true }, { ...hashes, exe: "d" })).toBeNull();
+  expect(
+    cachedComparison({ ...entry, hashes: { ...hashes, ir: null } }, { ok: true }, hashes),
+  ).toBeNull();
+  expect(cachedComparison(undefined, { ok: true }, hashes)).toBeNull();
+  expect(cachedComparison(entry, { ok: false, error: "SC1090" }, {})).toBeNull();
+  const failed = { ok: false, ms: 3, error: "SC1090" };
+  expect(cachedComparison(failed, { ok: false, error: "SC1090" }, {})).toEqual({
+    baseline: { ok: false, error: "SC1090", ms: 3, cached: true },
+    artifacts: {},
+  });
+  expect(cachedComparison(failed, { ok: false, error: "SC1091" }, {})).toBeNull();
+  expect(cachedComparison(failed, { ok: true }, hashes)).toBeNull();
+});
+
+test("a recorded baseline applies only to the program inputs it was recorded for", () => {
+  const files: Record<string, string> = {
+    "/p/tests/corpus/a/main.ts": 'import { f } from "./lib.ts";',
+    "/p/tests/corpus/a/lib.ts": "export const f = 1;",
+  };
+  const read = (file: string) => Buffer.from(files[file]!);
+  const paths = Object.keys(files);
+  const inputs = programInputsHash("/p", paths, "release", read);
+  expect(programInputsHash("/p", [...paths].reverse(), "release", read)).toBe(inputs);
+  expect(programInputsHash("/p", paths, "speed", read)).not.toBe(inputs);
+  expect(programInputsHash("/q", paths, "release", read)).not.toBe(inputs);
+  // An edit to an import changes the hash as an edit to the entry does.
+  files["/p/tests/corpus/a/lib.ts"] = "export const f = 2;";
+  const edited = programInputsHash("/p", paths, "release", read);
+  expect(edited).not.toBe(inputs);
+  const entry = { inputs, ok: false, ms: 3, error: "SC1090" };
+  const cache = { programs: { "corpus/a/main.ts": entry } };
+  expect(recordedBaseline(cache, "corpus/a/main.ts", inputs)).toBe(entry);
+  // A record of the old sources must not settle the new sources as both-fail or identical.
+  expect(recordedBaseline(cache, "corpus/a/main.ts", edited)).toBeUndefined();
+  expect(recordedBaseline(cache, "corpus/b/main.ts", inputs)).toBeUndefined();
+  expect(recordedBaseline(null, "corpus/a/main.ts", inputs)).toBeUndefined();
+});
+
+test("merged gate shards judge compile time and comparisons over every shard", () => {
+  const trees = { baseline: { revision: "a" }, candidate: { revision: "b" } };
+  const row = (id: string, ratio: number) => ({
+    id,
+    baselineMs: [100, 100, 100],
+    candidateMs: [100 * ratio, 100 * ratio, 100 * ratio],
+    ratios: [ratio, ratio, ratio],
+  });
+  const shard = (
+    programs: object[],
+    rows: object[],
+    failures: string[] = [],
+    globals: string[] = [],
+  ) => ({
+    schema: 1,
+    trees,
+    programs,
+    failures: [...failures, ...globals],
+    globalFailures: globals,
+    timing: { runs: 3, maxGrowth: 0.02, rows, groups: {} },
+    elapsedMs: 10,
+  });
+  const identical = (id: string) => ({ id, verdict: "identical", artifacts: {} });
+  const fast = shard([identical("bench/a")], [row("bench/a", 1.0)]);
+  // Alone, this shard's bench is slower; with the other shard it is not.
+  const slow = shard(
+    [
+      {
+        id: "corpus/x.ts",
+        verdict: "differs",
+        artifacts: { exe: { identical: false, first: "resized f" } },
+      },
+    ],
+    [row("bench/b", 1.03)],
+    ["corpus/x.ts: differs"],
+    ["bench compile time grew +3.00% (CI low +3.00%)"],
+  );
+  const { report, timingLines } = mergeReports([fast, slow]);
+  expect(report.counts).toEqual({ identical: 1, differs: 1 });
+  expect(report.failures).toEqual(["corpus/x.ts: differs"]);
+  expect(report.verdict).toBe("fail");
+  expect(timingLines).toEqual([timingVerdict("bench", report.timing.rows, 0.02).line]);
+  expect(timingVerdict("bench", report.timing.rows, 0.02).fails).toBe(false);
+  expect(programLines(report.programs[1])).toEqual(["DIFFERS    corpus/x.ts  exe: resized f"]);
+  expect(mergeReports([fast]).report.verdict).toBe("pass");
+  expect(mergeReports([shard([], [])]).report.failures).toEqual([
+    "no program built with both checkouts, so nothing was compared",
+  ]);
+  expect(() => mergeReports([fast, fast])).toThrow(/more than one gate report/);
+});