diff --git a/AGENTS.md b/AGENTS.md index 2350d7337..324ca87cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,7 +50,7 @@ Record a new or remeasured program's CI cost (used to balance the test shards) i Before opening a PR that adds or changes corpus, diagnostic, or frontend fixtures, run `pnpm test:ts7 --baselines-only`. This fast completeness check uses the same fixture discovery as the parity suite; it does not type-check or replace behavioral parity. Generate missing preflight/order baselines, review their diagnostics and module order, and run the affected `order-parity.test.ts` batches before pushing. Keep existing baseline entries unchanged unless the corresponding behavior intentionally changes. The full `pnpm test:ts7` command also checks completeness before starting its sweep. -A change that must not cost good code anything, such as a compatibility fix that replaces a refusal, proves it with the zero-cost gate. The gate compiles the benchmark workloads, the self-hosted compiler, and the TypeScript corpus with two built checkouts. It requires byte-identical IR, LLVM, and release executables, and compile time within tolerance. `--allow=` names the change's pattern signature: programs whose sources match may differ, and the PR lists them. Any other difference fails the gate, which names the first differing function. The gate also enforces the two-tier runtime separation: no executable may gain symbols from `DYNAMIC_TIER_PREFIXES` in `scripts/zero-cost-compare.mjs`, and fully typed benchmarks may link none; extend that list as dynamic-tier units land. `--quick` checks only the benchmarks; `--cachegrind` adds per-function instruction counts for executables that differ. Run it in a Sandbox with both checkouts prepared: +A change that must not cost good code anything, such as a compatibility fix that replaces a refusal, proves it with the zero-cost gate. The gate compiles the benchmark workloads, the self-hosted compiler, and the TypeScript corpus with two built checkouts. It requires byte-identical IR, LLVM, and release executables, and compile time within tolerance. `--allow=` names the change's pattern signature: programs whose sources match may differ, and the PR lists them. The gate classifies every other difference. A program is `runtime-only` when its IR and LLVM are identical and its executable differs only through runtime objects that differ between the two checkouts' runtime packs; the gate lists the changed runtime functions each such program links and the size change, and passes with a note, so a runtime C change does not fail the gate when no program's code changes. A program is `nondeterministic` when the baseline, building it again, differs from its own first build in the same artifacts; it also passes with a note. Any other difference is `differs` and fails the gate, which names the first differing function. `--strict` fails runtime-only and nondeterministic programs too, for a change that must leave every executable byte-identical. The gate also enforces the two-tier runtime separation: no executable may gain symbols from `DYNAMIC_TIER_PREFIXES` in `scripts/zero-cost-compare.mjs`, and fully typed benchmarks may link none; extend that list as dynamic-tier units land. `--quick` checks only the benchmarks; `--cachegrind` adds per-function instruction counts for executables that differ. Run it in a Sandbox with both checkouts prepared: ```bash node scripts/zero-cost-gate.mjs --baseline=/vercel/sandbox/prev --candidate=. --allow='\barguments\b' diff --git a/scripts/zero-cost-compare.mjs b/scripts/zero-cost-compare.mjs index df407bbb7..f8e4237fe 100644 --- a/scripts/zero-cost-compare.mjs +++ b/scripts/zero-cost-compare.mjs @@ -487,6 +487,78 @@ export function compareCounts(baseline, candidate) { }; } +/* ── runtime packs ─────────────────────────────────────────────────────── */ + +/** Every file a runtime-pack.json lists (each flavor's runtime unit + * variants, their bitcode, and the vendor archives), as path → sha256. */ +export function runtimePackFiles(manifest) { + const files = new Map(); + const visit = (value) => { + if (Array.isArray(value)) value.forEach(visit); + else if (value !== null && typeof value === "object") { + if (typeof value.path === "string" && typeof value.sha256 === "string") + files.set(value.path, value.sha256); + Object.values(value).forEach(visit); + } + }; + visit(manifest.flavors ?? {}); + visit(manifest.archives ?? {}); + return files; +} + +/** The runtime pack files that differ between two runtime-pack.json + * manifests: changed, added and removed paths, each sorted. */ +export function runtimePackDiff(baseline, candidate) { + const before = runtimePackFiles(baseline); + const after = runtimePackFiles(candidate); + const changed = [...before].filter(([path, sha]) => after.has(path) && after.get(path) !== sha); + return { + changed: changed.map(([path]) => path).sort(), + added: [...after.keys()].filter((path) => !before.has(path)).sort(), + removed: [...before.keys()].filter((path) => !after.has(path)).sort(), + }; +} + +/** Defined functions of textual LLVM IR (llvm-dis output) as name → hash of + * the body. Attribute group and metadata numbers are dropped first, since + * an unrelated change elsewhere in the unit renumbers them. */ +export function llvmFunctionHashes(text) { + const functions = new Map(); + const lines = text.split("\n"); + for (let index = 0; index < lines.length; index++) { + const head = /^define\b[^@]*@("(?:[^"\\]|\\.)*"|[\w.$-]+)\(/.exec(lines[index]); + if (head === null) continue; + const body = []; + for (; index < lines.length && lines[index] !== "}"; index++) body.push(lines[index]); + const normalized = body.join("\n").replace(/ #\d+/g, "").replace(/!\d+/g, "!"); + functions.set(head[1].replace(/^"|"$/g, ""), hash(normalized)); + } + return functions; +} + +/** Names whose hashes differ between two function maps, and the names only + * one side defines. */ +export function changedFunctions(before, after) { + return { + changed: [...before] + .filter(([name, h]) => after.has(name) && after.get(name) !== h) + .map(([name]) => name) + .sort(), + added: [...after.keys()].filter((name) => !before.has(name)).sort(), + removed: [...before.keys()].filter((name) => !after.has(name)).sort(), + }; +} + +/** Whether every symbol an executable comparison added, removed or resized + * is one of `runtimeSymbols`, the symbols the changed runtime objects + * define. Mach-O nm prints no sizes, so there only additions and removals + * are checked. */ +export function confinedToRuntime(exe, runtimeSymbols) { + return [...exe.added, ...exe.removed, ...exe.resized.map((s) => s.name)].every((name) => + runtimeSymbols.has(name), + ); +} + /* ── verdicts ──────────────────────────────────────────────────────────── */ /** One program's verdict from its build results and artifact comparisons. @@ -494,7 +566,21 @@ export function compareCounts(baseline, candidate) { /** `required` programs must build with both checkouts: a failure both share * means the gate compared nothing for them. Other programs may fail the same * way on both sides, as a corpus program the compiler refuses does. */ -export function programVerdict({ baseline, candidate, artifacts, allowed, required = false }) { +/** A difference in the release executable alone, with identical IR and + * LLVM, is "runtime-only" when `runtimeOnly` says the changed runtime + * objects explain it (the gate checks the runtime packs and the symbols). + * A difference the baseline also shows when it builds the program again is + * "nondeterministic". Neither fails the gate unless `strict` is set. */ +export function programVerdict({ + baseline, + candidate, + artifacts, + allowed, + required = false, + runtimeOnly = false, + nondeterministic = false, + strict = false, +}) { if (!baseline.ok || !candidate.ok) { if (!baseline.ok && !candidate.ok && baseline.error === candidate.error) return { verdict: "both-fail", fails: required }; @@ -502,7 +588,11 @@ export function programVerdict({ baseline, candidate, artifacts, allowed, requir } const differing = Object.entries(artifacts).filter(([, r]) => !r.identical); if (differing.length === 0) return { verdict: "identical", fails: false }; - return allowed ? { verdict: "allowed", fails: false } : { verdict: "differs", fails: true }; + if (allowed) return { verdict: "allowed", fails: false }; + if (runtimeOnly && differing.every(([kind]) => kind === "exe")) + return { verdict: "runtime-only", fails: strict }; + if (nondeterministic) return { verdict: "nondeterministic", fails: strict }; + return { verdict: "differs", fails: true }; } export function median(values) { diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index 99b1bc468..c756dd2ea 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -10,7 +10,11 @@ * The good-code set: the runtime benchmark workloads (and benchmarks/builds * applications), the self-hosted compiler (IR and LLVM), and the TypeScript * corpus programs. A program whose sources match --allow (the change's - * pattern signature) may differ; any other difference fails the gate. */ + * pattern signature) may differ. A program whose IR and LLVM are identical + * and whose executable differs only through changed runtime objects is + * "runtime-only", and a difference the baseline reproduces against itself is + * "nondeterministic": both pass with a note unless --strict. Any other + * difference fails the gate. */ import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; import { @@ -30,17 +34,23 @@ import { parseArgs } from "node:util"; import { readWorkloads } from "../benchmarks/runtime/workloads.mjs"; import { cachegrindCounts, + changedFunctions, compareArtifact, compareCounts, compareExecutables, + confinedToRuntime, DYNAMIC_WORKLOADS, definedFunctionNames, dynamicTierSymbols, + llvmFunctionHashes, median, nmOutput, + parseNm, programVerdict, + runtimePackDiff, timeRatio, } from "./zero-cost-compare.mjs"; +import { createHash } from "node:crypto"; const repoRoot = fileURLToPath(new URL("..", import.meta.url)); const { values } = parseArgs({ @@ -54,6 +64,7 @@ const { values } = parseArgs({ shard: { type: "string" }, only: { type: "string" }, allow: { type: "string" }, + strict: { type: "boolean", default: false }, "timing-runs": { type: "string" }, "self-timing-runs": { type: "string", default: "0" }, "max-time-growth": { type: "string", default: "2" }, @@ -69,7 +80,7 @@ if (values.help || values.baseline === undefined) { console.log(`Usage: node scripts/zero-cost-gate.mjs --baseline= [--candidate=] [--programs=baseline|candidate|] [--set=bench,self,corpus] [--quick] [--corpus=ts|all] [--shard=i/n] [--only=] [--allow=] - [--timing-runs=N] [--self-timing-runs=N] [--max-time-growth=2] + [--strict] [--timing-runs=N] [--self-timing-runs=N] [--max-time-growth=2] [--max-size-growth=0] [--cachegrind] [--jobs=N] [--json=] [--keep] @@ -84,8 +95,20 @@ JavaScript entries). --quick is --set=bench --timing-runs=0, for CI. --shard=i/n keeps every n-th corpus program; --only filters program ids. --allow= is the change's pattern signature: programs whose source -files match it may differ (reported as "allowed"). Any other difference in -IR, LLVM or the release executable fails the gate. +files match it may differ (reported as "allowed"). + +Other differences are classified: + runtime-only IR and LLVM identical; the executable differs only + through runtime objects that differ between the two + checkouts' runtime packs (every symbol the executable + gains, loses or resizes is defined by one of them), with + the same LLVM helper. Reported with the changed runtime + functions it links and its size change. + nondeterministic the baseline, building the program a second time, + differs from its own first build in the same artifacts. + differs anything else, in IR, LLVM or the executable. +"differs" fails the gate. runtime-only and nondeterministic pass with a +note; --strict fails them too, so any difference fails as before. --timing-runs=N (default 5) times N paired --emit=llvm builds per bench program, baseline and candidate side by side. The self-hosted compiler's @@ -260,10 +283,9 @@ function compile(tree, args) { * keeping the LLVM and IR beside the release executable. The self-hosted * compiler emits LLVM only: each of its frontend runs takes minutes, and its * LLVM names the first differing function as well as its IR would. */ -async function buildProgram(program, label) { +async function buildProgram(program, label, tree = trees[label]) { const dir = join(work, label, program.id.replace(/[^\w.-]+/g, "_")); mkdirSync(dir, { recursive: true }); - const tree = trees[label]; const failure = (result) => ({ ok: false, error: (result.stderr || result.stdout || `exit ${result.status ?? result.signal}`) @@ -400,6 +422,111 @@ async function instructionCounts(binary, args, out) { return cachegrindCounts(readFileSync(out, "utf8"), new Set(definedFunctions(binary))); } +/* ── runtime changes ───────────────────────────────────────────────────── */ + +const sha256 = (file) => createHash("sha256").update(readFileSync(file)).digest("hex"); + +/** llvm-dis from PATH or the pinned LLVM install, or null. */ +function llvmDis() { + for (const candidate of ["llvm-dis", "llvm-dis-22", "/usr/lib/llvm-22/bin/llvm-dis"]) + if (spawnSync(candidate, ["--version"]).status === 0) return candidate; + const config = spawnSync("llvm-config", ["--bindir"], { encoding: "utf8" }); + const fromConfig = config.status === 0 ? join(config.stdout.trim(), "llvm-dis") : null; + return fromConfig !== null && existsSync(fromConfig) ? fromConfig : null; +} + +/** What differs between the two checkouts' runtime packs for this host, or + * null when no object or archive does: the changed pack files, whether the + * LLVM helper is the same, the symbols the changed objects define (on either + * side), and the functions that changed in the bitcode of the changed + * release and speed objects. Bitcode alone is not a change: it records the + * checkout's source path, so it differs between any two checkouts. */ +function runtimeChanges() { + const pack = (root) => + readdirSync(join(root, "packages")) + .filter((name) => name.startsWith("runtime-")) + .filter((name) => existsSync(join(root, "packages", name, "runtime-pack.json"))); + const shared = pack(trees.candidate).filter((name) => pack(trees.baseline).includes(name)); + if (shared.length !== 1) return null; + const packDir = (root) => join(root, "packages", shared[0]); + const manifest = (root) => + JSON.parse(readFileSync(join(packDir(root), "runtime-pack.json"), "utf8")); + const linked = (paths) => paths.filter((path) => !path.endsWith(".bc")); + const all = runtimePackDiff(manifest(trees.baseline), manifest(trees.candidate)); + const diff = { + changed: linked(all.changed), + added: linked(all.added), + removed: linked(all.removed), + }; + if (diff.changed.length + diff.added.length + diff.removed.length === 0) return null; + const helper = (root) => { + const file = join( + root, + "packages", + shared[0].replace(/^runtime-/, "llvm-"), + "bin/scriptc-llvm-codegen", + ); + return existsSync(file) ? sha256(file) : null; + }; + const helperSame = + helper(trees.baseline) !== null && helper(trees.baseline) === helper(trees.candidate); + const symbols = new Set(); + const objects = [...diff.changed, ...diff.added, ...diff.removed].filter((path) => + path.endsWith(".o"), + ); + for (const path of objects) + for (const root of [trees.baseline, trees.candidate]) { + const file = join(packDir(root), path); + if (!existsSync(file)) continue; + const listed = spawnSync("nm", ["--defined-only", file], { + encoding: "utf8", + maxBuffer: 1 << 28, + }); + for (const symbol of parseNm(listed.stdout ?? "")) symbols.add(symbol.name); + } + // Changed functions, by unit, from the bitcode of the release and speed + // flavors (the ones the gate builds with). + const dis = llvmDis(); + const functions = { changed: new Set(), added: new Set(), removed: new Set(), units: new Set() }; + for (const object of objects) + functions.units.add(object.replace(/^.*\//, "").replace(/\.o$/, ".c")); + const bitcode = objects + .filter((path) => /^artifacts\/(release|speed)\//.test(path)) + .map((path) => path.replace(/\.o$/, ".bc")); + for (const path of bitcode) { + if (dis === null) continue; + const text = (root) => { + const file = join(packDir(root), path); + if (!existsSync(file)) return new Map(); + const out = spawnSync(dis, [file, "-o", "-"], { encoding: "utf8", maxBuffer: 1 << 30 }); + return llvmFunctionHashes(out.stdout ?? ""); + }; + const delta = changedFunctions(text(trees.baseline), text(trees.candidate)); + for (const kind of ["changed", "added", "removed"]) + for (const name of delta[kind]) functions[kind].add(name); + } + return { + pack: shared[0], + diff, + helperSame, + symbols, + disassembler: dis, + functions: Object.fromEntries( + Object.entries(functions).map(([kind, names]) => [kind, [...names].sort()]), + ), + }; +} + +/** The changed runtime functions an executable defines. */ +function linkedRuntimeChanges(runtime, binary) { + const defined = new Set(definedFunctions(binary)); + const pick = (names) => names.filter((name) => defined.has(name)); + return { + changed: pick(runtime.functions.changed), + added: pick(runtime.functions.added), + }; +} + /* ── main ──────────────────────────────────────────────────────────────── */ function revision(root) { @@ -416,6 +543,7 @@ const pct = (ratio) => `${ratio >= 1 ? "+" : ""}${((ratio - 1) * 100).toFixed(2) const programs = goodCodeSet(); assert.ok(programs.length > 0, "the selected good-code set is empty"); +const runtime = runtimeChanges(); const report = { schema: 1, date: new Date().toISOString(), @@ -425,6 +553,16 @@ const report = { ), programsRoot, allow: values.allow ?? null, + strict: values.strict, + runtime: + runtime === null + ? null + : { + pack: runtime.pack, + ...runtime.diff, + helperSame: runtime.helperSame, + functions: runtime.functions, + }, programs: [], timing: null, failures: [], @@ -433,6 +571,24 @@ console.log( `zero-cost gate: ${programs.length} programs (${[...sets].join(", ")}), ` + `baseline ${trees.baseline}, candidate ${trees.candidate}, programs from ${programsRoot}`, ); +if (runtime !== null) { + const { changed, added, removed } = runtime.diff; + console.log( + `runtime pack ${runtime.pack} differs: ${changed.length} changed, ${added.length} added, ` + + `${removed.length} removed files; units ${runtime.functions.units.join(", ") || "none"}` + + (runtime.helperSame ? "" : "; the LLVM helper differs too, so nothing is runtime-only"), + ); + if (runtime.disassembler === null) + console.log("runtime functions: llvm-dis not found, so changed functions are not listed"); + else + console.log( + `runtime functions: ${runtime.functions.changed.length} changed` + + (runtime.functions.changed.length + ? ` (${runtime.functions.changed.slice(0, 20).join(", ")})` + : "") + + `, ${runtime.functions.added.length} added, ${runtime.functions.removed.length} removed`, + ); +} let failed = false; const started = Date.now(); @@ -450,13 +606,39 @@ await pool( const artifacts = baseline.ok && candidate.ok ? compareBuilds(baseline, candidate) : {}; // The benchmark workloads and the self-hosted compiler always build. const required = !program.id.startsWith("corpus/"); - const { verdict, fails } = programVerdict({ - baseline, - candidate, - artifacts, - allowed, - required, - }); + const runtimeOnly = + runtime !== null && + runtime.helperSame && + artifacts.exe !== undefined && + !artifacts.exe.identical && + confinedToRuntime(artifacts.exe, runtime.symbols); + const judge = (nondeterministic) => + programVerdict({ + baseline, + candidate, + artifacts, + allowed, + required, + runtimeOnly, + nondeterministic, + strict: values.strict, + }); + let { verdict, fails } = judge(false); + // Build a difference the change cannot explain once more with the + // baseline: when the baseline's two builds differ in the same + // artifacts, the difference is the build's, not the change's. + let recheck = null; + if (verdict === "differs") { + const again = await buildProgram(program, "recheck", trees.baseline); + if (again.ok) { + const self = compareBuilds(baseline, again); + recheck = Object.keys(artifacts).filter( + (kind) => !artifacts[kind].identical && !self[kind]?.identical, + ); + if (again.dir && !values.keep) rmSync(again.dir, { recursive: true, force: true }); + } + if (recheck?.length) ({ verdict, fails } = judge(true)); + } const entry = { id: program.id, verdict, @@ -464,6 +646,8 @@ await pool( buildMs: { baseline: baseline.ms ?? null, candidate: candidate.ms ?? null }, artifacts, }; + if (verdict === "runtime-only") entry.runtime = linkedRuntimeChanges(runtime, candidate.exe); + if (verdict === "nondeterministic") entry.nondeterministic = recheck; if (!baseline.ok) entry.baselineError = baseline.error; if (!candidate.ok) entry.candidateError = candidate.error; const size = artifacts.exe?.bytes; @@ -517,6 +701,8 @@ function printProgram(entry) { identical: "identical", allowed: "allowed ", differs: "DIFFERS ", + "runtime-only": "runtime ", + nondeterministic: "nondet ", "build-error": "ERROR ", "both-fail": "both-fail", }[entry.verdict]; @@ -543,6 +729,17 @@ function printProgram(entry) { ); } else if (entry.cachegrind?.skipped) notes.push(`cachegrind skipped: ${entry.cachegrind.skipped}`); + if (entry.runtime) { + const linked = [...entry.runtime.changed, ...entry.runtime.added]; + notes.push( + `runtime-only: links ${linked.length} changed runtime function${linked.length === 1 ? "" : "s"}` + + (linked.length + ? ` (${linked.slice(0, 8).join(", ")}${linked.length > 8 ? ", ..." : ""})` + : ""), + ); + } + if (entry.nondeterministic) + notes.push(`the baseline's own rebuild differs in ${entry.nondeterministic.join(", ")}`); for (const failure of entry.dynamicTier?.failures ?? []) notes.push(`SYMBOLS: ${failure}`); console.log(`${label} ${entry.id}${notes.length ? ` ${notes.join("; ")}` : ""}`); // A build error's first line is often a warning; print all of it. @@ -631,7 +828,8 @@ if (timingRuns > 0) { const counts = {}; for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1; -if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) { +const compared = ["identical", "allowed", "runtime-only", "nondeterministic", "differs"]; +if (!report.programs.some((p) => compared.includes(p.verdict))) { failed = true; report.failures.push("no program built with both checkouts, so nothing was compared"); } @@ -644,10 +842,37 @@ if (values.json) { } if (!values.keep) rmSync(work, { recursive: true, force: true }); else console.log(`artifacts kept in ${work}`); +const noted = report.programs.filter((p) => + ["runtime-only", "nondeterministic"].includes(p.verdict), +); console.log( - `zero-cost gate: ${failed ? "FAIL" : "PASS"} — ${Object.entries(counts) + `zero-cost gate: ${failed ? "FAIL" : noted.length ? "PASS with notes" : "PASS"} — ${Object.entries( + counts, + ) .map(([k, v]) => `${v} ${k}`) .join(", ")} in ${fmtMs(report.elapsedMs)}`, ); +const runtimeOnlyPrograms = report.programs.filter((p) => p.verdict === "runtime-only"); +if (runtimeOnlyPrograms.length > 0) { + const deltas = runtimeOnlyPrograms.map( + (p) => p.artifacts.exe.bytes.candidate - p.artifacts.exe.bytes.baseline, + ); + const signed = (n) => `${n > 0 ? "+" : ""}${n}`; + const linked = new Set( + runtimeOnlyPrograms.flatMap((p) => [...p.runtime.changed, ...p.runtime.added]), + ); + console.log( + ` note: ${runtimeOnlyPrograms.length} runtime-only programs (IR and LLVM identical); ` + + `executable size ${signed(Math.min(...deltas))} to ${signed(Math.max(...deltas))} bytes; ` + + `changed runtime functions linked: ${[...linked].sort().join(", ") || "none listed"}` + + (values.strict ? "" : " (--strict fails these)"), + ); +} +const nondeterministicPrograms = report.programs.filter((p) => p.verdict === "nondeterministic"); +if (nondeterministicPrograms.length > 0) + console.log( + ` note: ${nondeterministicPrograms.length} nondeterministic programs, whose baseline rebuild differs too: ` + + nondeterministicPrograms.map((p) => p.id).join(", "), + ); for (const failure of report.failures) console.log(` ${failure}`); process.exit(failed ? 1 : 0); diff --git a/tests/harness/zero-cost-gate.test.ts b/tests/harness/zero-cost-gate.test.ts index 565f8ba71..734757c28 100644 --- a/tests/harness/zero-cost-gate.test.ts +++ b/tests/harness/zero-cost-gate.test.ts @@ -2,13 +2,17 @@ import { spawnSync } from "node:child_process"; import { expect, test } from "vitest"; import { cachegrindCounts, + changedFunctions, compareArtifact, compareCounts, compareExecutables, + confinedToRuntime, definedFunctionNames, dynamicTierSymbols, + llvmFunctionHashes, nmOutput, programVerdict, + runtimePackDiff, } from "../../scripts/zero-cost-compare.mjs"; const ir = (functions: object[], globals: object[]) => @@ -191,3 +195,92 @@ test("the zero-cost gate fails differing programs unless they match the signatur expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true }); expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true }); }); + +test("the zero-cost gate classifies runtime-only and nondeterministic differences", () => { + const ok = { ok: true }; + const exeOnly = { ir: { identical: true }, llvm: { identical: true }, exe: { identical: false } }; + const llvmToo = { + ir: { identical: true }, + llvm: { identical: false }, + exe: { identical: false }, + }; + const verdict = (artifacts: object, flags: object) => + programVerdict({ baseline: ok, candidate: ok, artifacts, allowed: false, ...flags }); + expect(verdict(exeOnly, { runtimeOnly: true })).toEqual({ + verdict: "runtime-only", + fails: false, + }); + expect(verdict(exeOnly, { runtimeOnly: true, strict: true })).toEqual({ + verdict: "runtime-only", + fails: true, + }); + // A changed runtime cannot explain different LLVM. + expect(verdict(llvmToo, { runtimeOnly: true })).toEqual({ verdict: "differs", fails: true }); + expect(verdict(llvmToo, { nondeterministic: true })).toEqual({ + verdict: "nondeterministic", + fails: false, + }); + expect(verdict(llvmToo, { nondeterministic: true, strict: true })).toEqual({ + verdict: "nondeterministic", + fails: true, + }); +}); + +test("the zero-cost gate names the runtime pack files and functions that changed", () => { + const unit = (sha: string, bc: string) => ({ + path: "artifacts/release/runtime/default/scr_a.o", + sha256: sha, + bitcode: { path: "artifacts/release/runtime/default/scr_a.bc", sha256: bc }, + }); + const pack = (a: object, extra: object[] = []) => ({ + flavors: { release: { runtime_units: [{ source: "scr_a.c", variants: [a, ...extra] }] } }, + archives: [{ path: "artifacts/vendor/libz.a", sha256: "z" }], + }); + const other = { path: "artifacts/release/runtime/default/scr_b.o", sha256: "b" }; + expect(runtimePackDiff(pack(unit("1", "2")), pack(unit("1", "2")))).toEqual({ + changed: [], + added: [], + removed: [], + }); + expect(runtimePackDiff(pack(unit("1", "2")), pack(unit("3", "4"), [other]))).toEqual({ + changed: [ + "artifacts/release/runtime/default/scr_a.bc", + "artifacts/release/runtime/default/scr_a.o", + ], + added: ["artifacts/release/runtime/default/scr_b.o"], + removed: [], + }); + + const ll = (body: string, attrs: number) => + [ + `define i32 @scr_same(i32 %x) #${attrs} {`, + " ret i32 %x, !dbg !" + attrs, + "}", + "define internal void @scr_changed() {", + ` ${body}`, + "}", + "", + ].join("\n"); + const before = llvmFunctionHashes(ll("ret void", 0)); + const after = llvmFunctionHashes( + ll("call void @f()\n ret void", 7) + 'define void @"scr new"() {\n}\n', + ); + // Renumbered attribute groups and metadata are not changes. + expect(changedFunctions(before, after)).toEqual({ + changed: ["scr_changed"], + added: ["scr new"], + removed: [], + }); +}); + +test("the zero-cost gate confines runtime-only differences to runtime symbols", () => { + const exe = (added: string[], removed: string[], resized: string[]) => ({ + added, + removed, + resized: resized.map((name) => ({ name, baseline: 1, candidate: 2 })), + }); + const runtime = new Set(["scr_a", "scr_b"]); + expect(confinedToRuntime(exe([], [], ["scr_a"]), runtime)).toBe(true); + expect(confinedToRuntime(exe(["scr_b"], ["scr_a"], []), runtime)).toBe(true); + expect(confinedToRuntime(exe([], [], ["main.$1"]), runtime)).toBe(false); +});