mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-01 07:33:12 +08:00
Chris Tate
08fbecb7e7
A program is nondeterministic only when every artifact that differs also differs between two baseline builds; a deterministic difference in another artifact still fails the gate. The runtime-only check now reads the symbols of changed vendor archives as well as runtime objects.
428 lines
16 KiB
TypeScript
428 lines
16 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { expect, test } from "vitest";
|
|
import {
|
|
cachedComparison,
|
|
cachegrindCounts,
|
|
changedFunctions,
|
|
compareArtifact,
|
|
compareCounts,
|
|
compareExecutables,
|
|
confinedToRuntime,
|
|
definedFunctionNames,
|
|
dynamicTierSymbols,
|
|
linkedRuntimeFiles,
|
|
llvmFunctionHashes,
|
|
mergeReports,
|
|
nmOutput,
|
|
programInputsHash,
|
|
programLines,
|
|
programVerdict,
|
|
recordedBaseline,
|
|
runtimePackDiff,
|
|
timingVerdict,
|
|
} from "../../scripts/zero-cost-compare.mjs";
|
|
|
|
const ir = (functions: object[], globals: object[]) =>
|
|
Buffer.from(JSON.stringify({ irVersion: 16, functions, globals, sourceFile: "/p/main.ts" }));
|
|
|
|
const llvm = (body: string, extra = "") =>
|
|
Buffer.from(
|
|
[
|
|
"; Generated by scriptc (LLVM backend) from /p/main.ts. Do not edit.",
|
|
'@.str = private constant [3 x i8] c"hi\\00"',
|
|
"define internal i64 @add(i64 %a, i64 %b) {",
|
|
` ${body}`,
|
|
"}",
|
|
'define void @"main.$1"() { ret void }',
|
|
extra,
|
|
"declare void @scr_print(ptr)",
|
|
"",
|
|
].join("\n"),
|
|
);
|
|
|
|
test("the zero-cost gate names the first differing IR and LLVM unit", () => {
|
|
const f = { name: "f", params: [], body: [{ kind: "return" }] };
|
|
const g = { name: "g", params: [], body: [] };
|
|
const base = ir([f, g, f], [{ name: "a", type: "i64" }]);
|
|
expect(compareArtifact("ir", base, Buffer.from(base))).toEqual({ identical: true });
|
|
|
|
const changed = compareArtifact(
|
|
"ir",
|
|
base,
|
|
ir([f, { ...g, params: [1] }, f], [{ name: "a", type: "i64" }]),
|
|
);
|
|
expect(changed).toMatchObject({
|
|
identical: false,
|
|
first: "changed g",
|
|
changed: ["g"],
|
|
added: [],
|
|
module: null,
|
|
});
|
|
|
|
const header = compareArtifact("ir", base, ir([f, g, f], [{ name: "a", type: "f64" }]));
|
|
expect(header).toMatchObject({ first: "module globals[0:a]", changed: [] });
|
|
// Repeated names are matched by occurrence, so an insertion shows as added.
|
|
expect(compareArtifact("ir", base, ir([f, g, f, f], [{ name: "a", type: "i64" }]))).toMatchObject(
|
|
{
|
|
first: "added f#3",
|
|
},
|
|
);
|
|
|
|
const ll = llvm("%r = add i64 %a, %b\n ret i64 %r");
|
|
expect(compareArtifact("llvm", ll, Buffer.from(ll))).toEqual({ identical: true });
|
|
expect(compareArtifact("llvm", ll, llvm("%r = sub i64 %a, %b\n ret i64 %r"))).toMatchObject({
|
|
first: "changed add",
|
|
changed: ["add"],
|
|
functions: { baseline: 2, candidate: 2 },
|
|
});
|
|
expect(
|
|
compareArtifact(
|
|
"llvm",
|
|
ll,
|
|
llvm("%r = add i64 %a, %b\n ret i64 %r", "define void @new() { ret void }"),
|
|
),
|
|
).toMatchObject({ first: "added new", added: ["new"] });
|
|
const global = compareArtifact("llvm", ll, Buffer.from(ll.toString().replace("hi", "ho")));
|
|
expect(global).toMatchObject({ first: "module line 2", changed: [] });
|
|
expect(global.module.candidate).toContain("ho");
|
|
});
|
|
|
|
test("the zero-cost gate compares executables and instruction counts per function", () => {
|
|
const nm =
|
|
"0000000000001000 0000000000000010 T main\n0000000000001010 0000000000000020 t scr_add\n";
|
|
const resized = compareExecutables(
|
|
Buffer.from("abcd"),
|
|
Buffer.from("abXd"),
|
|
nm,
|
|
nm.replace("0000000000000020 t", "0000000000000030 t"),
|
|
);
|
|
expect(resized).toMatchObject({ identical: false, first: "resized scr_add", offset: 2 });
|
|
|
|
const output = (main: number, prefault: number, loader: number) =>
|
|
[
|
|
"events: Ir",
|
|
"fl=???",
|
|
"fn=main",
|
|
`0 ${main}`,
|
|
"fn=scr_sa_prefault",
|
|
`0 ${prefault}`,
|
|
"fn=_dl_lookup_symbol_x",
|
|
`0 ${loader}`,
|
|
"fn=scr_add",
|
|
"0 7",
|
|
"0 3",
|
|
].join("\n");
|
|
const own = new Set(["main", "scr_add", "scr_sa_prefault"]);
|
|
const base = cachegrindCounts(output(100, 50, 9000), own);
|
|
expect([...base]).toEqual([
|
|
["main", 100],
|
|
["scr_add", 10],
|
|
]);
|
|
expect(compareCounts(base, cachegrindCounts(output(100, 75, 9300), own)).identical).toBe(true);
|
|
expect(compareCounts(base, cachegrindCounts(output(104, 50, 9000), own))).toMatchObject({
|
|
identical: false,
|
|
total: { baseline: 110, candidate: 114 },
|
|
deltas: [{ name: "main", delta: 4 }],
|
|
});
|
|
});
|
|
|
|
test("the zero-cost gate throws when nm does not list a binary's symbols", () => {
|
|
const ok = { status: 0, signal: null, stdout: "0000000000001000 T main\n", stderr: "" };
|
|
expect(nmOutput("prog", ok)).toBe(ok.stdout);
|
|
const missing = spawnSync("scriptc-missing-nm", ["prog"], { encoding: "utf8" });
|
|
expect(() => nmOutput("prog", missing)).toThrow(/^nm prog: .*ENOENT/);
|
|
expect(() =>
|
|
nmOutput("prog", { status: null, signal: "SIGKILL", stdout: "", stderr: "" }),
|
|
).toThrow("nm prog: killed by SIGKILL");
|
|
expect(() =>
|
|
nmOutput("prog", { status: 1, signal: null, stdout: "", stderr: "nm: prog: no symbols\n" }),
|
|
).toThrow("nm prog: exited with status 1: nm: prog: no symbols");
|
|
});
|
|
|
|
test("the zero-cost gate recognizes dynamic-tier runtime symbols", () => {
|
|
expect(
|
|
dynamicTierSymbols([
|
|
"main",
|
|
"scr_dyn_release",
|
|
"scr_dyn_obj_drop_keys",
|
|
"scr_dyn_this_push_dyn",
|
|
"scr_js_load_ic",
|
|
"scr_dyn_obj_read",
|
|
"scr_dyn_obj_read",
|
|
]),
|
|
).toEqual(["scr_dyn_obj_read", "scr_js_load_ic"]);
|
|
});
|
|
|
|
test("the zero-cost gate reads function names from ELF and Mach-O nm output", () => {
|
|
const elf = [
|
|
"0000000000001000 0000000000000010 T scr_dyn_call",
|
|
"0000000000001010 0000000000000008 t scr_dyn_obj_get",
|
|
"0000000000002000 0000000000000004 D scr_dyn_table",
|
|
" U memcpy",
|
|
].join("\n");
|
|
expect(definedFunctionNames(elf, "linux")).toEqual(["scr_dyn_call", "scr_dyn_obj_get"]);
|
|
// Mach-O names carry a leading underscore, which would hide every
|
|
// dynamic-tier symbol from the prefix match.
|
|
const macho = [
|
|
"0000000100001000 T _scr_dyn_call",
|
|
"0000000100001010 t _scr_dyn_obj_get",
|
|
"0000000100002000 W __main",
|
|
"0000000100003000 s _scr_dyn_add.message",
|
|
" U _memcpy",
|
|
].join("\n");
|
|
expect(definedFunctionNames(macho, "darwin")).toEqual([
|
|
"scr_dyn_call",
|
|
"scr_dyn_obj_get",
|
|
"_main",
|
|
]);
|
|
expect(dynamicTierSymbols(definedFunctionNames(macho, "darwin"))).toEqual([
|
|
"scr_dyn_call",
|
|
"scr_dyn_obj_get",
|
|
]);
|
|
});
|
|
|
|
test("the zero-cost gate fails differing programs unless they match the signature", () => {
|
|
const ok = { ok: true };
|
|
const same = { ir: { identical: true }, exe: { identical: true } };
|
|
const differs = { ir: { identical: true }, exe: { identical: false } };
|
|
const verdict = (
|
|
artifacts: object,
|
|
allowed = false,
|
|
baseline: object = ok,
|
|
candidate: object = ok,
|
|
required = false,
|
|
) => programVerdict({ baseline, candidate, artifacts, allowed, required });
|
|
expect(verdict(same)).toEqual({ verdict: "identical", fails: false });
|
|
expect(verdict(differs)).toEqual({ verdict: "differs", fails: true });
|
|
expect(verdict(differs, true)).toEqual({ verdict: "allowed", fails: false });
|
|
const failed = { ok: false, error: "SC1090" };
|
|
expect(verdict({}, false, failed, failed)).toEqual({ verdict: "both-fail", fails: false });
|
|
// A benchmark or the self-hosted compiler that neither checkout builds
|
|
// was never compared.
|
|
expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true });
|
|
expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true });
|
|
});
|
|
|
|
test("the zero-cost gate classifies runtime-only and nondeterministic differences", () => {
|
|
const ok = { ok: true };
|
|
const exeOnly = { ir: { identical: true }, llvm: { identical: true }, exe: { identical: false } };
|
|
const llvmToo = {
|
|
ir: { identical: true },
|
|
llvm: { identical: false },
|
|
exe: { identical: false },
|
|
};
|
|
const verdict = (artifacts: object, flags: object) =>
|
|
programVerdict({ baseline: ok, candidate: ok, artifacts, allowed: false, ...flags });
|
|
expect(verdict(exeOnly, { runtimeOnly: true })).toEqual({
|
|
verdict: "runtime-only",
|
|
fails: false,
|
|
});
|
|
expect(verdict(exeOnly, { runtimeOnly: true, strict: true })).toEqual({
|
|
verdict: "runtime-only",
|
|
fails: true,
|
|
});
|
|
// A changed runtime cannot explain different LLVM.
|
|
expect(verdict(llvmToo, { runtimeOnly: true })).toEqual({ verdict: "differs", fails: true });
|
|
expect(verdict(llvmToo, { nondeterministic: ["llvm", "exe"] })).toEqual({
|
|
verdict: "nondeterministic",
|
|
fails: false,
|
|
});
|
|
expect(verdict(llvmToo, { nondeterministic: ["llvm", "exe"], strict: true })).toEqual({
|
|
verdict: "nondeterministic",
|
|
fails: true,
|
|
});
|
|
// A nondeterministic executable does not excuse a deterministic LLVM difference.
|
|
expect(verdict(llvmToo, { nondeterministic: ["exe"] })).toEqual({
|
|
verdict: "differs",
|
|
fails: true,
|
|
});
|
|
});
|
|
|
|
test("the zero-cost gate reads the symbols of changed runtime objects and vendor archives", () => {
|
|
expect(
|
|
linkedRuntimeFiles({
|
|
changed: [
|
|
"artifacts/release/runtime/default/scr_a.o",
|
|
"artifacts/release/runtime/default/scr_a.bc",
|
|
],
|
|
added: ["artifacts/vendor/libz/libscriptc-zlib.a"],
|
|
removed: [],
|
|
}),
|
|
).toEqual([
|
|
"artifacts/release/runtime/default/scr_a.o",
|
|
"artifacts/vendor/libz/libscriptc-zlib.a",
|
|
]);
|
|
});
|
|
|
|
test("the zero-cost gate names the runtime pack files and functions that changed", () => {
|
|
const unit = (sha: string, bc: string) => ({
|
|
path: "artifacts/release/runtime/default/scr_a.o",
|
|
sha256: sha,
|
|
bitcode: { path: "artifacts/release/runtime/default/scr_a.bc", sha256: bc },
|
|
});
|
|
const pack = (a: object, extra: object[] = []) => ({
|
|
flavors: { release: { runtime_units: [{ source: "scr_a.c", variants: [a, ...extra] }] } },
|
|
archives: [{ path: "artifacts/vendor/libz.a", sha256: "z" }],
|
|
});
|
|
const other = { path: "artifacts/release/runtime/default/scr_b.o", sha256: "b" };
|
|
expect(runtimePackDiff(pack(unit("1", "2")), pack(unit("1", "2")))).toEqual({
|
|
changed: [],
|
|
added: [],
|
|
removed: [],
|
|
});
|
|
expect(runtimePackDiff(pack(unit("1", "2")), pack(unit("3", "4"), [other]))).toEqual({
|
|
changed: [
|
|
"artifacts/release/runtime/default/scr_a.bc",
|
|
"artifacts/release/runtime/default/scr_a.o",
|
|
],
|
|
added: ["artifacts/release/runtime/default/scr_b.o"],
|
|
removed: [],
|
|
});
|
|
|
|
const ll = (body: string, attrs: number) =>
|
|
[
|
|
`define i32 @scr_same(i32 %x) #${attrs} {`,
|
|
" ret i32 %x, !dbg !" + attrs,
|
|
"}",
|
|
"define internal void @scr_changed() {",
|
|
` ${body}`,
|
|
"}",
|
|
"",
|
|
].join("\n");
|
|
const before = llvmFunctionHashes(ll("ret void", 0));
|
|
const after = llvmFunctionHashes(
|
|
ll("call void @f()\n ret void", 7) + 'define void @"scr new"() {\n}\n',
|
|
);
|
|
// Renumbered attribute groups and metadata are not changes.
|
|
expect(changedFunctions(before, after)).toEqual({
|
|
changed: ["scr_changed"],
|
|
added: ["scr new"],
|
|
removed: [],
|
|
});
|
|
});
|
|
|
|
test("the zero-cost gate confines runtime-only differences to runtime symbols", () => {
|
|
const exe = (added: string[], removed: string[], resized: string[]) => ({
|
|
added,
|
|
removed,
|
|
resized: resized.map((name) => ({ name, baseline: 1, candidate: 2 })),
|
|
});
|
|
const runtime = new Set(["scr_a", "scr_b"]);
|
|
expect(confinedToRuntime(exe([], [], ["scr_a"]), runtime)).toBe(true);
|
|
expect(confinedToRuntime(exe(["scr_b"], ["scr_a"], []), runtime)).toBe(true);
|
|
expect(confinedToRuntime(exe([], [], ["main.$1"]), runtime)).toBe(false);
|
|
});
|
|
|
|
test("a recorded baseline settles identical programs and shared build errors", () => {
|
|
const hashes = { ir: "a", llvm: "b", exe: "c" };
|
|
const entry = { ok: true, ms: 12, hashes, exeBytes: 100 };
|
|
expect(cachedComparison(entry, { ok: true }, { ...hashes })).toEqual({
|
|
baseline: { ok: true, ms: 12 },
|
|
artifacts: {
|
|
ir: { identical: true },
|
|
llvm: { identical: true },
|
|
exe: { identical: true, bytes: { baseline: 100, candidate: 100 } },
|
|
},
|
|
});
|
|
// Any other hash, a missing artifact or no record builds the baseline.
|
|
expect(cachedComparison(entry, { ok: true }, { ...hashes, exe: "d" })).toBeNull();
|
|
expect(
|
|
cachedComparison({ ...entry, hashes: { ...hashes, ir: null } }, { ok: true }, hashes),
|
|
).toBeNull();
|
|
expect(cachedComparison(undefined, { ok: true }, hashes)).toBeNull();
|
|
expect(cachedComparison(entry, { ok: false, error: "SC1090" }, {})).toBeNull();
|
|
const failed = { ok: false, ms: 3, error: "SC1090" };
|
|
expect(cachedComparison(failed, { ok: false, error: "SC1090" }, {})).toEqual({
|
|
baseline: { ok: false, error: "SC1090", ms: 3, cached: true },
|
|
artifacts: {},
|
|
});
|
|
expect(cachedComparison(failed, { ok: false, error: "SC1091" }, {})).toBeNull();
|
|
expect(cachedComparison(failed, { ok: true }, hashes)).toBeNull();
|
|
});
|
|
|
|
test("a recorded baseline applies only to the program inputs it was recorded for", () => {
|
|
const files: Record<string, string> = {
|
|
"/p/tests/corpus/a/main.ts": 'import { f } from "./lib.ts";',
|
|
"/p/tests/corpus/a/lib.ts": "export const f = 1;",
|
|
};
|
|
const read = (file: string) => Buffer.from(files[file]!);
|
|
const paths = Object.keys(files);
|
|
const inputs = programInputsHash("/p", paths, "release", read);
|
|
expect(programInputsHash("/p", [...paths].reverse(), "release", read)).toBe(inputs);
|
|
expect(programInputsHash("/p", paths, "speed", read)).not.toBe(inputs);
|
|
expect(programInputsHash("/q", paths, "release", read)).not.toBe(inputs);
|
|
// An edit to an import changes the hash as an edit to the entry does.
|
|
files["/p/tests/corpus/a/lib.ts"] = "export const f = 2;";
|
|
const edited = programInputsHash("/p", paths, "release", read);
|
|
expect(edited).not.toBe(inputs);
|
|
const entry = { inputs, ok: false, ms: 3, error: "SC1090" };
|
|
const cache = { programs: { "corpus/a/main.ts": entry } };
|
|
expect(recordedBaseline(cache, "corpus/a/main.ts", inputs)).toBe(entry);
|
|
// A record of the old sources must not settle the new sources as both-fail or identical.
|
|
expect(recordedBaseline(cache, "corpus/a/main.ts", edited)).toBeUndefined();
|
|
expect(recordedBaseline(cache, "corpus/b/main.ts", inputs)).toBeUndefined();
|
|
expect(recordedBaseline(null, "corpus/a/main.ts", inputs)).toBeUndefined();
|
|
});
|
|
|
|
test("merged gate shards judge compile time and comparisons over every shard", () => {
|
|
const trees = { baseline: { revision: "a" }, candidate: { revision: "b" } };
|
|
const row = (id: string, ratio: number) => ({
|
|
id,
|
|
baselineMs: [100, 100, 100],
|
|
candidateMs: [100 * ratio, 100 * ratio, 100 * ratio],
|
|
ratios: [ratio, ratio, ratio],
|
|
});
|
|
const shard = (
|
|
programs: object[],
|
|
rows: object[],
|
|
failures: string[] = [],
|
|
globals: string[] = [],
|
|
) => ({
|
|
schema: 1,
|
|
trees,
|
|
programs,
|
|
failures: [...failures, ...globals],
|
|
globalFailures: globals,
|
|
timing: { runs: 3, maxGrowth: 0.02, rows, groups: {} },
|
|
elapsedMs: 10,
|
|
});
|
|
const identical = (id: string) => ({ id, verdict: "identical", artifacts: {} });
|
|
const fast = shard([identical("bench/a")], [row("bench/a", 1.0)]);
|
|
// Alone, this shard's bench is slower; with the other shard it is not.
|
|
const slow = shard(
|
|
[
|
|
{
|
|
id: "corpus/x.ts",
|
|
verdict: "differs",
|
|
artifacts: { exe: { identical: false, first: "resized f" } },
|
|
},
|
|
],
|
|
[row("bench/b", 1.03)],
|
|
["corpus/x.ts: differs"],
|
|
["bench compile time grew +3.00% (CI low +3.00%)"],
|
|
);
|
|
const { report, timingLines } = mergeReports([fast, slow]);
|
|
expect(report.counts).toEqual({ identical: 1, differs: 1 });
|
|
expect(report.failures).toEqual(["corpus/x.ts: differs"]);
|
|
expect(report.verdict).toBe("fail");
|
|
expect(timingLines).toEqual([timingVerdict("bench", report.timing.rows, 0.02).line]);
|
|
expect(timingVerdict("bench", report.timing.rows, 0.02).fails).toBe(false);
|
|
expect(programLines(report.programs[1])).toEqual(["DIFFERS corpus/x.ts exe: resized f"]);
|
|
expect(mergeReports([fast]).report.verdict).toBe("pass");
|
|
expect(mergeReports([shard([], [])]).report.failures).toEqual([
|
|
"no program built with both checkouts, so nothing was compared",
|
|
]);
|
|
expect(() => mergeReports([fast, fast])).toThrow(/more than one gate report/);
|
|
// A runtime-only program was compared, and its line names the runtime change.
|
|
const runtimeOnly = {
|
|
id: "bench/r",
|
|
verdict: "runtime-only",
|
|
artifacts: { exe: { identical: false, first: "resized scr_a" } },
|
|
runtime: { changed: ["scr_a"], added: [] },
|
|
};
|
|
expect(mergeReports([shard([runtimeOnly], [])]).report.verdict).toBe("pass");
|
|
expect(programLines(runtimeOnly)).toEqual([
|
|
"runtime bench/r exe: resized scr_a; runtime-only: links 1 changed runtime function (scr_a)",
|
|
]);
|
|
});
|