scriptc/tests/harness/zero-cost-gate.test.ts

import { spawnSync } from "node:child_process";
import { expect, test } from "vitest";
import {
  cachegrindCounts,
  compareArtifact,
  compareCounts,
  compareExecutables,
  definedFunctionNames,
  dynamicTierSymbols,
  nmOutput,
  programVerdict,
} from "../../scripts/zero-cost-compare.mjs";

const ir = (functions: object[], globals: object[]) =>
  Buffer.from(JSON.stringify({ irVersion: 16, functions, globals, sourceFile: "/p/main.ts" }));

const llvm = (body: string, extra = "") =>
  Buffer.from(
    [
      "; Generated by scriptc (LLVM backend) from /p/main.ts. Do not edit.",
      '@.str = private constant [3 x i8] c"hi\\00"',
      "define internal i64 @add(i64 %a, i64 %b) {",
      `  ${body}`,
      "}",
      'define void @"main.$1"() { ret void }',
      extra,
      "declare void @scr_print(ptr)",
      "",
    ].join("\n"),
  );

test("the zero-cost gate names the first differing IR and LLVM unit", () => {
  const f = { name: "f", params: [], body: [{ kind: "return" }] };
  const g = { name: "g", params: [], body: [] };
  const base = ir([f, g, f], [{ name: "a", type: "i64" }]);
  expect(compareArtifact("ir", base, Buffer.from(base))).toEqual({ identical: true });

  const changed = compareArtifact(
    "ir",
    base,
    ir([f, { ...g, params: [1] }, f], [{ name: "a", type: "i64" }]),
  );
  expect(changed).toMatchObject({
    identical: false,
    first: "changed g",
    changed: ["g"],
    added: [],
    module: null,
  });

  const header = compareArtifact("ir", base, ir([f, g, f], [{ name: "a", type: "f64" }]));
  expect(header).toMatchObject({ first: "module globals[0:a]", changed: [] });
  // Repeated names are matched by occurrence, so an insertion shows as added.
  expect(compareArtifact("ir", base, ir([f, g, f, f], [{ name: "a", type: "i64" }]))).toMatchObject(
    {
      first: "added f#3",
    },
  );

  const ll = llvm("%r = add i64 %a, %b\n  ret i64 %r");
  expect(compareArtifact("llvm", ll, Buffer.from(ll))).toEqual({ identical: true });
  expect(compareArtifact("llvm", ll, llvm("%r = sub i64 %a, %b\n  ret i64 %r"))).toMatchObject({
    first: "changed add",
    changed: ["add"],
    functions: { baseline: 2, candidate: 2 },
  });
  expect(
    compareArtifact(
      "llvm",
      ll,
      llvm("%r = add i64 %a, %b\n  ret i64 %r", "define void @new() { ret void }"),
    ),
  ).toMatchObject({ first: "added new", added: ["new"] });
  const global = compareArtifact("llvm", ll, Buffer.from(ll.toString().replace("hi", "ho")));
  expect(global).toMatchObject({ first: "module line 2", changed: [] });
  expect(global.module.candidate).toContain("ho");
});

test("the zero-cost gate compares executables and instruction counts per function", () => {
  const nm =
    "0000000000001000 0000000000000010 T main\n0000000000001010 0000000000000020 t scr_add\n";
  const resized = compareExecutables(
    Buffer.from("abcd"),
    Buffer.from("abXd"),
    nm,
    nm.replace("0000000000000020 t", "0000000000000030 t"),
  );
  expect(resized).toMatchObject({ identical: false, first: "resized scr_add", offset: 2 });

  const output = (main: number, prefault: number, loader: number) =>
    [
      "events: Ir",
      "fl=???",
      "fn=main",
      `0 ${main}`,
      "fn=scr_sa_prefault",
      `0 ${prefault}`,
      "fn=_dl_lookup_symbol_x",
      `0 ${loader}`,
      "fn=scr_add",
      "0 7",
      "0 3",
    ].join("\n");
  const own = new Set(["main", "scr_add", "scr_sa_prefault"]);
  const base = cachegrindCounts(output(100, 50, 9000), own);
  expect([...base]).toEqual([
    ["main", 100],
    ["scr_add", 10],
  ]);
  expect(compareCounts(base, cachegrindCounts(output(100, 75, 9300), own)).identical).toBe(true);
  expect(compareCounts(base, cachegrindCounts(output(104, 50, 9000), own))).toMatchObject({
    identical: false,
    total: { baseline: 110, candidate: 114 },
    deltas: [{ name: "main", delta: 4 }],
  });
});

test("the zero-cost gate throws when nm does not list a binary's symbols", () => {
  const ok = { status: 0, signal: null, stdout: "0000000000001000 T main\n", stderr: "" };
  expect(nmOutput("prog", ok)).toBe(ok.stdout);
  const missing = spawnSync("scriptc-missing-nm", ["prog"], { encoding: "utf8" });
  expect(() => nmOutput("prog", missing)).toThrow(/^nm prog: .*ENOENT/);
  expect(() =>
    nmOutput("prog", { status: null, signal: "SIGKILL", stdout: "", stderr: "" }),
  ).toThrow("nm prog: killed by SIGKILL");
  expect(() =>
    nmOutput("prog", { status: 1, signal: null, stdout: "", stderr: "nm: prog: no symbols\n" }),
  ).toThrow("nm prog: exited with status 1: nm: prog: no symbols");
});

test("the zero-cost gate recognizes dynamic-tier runtime symbols", () => {
  expect(
    dynamicTierSymbols([
      "main",
      "scr_dyn_release",
      "scr_dyn_obj_drop_keys",
      "scr_dyn_this_push_dyn",
      "scr_js_load_ic",
      "scr_dyn_obj_read",
      "scr_dyn_obj_read",
    ]),
  ).toEqual(["scr_dyn_obj_read", "scr_js_load_ic"]);
});

test("the zero-cost gate reads function names from ELF and Mach-O nm output", () => {
  const elf = [
    "0000000000001000 0000000000000010 T scr_dyn_call",
    "0000000000001010 0000000000000008 t scr_dyn_obj_get",
    "0000000000002000 0000000000000004 D scr_dyn_table",
    "                 U memcpy",
  ].join("\n");
  expect(definedFunctionNames(elf, "linux")).toEqual(["scr_dyn_call", "scr_dyn_obj_get"]);
  // Mach-O names carry a leading underscore, which would hide every
  // dynamic-tier symbol from the prefix match.
  const macho = [
    "0000000100001000 T _scr_dyn_call",
    "0000000100001010 t _scr_dyn_obj_get",
    "0000000100002000 W __main",
    "0000000100003000 s _scr_dyn_add.message",
    "                 U _memcpy",
  ].join("\n");
  expect(definedFunctionNames(macho, "darwin")).toEqual([
    "scr_dyn_call",
    "scr_dyn_obj_get",
    "_main",
  ]);
  expect(dynamicTierSymbols(definedFunctionNames(macho, "darwin"))).toEqual([
    "scr_dyn_call",
    "scr_dyn_obj_get",
  ]);
});

test("the zero-cost gate fails differing programs unless they match the signature", () => {
  const ok = { ok: true };
  const same = { ir: { identical: true }, exe: { identical: true } };
  const differs = { ir: { identical: true }, exe: { identical: false } };
  const verdict = (
    artifacts: object,
    allowed = false,
    baseline: object = ok,
    candidate: object = ok,
    required = false,
  ) => programVerdict({ baseline, candidate, artifacts, allowed, required });
  expect(verdict(same)).toEqual({ verdict: "identical", fails: false });
  expect(verdict(differs)).toEqual({ verdict: "differs", fails: true });
  expect(verdict(differs, true)).toEqual({ verdict: "allowed", fails: false });
  const failed = { ok: false, error: "SC1090" };
  expect(verdict({}, false, failed, failed)).toEqual({ verdict: "both-fail", fails: false });
  // A benchmark or the self-hosted compiler that neither checkout builds
  // was never compared.
  expect(verdict({}, false, failed, failed, true)).toEqual({ verdict: "both-fail", fails: true });
  expect(verdict({}, true, ok, failed)).toEqual({ verdict: "build-error", fails: true });
});