scriptc/scripts/bench-runtime.mjs

/* Runtime performance A/B harness for compiled programs.
 *
 * Builds every workload in benchmarks/runtime/workloads.json with one or two
 * scriptc checkouts, verifies each executable against Node byte-for-byte
 * (stdout, stderr, exit status), then times interleaved launches and reports
 * per-workload medians with bootstrap confidence intervals for the
 * candidate/baseline ratio. Executables that diverge from Node are never
 * timed. A per-host advisory lock serializes measurements so concurrent
 * agents on one machine do not disturb each other's numbers. */
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import {
  chmodSync,
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  rmSync,
  statSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";

const repoRoot = fileURLToPath(new URL("..", import.meta.url));
const suiteRoot = join(repoRoot, "benchmarks/runtime");

const { values } = parseArgs({
  allowNegative: true,
  options: {
    candidate: { type: "string", default: repoRoot },
    baseline: { type: "string" },
    workloads: { type: "string" },
    runs: { type: "string", default: "15" },
    warmup: { type: "string", default: "2" },
    timeout: { type: "string", default: "120" },
    layouts: { type: "string", default: "1" },
    json: { type: "string" },
    "no-lock": { type: "boolean", default: false },
    node: { type: "boolean", default: true },
    "keep": { type: "boolean", default: false },
    help: { type: "boolean", default: false },
  },
});
if (values.help) {
  console.log(`Usage: node scripts/bench-runtime.mjs [--candidate=<checkout>] [--baseline=<checkout>]
       [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--layouts=1] [--json=<file>]
       [--no-lock] [--no-node] [--keep]

Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native
artifacts for this host. Without --baseline, reports absolute times only. Node is
timed in the same interleaved loop as the reference to beat (--no-node skips it).

--layouts=N (macOS/Linux) links every executable N times, shifting the program
object by 0, 16, 32, ... bytes of padding, and spreads the timed runs evenly over
those layouts, so verdicts cover code-placement luck instead of one placement.
Use it for small codegen changes (e.g. --layouts=4 --runs=16).`);
  process.exit(0);
}
const runs = Number(values.runs);
const warmup = Number(values.warmup);
const timeoutMs = Number(values.timeout) * 1000;
assert.ok(Number.isInteger(runs) && runs >= 3 && runs <= 200, "--runs must be 3..200");
assert.ok(Number.isInteger(warmup) && warmup >= 0 && warmup <= 20, "--warmup must be 0..20");
const layouts = Number(values.layouts);
assert.ok(Number.isInteger(layouts) && layouts >= 1 && layouts <= 8, "--layouts must be 1..8");
assert.ok(
  layouts === 1 || process.platform === "darwin" || process.platform === "linux",
  "--layouts needs macOS or Linux",
);

const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8"));
const selected = values.workloads ? new Set(values.workloads.split(",")) : null;
const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name));
if (selected) {
  for (const name of selected)
    assert.ok(
      manifest.workloads.some((w) => w.name === name),
      `unknown workload ${name}`,
    );
}

const work = mkdtempSync(
  join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-runtime-"),
);

/* ── code-layout perturbation ──────────────────────────────────────────── */
// Layout k > 0 links a retained k*16-byte padding object directly before the
// program object, through a SCRIPTC_LINKER wrapper that forwards to clang (the
// default linker driver on macOS and Linux). Code after that point moves as if
// earlier code had grown; any checkout's compiler can be perturbed this way.
function layoutTooling() {
  if (layouts === 1) return null;
  const root = join(work, "layouts");
  mkdirSync(root, { recursive: true });
  const symbol =
    process.platform === "darwin" ? "_scriptc_bench_layout_pad" : "scriptc_bench_layout_pad";
  const pads = [null];
  for (let k = 1; k < layouts; k++) {
    const source = join(root, `pad${k}.s`);
    const object = join(root, `pad${k}.o`);
    writeFileSync(
      source,
      process.platform === "darwin"
        ? `.section __TEXT,__text,regular,pure_instructions\n.globl ${symbol}\n.no_dead_strip ${symbol}\n.p2align 0\n${symbol}:\n.fill ${k * 16},1,0xcc\n`
        : `.section .text.scriptc_bench_layout_pad,"axR",@progbits\n.globl ${symbol}\n.p2align 0\n${symbol}:\n.fill ${k * 16},1,0xcc\n`,
    );
    const assembled = run("clang", ["-c", source, "-o", object], { timeout: 60_000 });
    assert.equal(assembled.status, 0, `cannot assemble layout padding: ${assembled.stderr}`);
    pads.push(object);
  }
  const wrapper = join(root, "link.mjs");
  writeFileSync(
    wrapper,
    `#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import { basename } from "node:path";
const args = process.argv.slice(2);
const pad = process.env.SCRIPTC_BENCH_LAYOUT_PAD;
let out = args;
if (pad && !args.some((a) => a.startsWith("-print-prog-name"))) {
  const at = args.findIndex((a) => basename(a).startsWith(".scriptc-native-program-object-"));
  if (at < 0) {
    process.stderr.write("bench layout wrapper: no program object on the link line\\n");
    process.exit(2);
  }
  out = ["-Wl,-u,${symbol}", ...args.slice(0, at), pad, ...args.slice(at)];
}
const result = spawnSync("clang", out, { stdio: "inherit" });
process.exit(result.status ?? 1);
`,
  );
  chmodSync(wrapper, 0o755);
  return { wrapper, pads };
}
const layoutTools = layoutTooling();

/* ── inputs for file-processing applications ──────────────────────────── */
function generateInput(kind) {
  const path = join(work, `${kind}.input`);
  if (kind === "log-summary") {
    writeFileSync(
      path,
      Array.from(
        { length: 300_000 },
        (_, i) =>
          `/route/${i % 37}?request=${i} ${i % 13 === 0 ? 500 : 200} ${i % 250} ${200 + (i % 4000)}`,
      ).join("\n") + "\n",
    );
  } else if (kind === "inventory-report") {
    writeFileSync(
      path,
      Array.from({ length: 300_000 }, (_, i) =>
        ["sku-" + i, "category-" + (i % 17), "product " + i, i % 40, 199 + (i % 5000)].join("\t"),
      ).join("\n") + "\n",
    );
  } else throw new Error(`unknown input kind ${kind}`);
  return path;
}

/* ── advisory lock ─────────────────────────────────────────────────────── */
const lockDir = join(
  process.platform === "win32" ? tmpdir() : "/tmp",
  "scriptc-bench-runtime.lock",
);
function acquireLock() {
  const started = Date.now();
  let announced = false;
  for (;;) {
    try {
      mkdirSync(lockDir);
      writeFileSync(join(lockDir, "pid"), String(process.pid));
      return;
    } catch (error) {
      if (error.code !== "EEXIST") throw error;
      let owner = NaN;
      try {
        owner = Number(readFileSync(join(lockDir, "pid"), "utf8"));
      } catch {}
      let alive = false;
      if (Number.isInteger(owner)) {
        try {
          process.kill(owner, 0);
          alive = true;
        } catch {}
      }
      if (!alive && Date.now() - started > 2000) {
        rmSync(lockDir, { recursive: true, force: true });
        continue;
      }
      if (!announced) {
        process.stderr.write(`waiting for benchmark lock held by pid ${owner}\n`);
        announced = true;
      }
      spawnSync("sleep", ["2"]);
    }
  }
}
function releaseLock() {
  rmSync(lockDir, { recursive: true, force: true });
}

/* ── process helpers ───────────────────────────────────────────────────── */
function run(command, args, options = {}) {
  const start = process.hrtime.bigint();
  const result = spawnSync(command, args, {
    encoding: "utf8",
    maxBuffer: 64 * 1024 * 1024,
    timeout: options.timeout ?? timeoutMs,
    env: options.env ?? process.env,
    cwd: options.cwd,
  });
  const ms = Number(process.hrtime.bigint() - start) / 1e6;
  if (result.error && result.error.code !== "ETIMEDOUT") throw result.error;
  return {
    stdout: result.stdout ?? "",
    stderr: result.stderr ?? "",
    status: result.status,
    signal: result.signal,
    timedOut: result.error?.code === "ETIMEDOUT" || result.signal === "SIGTERM",
    ms,
  };
}

function peakRss(binary, args) {
  if (process.platform !== "darwin" && process.platform !== "linux") return null;
  if (!existsSync("/usr/bin/time")) return null;
  const timeArgs =
    process.platform === "darwin" ? ["-l", binary, ...args] : ["-f", "rss_kib=%M", binary, ...args];
  const result = run("/usr/bin/time", timeArgs);
  const match =
    process.platform === "darwin"
      ? /(\d+)\s+maximum resident set size/.exec(result.stderr)
      : /rss_kib=(\d+)/.exec(result.stderr);
  if (!match) return null;
  return Number(match[1]) * (process.platform === "darwin" ? 1 : 1024);
}

/* ── statistics ────────────────────────────────────────────────────────── */
function median(xs) {
  const s = [...xs].sort((a, b) => a - b);
  const m = s.length >> 1;
  return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
}
function makeRandom(seed) {
  let state = seed >>> 0;
  return () => {
    state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
    return state / 4294967296;
  };
}
function bootstrapRatio(candidate, baseline, resamples = 2000) {
  const random = makeRandom(0x5eed);
  const ratios = [];
  const pick = (xs) =>
    Array.from({ length: xs.length }, () => xs[Math.floor(random() * xs.length)]);
  for (let i = 0; i < resamples; i++) ratios.push(median(pick(candidate)) / median(pick(baseline)));
  ratios.sort((a, b) => a - b);
  return {
    low: ratios[Math.floor(resamples * 0.025)],
    high: ratios[Math.floor(resamples * 0.975)],
  };
}

/* ── build ─────────────────────────────────────────────────────────────── */
function cliFor(root) {
  const cli = join(resolve(root), "packages/cli/dist/bootstrap.js");
  statSync(cli);
  return cli;
}
function revision(root) {
  const r = run("git", ["-C", root, "rev-parse", "--short", "HEAD"], { timeout: 10_000 });
  const dirty = run("git", ["-C", root, "status", "--porcelain", "--untracked-files=no"], {
    timeout: 10_000,
  });
  return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : "");
}
function build(label, root, workload, layout = 0) {
  const cli = cliFor(root);
  const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`);
  mkdirSync(dirname(out), { recursive: true });
  const cache = join(work, label, ".cache");
  mkdirSync(cache, { recursive: true });
  const env = { ...process.env, SCRIPTC_CACHE_DIR: cache };
  delete env.SCRIPTC_NO_CACHE;
  delete env.SCRIPTC_TIMING;
  if (layout !== 0) {
    env.SCRIPTC_LINKER = layoutTools.wrapper;
    env.SCRIPTC_BENCH_LAYOUT_PAD = layoutTools.pads[layout];
  }
  const entry = join(suiteRoot, workload.entry);
  const result = run(process.execPath, [cli, "build", entry, "--optimization=release", "-o", out], {
    env,
    timeout: 900_000,
  });
  if (result.status !== 0)
    return {
      ok: false,
      error: (result.stderr || result.stdout).slice(0, 4000),
      buildMs: result.ms,
    };
  return { ok: true, binary: out, bytes: statSync(out).size, buildMs: result.ms };
}

/* ── main ──────────────────────────────────────────────────────────────── */
const contenders = [
  ...(values.baseline ? [{ label: "baseline", root: resolve(values.baseline) }] : []),
  { label: "candidate", root: resolve(values.candidate) },
];
const report = {
  schema: 1,
  date: new Date().toISOString(),
  host: { platform: process.platform, arch: process.arch, node: process.version },
  runs,
  layouts,
  contenders: contenders.map((c) => ({ ...c, revision: revision(c.root) })),
  workloads: [],
};

try {
  const inputs = new Map();
  for (const w of workloads)
    if (w.input && !inputs.has(w.input)) inputs.set(w.input, generateInput(w.input));

  // Builds and the Node oracle run outside the lock: they do not need a
  // quiet machine, and holding the lock while compiling would serialize
  // every agent's slowest phase.
  const prepared = [];
  for (const w of workloads) {
    const args = w.args.map((a) => (a === "{input}" ? inputs.get(w.input) : a));
    // The application workloads are ESM without a package.json "type"; Node's
    // reparse warning is an oracle artifact, not program output.
    const oracle = run(process.execPath, [
      "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON",
      join(suiteRoot, w.entry),
      ...args,
    ]);
    const entry = { name: w.name, args, results: {} };
    if (oracle.status !== 0 || oracle.timedOut) {
      entry.error = `node oracle failed (status ${oracle.status}): ${oracle.stderr.slice(0, 2000)}`;
      prepared.push(entry);
      continue;
    }
    for (const c of contenders) {
      const result = { build_ms: 0 };
      entry.results[c.label] = result;
      result.binaries = [];
      for (let layout = 0; layout < layouts && result.status === undefined; layout++) {
        const built = build(c.label, c.root, w, layout);
        result.build_ms += Math.round(built.buildMs);
        if (!built.ok) {
          result.status = "build-failed";
          result.error = built.error;
          break;
        }
        if (layout === 0) result.bytes = built.bytes;
        result.binaries.push(built.binary);
        const check = run(built.binary, args);
        if (check.timedOut) {
          result.status = "timeout";
          break;
        }
        if (
          check.stdout !== oracle.stdout ||
          check.stderr !== oracle.stderr ||
          check.status !== oracle.status
        ) {
          result.status = "mismatch";
          result.error = `${layout === 0 ? "" : `layout ${layout}: `}expected status ${oracle.status}, got ${check.status}; stdout ${check.stdout === oracle.stdout ? "matches" : "differs"}; stderr ${check.stderr === oracle.stderr ? "matches" : "differs"}`;
          break;
        }
      }
      result.binary = result.binaries[0];
      if (result.status !== undefined) continue;
      result.status = "ok";
      result.samples = [];
      if (layouts > 1) result.layout_samples = result.binaries.map(() => []);
    }
    entry.node_ms = Math.round(oracle.ms);
    entry.nodeCommand = [
      "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON",
      join(suiteRoot, w.entry),
      ...args,
    ];
    prepared.push(entry);
    process.stderr.write(`prepared ${w.name}\n`);
  }

  if (!values["no-lock"]) acquireLock();
  try {
    for (const entry of prepared) {
      const live = contenders.filter((c) => entry.results[c.label]?.status === "ok");
      if (live.length === 0) continue;
      for (const c of live)
        for (const binary of entry.results[c.label].binaries)
          for (let i = 0; i < warmup; i++) run(binary, entry.args);
      // Node is the reference the project aims to beat, so it is timed in the
      // same interleaved loop (whole-process wall time, startup included).
      const nodeSamples = [];
      if (values.node) run(process.execPath, entry.nodeCommand);
      const random = makeRandom(entry.name.length * 7919);
      for (let r = 0; r < runs; r++) {
        const order = [...live];
        if (order.length === 2 && random() < 0.5) order.reverse();
        for (const c of order) {
          const result = entry.results[c.label];
          // Runs rotate through the layouts so each gets an equal share.
          const layout = r % result.binaries.length;
          const sample = run(result.binaries[layout], entry.args);
          if (sample.timedOut || sample.status !== 0) {
            result.status = sample.timedOut ? "timeout" : "unstable-exit";
            break;
          }
          const ms = Math.round(sample.ms * 100) / 100;
          result.samples.push(ms);
          result.layout_samples?.[layout].push(ms);
        }
        if (values.node) nodeSamples.push(run(process.execPath, entry.nodeCommand).ms);
      }
      if (nodeSamples.length > 0) entry.node_ms = Math.round(median(nodeSamples) * 100) / 100;
      const reference = entry.results.candidate;
      if (reference?.status === "ok" && reference.samples.length > 0 && nodeSamples.length > 0)
        entry.vs_node = Math.round((median(reference.samples) / entry.node_ms) * 1000) / 1000;
      for (const c of live) {
        const result = entry.results[c.label];
        if (result.status !== "ok") continue;
        result.median_ms = Math.round(median(result.samples) * 100) / 100;
        if (result.layout_samples)
          result.layout_medians_ms = result.layout_samples.map(
            (xs) => Math.round(median(xs) * 100) / 100,
          );
        result.peak_rss_bytes = peakRss(result.binary, entry.args);
      }
      const base = entry.results.baseline;
      const cand = entry.results.candidate;
      if (base?.status === "ok" && cand?.status === "ok") {
        const ci = bootstrapRatio(cand.samples, base.samples);
        entry.ratio = Math.round((cand.median_ms / base.median_ms) * 10000) / 10000;
        entry.ci95 = [Math.round(ci.low * 10000) / 10000, Math.round(ci.high * 10000) / 10000];
        entry.verdict = ci.high < 1 ? "faster" : ci.low > 1 ? "slower" : "neutral";
        entry.size_ratio = Math.round((cand.bytes / base.bytes) * 10000) / 10000;
      }
      process.stderr.write(`measured ${entry.name}\n`);
    }
  } finally {
    if (!values["no-lock"]) releaseLock();
  }
  for (const entry of prepared) {
    for (const result of Object.values(entry.results)) {
      delete result.binary;
      delete result.binaries;
    }
    delete entry.nodeCommand;
    report.workloads.push(entry);
  }
  const vsNode = report.workloads.filter((w) => w.vs_node !== undefined).map((w) => w.vs_node);
  if (vsNode.length > 0)
    report.geomean_vs_node =
      Math.round(Math.exp(vsNode.reduce((s, r) => s + Math.log(r), 0) / vsNode.length) * 1000) /
      1000;
  const ratios = report.workloads.filter((w) => w.ratio !== undefined).map((w) => w.ratio);
  if (ratios.length > 0)
    report.geomean_ratio =
      Math.round(Math.exp(ratios.reduce((s, r) => s + Math.log(r), 0) / ratios.length) * 10000) /
      10000;
} finally {
  if (!values.keep) rmSync(work, { recursive: true, force: true });
}

if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n");

/* ── summary table ─────────────────────────────────────────────────────── */
const pct = (r) => `${r < 1 ? "" : "+"}${((r - 1) * 100).toFixed(1)}%`;
const lines = [];
const xNode = (w) => (w.vs_node === undefined ? "-" : `${w.vs_node.toFixed(2)}x`);
if (values.baseline) {
  lines.push(
    "| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms | candidate/node |",
  );
  lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: |");
} else {
  lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms | scriptc/node |");
  lines.push("| --- | ---: | --- | ---: | ---: | ---: | ---: |");
}
for (const w of report.workloads) {
  const b = w.results.baseline;
  const c = w.results.candidate;
  if (w.error) {
    lines.push(`| ${w.name} | ${w.error.split("\n")[0]} |`);
  } else if (values.baseline) {
    const show = (r) => (r?.status === "ok" ? r.median_ms.toFixed(1) : (r?.status ?? "-"));
    lines.push(
      `| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} | ${xNode(w)} |`,
    );
  } else {
    lines.push(
      `| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} | ${xNode(w)} |`,
    );
  }
}
if (layouts > 1) {
  // Each layout gets only runs/layouts samples, so these spreads include run
  // noise; the verdicts above already pool every layout.
  lines.push(
    "",
    `layout spread (max/min of per-layout medians, ${layouts} layouts x ~${Math.floor(runs / layouts)} runs; includes run noise):`,
  );
  for (const w of report.workloads) {
    const spreads = Object.entries(w.results)
      .filter(([, r]) => r.layout_medians_ms)
      .map(([label, r]) => {
        const ms = r.layout_medians_ms;
        return `${label} ${((Math.max(...ms) / Math.min(...ms) - 1) * 100).toFixed(1)}%`;
      });
    if (spreads.length > 0) lines.push(`- ${w.name}: ${spreads.join(", ")}`);
  }
}
if (report.geomean_ratio !== undefined)
  lines.push("", `geomean change: ${pct(report.geomean_ratio)}`);
if (report.geomean_vs_node !== undefined)
  lines.push(
    `geomean candidate/node: ${report.geomean_vs_node.toFixed(3)}x (lower is better; goal: well below 1)`,
  );
console.log(lines.join("\n"));
const broken = report.workloads.some(
  (w) => w.error || Object.values(w.results).some((r) => r.status !== "ok"),
);
process.exitCode = broken ? 1 : 0;