mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-29 07:04:09 +08:00
Malte Ubl
1c5dfda937
benchmarks/scaling: 128 probes of common operations (arrays, strings, Map/Set/records, JSON, closures, exceptions, regex, dates, promises, timers, console, fs) run at 3-4 sizes under scriptc and Node with identical-stdout checks; run.mjs flags superlinear growth and >3x Node, --strace-only reports syscalls per operation, strace-workloads.mjs straces the runtime benchmark suite. Results and the full probe table: results/h40-pathology-hunt/REPORT.md. Pathologies fixed on this branch (probe ms at 1e3/1e4/1e5/1e6, Linux): i in number[] base 43/5291/timeout -> 0.03/0.20/1.5/15 (Node 0.19/0.73/6.8/39) k in Record<string> base 4.8/436/timeout -> 0.15/1.5/20/319 (Node 0.96/8.9/100/1452) re.exec /g loop base 1.1/70/7910/t.o. -> 0.49/4.1/40/416 (Node 0.47/4.6/43/398) clearTimeout n live base 1.3/10/1014/t.o. -> 1.3/3.9/38/740 (Node 3.8/9.4/46/527) new RegExp repeat base 3.9/38/368/3811 -> 0.13/1.1/11/106 (Node 0.49/2.9/23/174) console.error 2 writes/line -> 1 write/line Suite A/B, Linux x86_64 sandbox, candidate vs perf/stack/06 (--layouts=4 --runs=16), all workloads ok: | workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms | candidate/node | | --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: | | json-records | 202.3 | 201.9 | -0.2% | -7.0% .. +3.8% | neutral | +0.0% | 458.29 | 0.44x | | regex-logs | 166.8 | 166.4 | -0.2% | -1.6% .. +1.1% | neutral | +0.1% | 239.97 | 0.69x | | ast-interp | 268.6 | 269.9 | +0.5% | -4.6% .. +3.8% | neutral | +0.0% | 242.7 | 1.11x | | records-sort | 648.7 | 670.4 | +3.3% | -4.7% .. +10.0% | neutral | +0.0% | 222.69 | 3.01x | | template-render | 69.6 | 70.7 | +1.7% | -0.2% .. +2.6% | neutral | +0.0% | 187.3 | 0.38x | | functional-pipeline | 221.8 | 214.9 | -3.1% | -7.3% .. +0.7% | neutral | +0.0% | 249.06 | 0.86x | | async-pipeline | 197.3 | 198.2 | +0.4% | -1.2% .. +3.0% | neutral | +0.2% | 89.25 | 2.22x | | word-graph | 259.2 | 256.0 | -1.3% | -6.3% .. +3.0% | neutral | +0.0% | 469.48 | 0.55x | | alloc-trees | 318.8 | 316.8 | -0.6% | -3.0% .. +1.2% | neutral | +0.0% | 127.68 | 2.48x | | numeric-kernels | 46.4 | 46.5 | +0.1% | -1.6% .. +3.9% | neutral | +0.0% | 133.42 | 0.35x | | validate-errors | 176.5 | 178.1 | +0.9% | -3.1% .. +4.7% | neutral | +0.0% | 702.45 | 0.25x | | csv-numbers | 336.2 | 336.8 | +0.2% | -2.0% .. +1.2% | neutral | +0.0% | 500.14 | 0.67x | | log-summary | 165.7 | 164.6 | -0.6% | -5.9% .. +1.9% | neutral | +0.0% | 290.81 | 0.57x | | inventory-report | 321.9 | 320.1 | -0.6% | -6.7% .. +9.5% | neutral | +0.0% | 431.4 | 0.74x | geomean change: +0.0% geomean candidate/node: 0.768x (lower is better; goal: well below 1) macOS lane not run (all measurement in the sandbox per task instructions).
76 lines
2.9 KiB
JavaScript
76 lines
2.9 KiB
JavaScript
#!/usr/bin/env node
|
|
// strace -c every runtime benchmark workload built with a scriptc checkout and
|
|
// print its syscall totals (top calls first).
|
|
// node benchmarks/scaling/strace-workloads.mjs --scriptc=<checkout> [--out=dir]
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { dirname, join, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { parseArgs } from "node:util";
|
|
|
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
const suite = resolve(here, "../runtime");
|
|
const { values } = parseArgs({
|
|
options: {
|
|
scriptc: { type: "string", default: resolve(here, "../..") },
|
|
out: { type: "string", default: "/tmp/strace-workloads" },
|
|
json: { type: "string" },
|
|
},
|
|
});
|
|
mkdirSync(values.out, { recursive: true });
|
|
const { workloads } = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8"));
|
|
const cli = join(values.scriptc, "packages/cli/dist/bootstrap.js");
|
|
// Same inputs as scripts/bench-runtime.mjs generates.
|
|
function input(kind) {
|
|
const path = join(values.out, `${kind}.input`);
|
|
const lines =
|
|
kind === "log-summary"
|
|
? Array.from(
|
|
{ length: 300_000 },
|
|
(_, i) =>
|
|
`/route/${i % 37}?request=${i} ${i % 13 === 0 ? 500 : 200} ${i % 250} ${200 + (i % 4000)}`,
|
|
)
|
|
: Array.from({ length: 300_000 }, (_, i) =>
|
|
["sku-" + i, "category-" + (i % 17), "product " + i, i % 40, 199 + (i % 5000)].join("\t"),
|
|
);
|
|
writeFileSync(path, lines.join("\n") + "\n");
|
|
return path;
|
|
}
|
|
const results = [];
|
|
for (const w of workloads) {
|
|
const bin = join(values.out, w.name);
|
|
const build = spawnSync(
|
|
process.execPath,
|
|
[cli, "build", join(suite, w.entry), "--optimization=release", "-o", bin],
|
|
{ encoding: "utf8", cwd: dirname(join(suite, w.entry)) },
|
|
);
|
|
if (build.status !== 0) {
|
|
console.log(`${w.name}: build failed ${build.stderr.slice(0, 300)}`);
|
|
continue;
|
|
}
|
|
const trace = join(values.out, `${w.name}.strace`);
|
|
const args = (w.args ?? []).map((a) => (a === "{input}" ? input(w.input) : a));
|
|
const r = spawnSync("strace", ["-f", "-c", "-o", trace, bin, ...args], {
|
|
encoding: "utf8",
|
|
maxBuffer: 1 << 30,
|
|
cwd: dirname(join(suite, w.entry)),
|
|
});
|
|
const calls = {};
|
|
let total = 0;
|
|
for (const line of readFileSync(trace, "utf8").split("\n")) {
|
|
const m = /^\s*[0-9.]+\s+[0-9.]+\s+\d+\s+(\d+)\s+(?:\d+\s+)?(\w+)\s*$/.exec(line);
|
|
if (m && m[2] !== "total") {
|
|
calls[m[2]] = Number(m[1]);
|
|
total += Number(m[1]);
|
|
}
|
|
}
|
|
const top = Object.entries(calls)
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 8)
|
|
.map(([k, v]) => `${k}=${v}`)
|
|
.join(" ");
|
|
console.log(`${w.name} status=${r.status} total=${total} ${top}`);
|
|
results.push({ name: w.name, status: r.status, total, calls });
|
|
}
|
|
if (values.json) writeFileSync(values.json, JSON.stringify(results, null, 1));
|