mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-27 06:43:19 +08:00
Merge pull request #742 from vercel-labs/perf/rebased/harness
Add a runtime benchmark suite that times compiled programs against Node
This commit is contained in:
Malte Ubl
@@ -0,0 +1,27 @@
|
||||
# Runtime benchmarks
|
||||
|
||||
Application-shaped workloads for measuring the speed of compiled executables. Each workload is deterministic, generates its own input (or reads a generated file), and prints a summary that must match Node.js byte-for-byte before it is timed.
|
||||
|
||||
| Workload | Exercises |
|
||||
| --- | --- |
|
||||
| `json-records` | `JSON.stringify`, typed `JSON.parse`, Map aggregation |
|
||||
| `regex-logs` | regex `exec` with groups, `test`, global `replace` |
|
||||
| `ast-interp` | tokenizer, recursive-descent parser, class dispatch |
|
||||
| `records-sort` | comparator sorts, group-by, `reduce`/`filter` |
|
||||
| `template-render` | template literals, escaping, `map`/`join` |
|
||||
| `functional-pipeline` | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records |
|
||||
| `async-pipeline` | `async`/`await`, `Promise.all` fan-out |
|
||||
| `word-graph` | string-keyed Map/Set, BFS |
|
||||
| `alloc-trees` | allocation-heavy trees, immutable object-spread updates |
|
||||
| `numeric-kernels` | sieve, hashing, `number[][]` matmul, `Float64Array` |
|
||||
| `validate-errors` | throw/catch on a fraction of inputs, small call chains |
|
||||
| `csv-numbers` | `split`, `Number`, `parseInt`, `toFixed` |
|
||||
| `log-summary`, `inventory-report` | the multi-module applications from `benchmarks/builds`, on 300k-line inputs |
|
||||
|
||||
Compare two checkouts (each with a built CLI and native artifacts for the host):
|
||||
|
||||
```console
|
||||
$ pnpm bench:runtime --baseline=../scriptc-main --candidate=. --runs=15 --json=result.json
|
||||
```
|
||||
|
||||
Without `--baseline`, the runner reports absolute medians. Baseline and candidate launches are interleaved, and each workload reports the median ratio with a bootstrap 95% confidence interval. `faster`/`slower` verdicts require the whole interval on one side of 1.0. A per-host lock serializes the timed phase so concurrent runs on one machine do not distort each other. Executables that time out or diverge from Node are reported, never timed.
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"workloads": [
|
||||
{
|
||||
"name": "json-records",
|
||||
"entry": "workloads/json-records.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regex-logs",
|
||||
"entry": "workloads/regex-logs.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ast-interp",
|
||||
"entry": "workloads/ast-interp.ts",
|
||||
"args": [
|
||||
"5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "records-sort",
|
||||
"entry": "workloads/records-sort.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "template-render",
|
||||
"entry": "workloads/template-render.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "functional-pipeline",
|
||||
"entry": "workloads/functional-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "async-pipeline",
|
||||
"entry": "workloads/async-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "word-graph",
|
||||
"entry": "workloads/word-graph.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "alloc-trees",
|
||||
"entry": "workloads/alloc-trees.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "numeric-kernels",
|
||||
"entry": "workloads/numeric-kernels.ts",
|
||||
"args": [
|
||||
"0.5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "validate-errors",
|
||||
"entry": "workloads/validate-errors.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "csv-numbers",
|
||||
"entry": "workloads/csv-numbers.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "log-summary",
|
||||
"entry": "../builds/log-summary/main.ts",
|
||||
"input": "log-summary",
|
||||
"args": [
|
||||
"{input}",
|
||||
"--min-status",
|
||||
"200",
|
||||
"--limit",
|
||||
"12"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inventory-report",
|
||||
"entry": "../builds/inventory-report/main.ts",
|
||||
"input": "inventory-report",
|
||||
"args": [
|
||||
"{input}"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Allocation-heavy code: binary trees built and discarded, plus a
|
||||
// reducer that produces immutable state updates with object spread.
|
||||
class TreeNode {
|
||||
readonly left: TreeNode | null;
|
||||
readonly right: TreeNode | null;
|
||||
constructor(left: TreeNode | null, right: TreeNode | null) {
|
||||
this.left = left;
|
||||
this.right = right;
|
||||
}
|
||||
check(): number {
|
||||
return 1 + (this.left === null ? 0 : this.left.check()) + (this.right === null ? 0 : this.right.check());
|
||||
}
|
||||
}
|
||||
function build(depth: number): TreeNode {
|
||||
return depth === 0 ? new TreeNode(null, null) : new TreeNode(build(depth - 1), build(depth - 1));
|
||||
}
|
||||
|
||||
interface State {
|
||||
count: number;
|
||||
total: number;
|
||||
last: string;
|
||||
history: number[];
|
||||
}
|
||||
interface Action {
|
||||
type: string;
|
||||
amount: number;
|
||||
}
|
||||
function reduce(state: State, action: Action): State {
|
||||
if (action.type === "add") return { ...state, count: state.count + 1, total: state.total + action.amount, last: action.type };
|
||||
if (action.type === "reset") return { ...state, total: 0, last: action.type, history: [...state.history.slice(-4), state.total] };
|
||||
return { ...state, last: action.type };
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const maxDepth = 14;
|
||||
let checks = 0;
|
||||
for (let depth = 4; depth <= maxDepth; depth += 2) {
|
||||
const iterations = Math.floor((1 << (maxDepth - depth + 4)) * scale);
|
||||
for (let i = 0; i < iterations; i++) checks += build(depth).check();
|
||||
}
|
||||
const longLived = build(maxDepth);
|
||||
let state: State = { count: 0, total: 0, last: "", history: [] };
|
||||
for (let i = 0; i < Math.floor(300000 * scale); i++) {
|
||||
state = reduce(state, { type: i % 97 === 0 ? "reset" : i % 3 === 0 ? "noop" : "add", amount: i % 13 });
|
||||
}
|
||||
console.log("checks", checks, "long", longLived.check(), "state", state.count, state.total, state.history.join(","));
|
||||
@@ -0,0 +1,199 @@
|
||||
// A tokenizer, recursive-descent parser, and class-based tree-walking
|
||||
// evaluator for a small expression language with variables.
|
||||
abstract class Expr {
|
||||
abstract evaluate(env: Map<string, number>): number;
|
||||
}
|
||||
class Num extends Expr {
|
||||
readonly value: number;
|
||||
constructor(value: number) {
|
||||
super();
|
||||
this.value = value;
|
||||
}
|
||||
evaluate(_env: Map<string, number>): number {
|
||||
return this.value;
|
||||
}
|
||||
}
|
||||
class Var extends Expr {
|
||||
readonly name: string;
|
||||
constructor(name: string) {
|
||||
super();
|
||||
this.name = name;
|
||||
}
|
||||
evaluate(env: Map<string, number>): number {
|
||||
return env.get(this.name) ?? 0;
|
||||
}
|
||||
}
|
||||
class Binary extends Expr {
|
||||
readonly op: string;
|
||||
readonly left: Expr;
|
||||
readonly right: Expr;
|
||||
constructor(op: string, left: Expr, right: Expr) {
|
||||
super();
|
||||
this.op = op;
|
||||
this.left = left;
|
||||
this.right = right;
|
||||
}
|
||||
evaluate(env: Map<string, number>): number {
|
||||
const l = this.left.evaluate(env);
|
||||
const r = this.right.evaluate(env);
|
||||
switch (this.op) {
|
||||
case "+":
|
||||
return l + r;
|
||||
case "-":
|
||||
return l - r;
|
||||
case "*":
|
||||
return l * r;
|
||||
case "/":
|
||||
return r === 0 ? 0 : l / r;
|
||||
default:
|
||||
return l % (r === 0 ? 1 : r);
|
||||
}
|
||||
}
|
||||
}
|
||||
class Call extends Expr {
|
||||
readonly name: string;
|
||||
readonly args: Expr[];
|
||||
constructor(name: string, args: Expr[]) {
|
||||
super();
|
||||
this.name = name;
|
||||
this.args = args;
|
||||
}
|
||||
evaluate(env: Map<string, number>): number {
|
||||
const values = this.args.map((arg) => arg.evaluate(env));
|
||||
if (this.name === "max") return Math.max(...values);
|
||||
if (this.name === "min") return Math.min(...values);
|
||||
return values.reduce((a, b) => a + b, 0);
|
||||
}
|
||||
}
|
||||
|
||||
type TokenKind = "num" | "ident" | "op" | "lparen" | "rparen" | "comma" | "end";
|
||||
interface Token {
|
||||
kind: TokenKind;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function isDigit(c: number): boolean {
|
||||
return (c >= 48 && c <= 57) || c === 46;
|
||||
}
|
||||
function isIdent(c: number): boolean {
|
||||
return (c >= 97 && c <= 122) || c === 95;
|
||||
}
|
||||
function tokenize(source: string): Token[] {
|
||||
const tokens: Token[] = [];
|
||||
let i = 0;
|
||||
while (i < source.length) {
|
||||
const c = source.charCodeAt(i);
|
||||
if (c === 32) {
|
||||
i++;
|
||||
} else if (c >= 48 && c <= 57) {
|
||||
const start = i;
|
||||
while (i < source.length && isDigit(source.charCodeAt(i))) i++;
|
||||
tokens.push({ kind: "num", text: source.slice(start, i) });
|
||||
} else if (isIdent(c)) {
|
||||
const start = i;
|
||||
while (i < source.length && isIdent(source.charCodeAt(i))) i++;
|
||||
tokens.push({ kind: "ident", text: source.slice(start, i) });
|
||||
} else if (c === 40) {
|
||||
tokens.push({ kind: "lparen", text: "(" });
|
||||
i++;
|
||||
} else if (c === 41) {
|
||||
tokens.push({ kind: "rparen", text: ")" });
|
||||
i++;
|
||||
} else if (c === 44) {
|
||||
tokens.push({ kind: "comma", text: "," });
|
||||
i++;
|
||||
} else {
|
||||
tokens.push({ kind: "op", text: source[i]! });
|
||||
i++;
|
||||
}
|
||||
}
|
||||
tokens.push({ kind: "end", text: "" });
|
||||
return tokens;
|
||||
}
|
||||
|
||||
class Parser {
|
||||
private pos = 0;
|
||||
private readonly tokens: Token[];
|
||||
constructor(tokens: Token[]) {
|
||||
this.tokens = tokens;
|
||||
}
|
||||
private peek(): Token {
|
||||
return this.tokens[this.pos]!;
|
||||
}
|
||||
private next(): Token {
|
||||
return this.tokens[this.pos++]!;
|
||||
}
|
||||
parseExpr(): Expr {
|
||||
let left = this.parseTerm();
|
||||
while (this.peek().kind === "op" && (this.peek().text === "+" || this.peek().text === "-")) {
|
||||
const op = this.next().text;
|
||||
left = new Binary(op, left, this.parseTerm());
|
||||
}
|
||||
return left;
|
||||
}
|
||||
private parseTerm(): Expr {
|
||||
let left = this.parseFactor();
|
||||
while (this.peek().kind === "op" && (this.peek().text === "*" || this.peek().text === "/" || this.peek().text === "%")) {
|
||||
const op = this.next().text;
|
||||
left = new Binary(op, left, this.parseFactor());
|
||||
}
|
||||
return left;
|
||||
}
|
||||
private parseFactor(): Expr {
|
||||
const token = this.next();
|
||||
if (token.kind === "num") return new Num(Number(token.text));
|
||||
if (token.kind === "lparen") {
|
||||
const inner = this.parseExpr();
|
||||
this.next();
|
||||
return inner;
|
||||
}
|
||||
if (this.peek().kind === "lparen") {
|
||||
this.next();
|
||||
const args: Expr[] = [];
|
||||
while (this.peek().kind !== "rparen") {
|
||||
args.push(this.parseExpr());
|
||||
if (this.peek().kind === "comma") this.next();
|
||||
}
|
||||
this.next();
|
||||
return new Call(token.text, args);
|
||||
}
|
||||
return new Var(token.text);
|
||||
}
|
||||
}
|
||||
|
||||
let seed = 7;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
const names = ["alpha", "beta", "gamma", "delta", "x", "y"];
|
||||
const ops = ["+", "-", "*", "/", "%"];
|
||||
function generate(depth: number): string {
|
||||
const r = random();
|
||||
if (depth === 0 || r < 0.2) return r < 0.1 ? String(Math.floor(random() * 100)) : names[Math.floor(random() * names.length)]!;
|
||||
if (r < 0.3) return `max(${generate(depth - 1)}, ${generate(depth - 1)}, ${generate(depth - 1)})`;
|
||||
if (r < 0.4) return `(${generate(depth - 1)})`;
|
||||
return `${generate(depth - 1)} ${ops[Math.floor(random() * ops.length)]!} ${generate(depth - 1)}`;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const sources: string[] = [];
|
||||
for (let i = 0; i < Math.floor(400 * scale); i++) sources.push(generate(7));
|
||||
const env = new Map<string, number>();
|
||||
let total = 0;
|
||||
let chars = 0;
|
||||
for (const source of sources) {
|
||||
const tree = new Parser(tokenize(source)).parseExpr();
|
||||
chars += source.length;
|
||||
for (let step = 0; step < 40; step++) {
|
||||
env.set("alpha", step);
|
||||
env.set("beta", step * 2 + 1);
|
||||
env.set("gamma", 3);
|
||||
env.set("delta", step % 7);
|
||||
env.set("x", 1.5);
|
||||
env.set("y", -2);
|
||||
const value = tree.evaluate(env);
|
||||
if (Number.isFinite(value)) total += value;
|
||||
}
|
||||
}
|
||||
console.log("sources", sources.length, "chars", chars, "total", total.toFixed(6));
|
||||
@@ -0,0 +1,47 @@
|
||||
// Async application code: request handlers awaiting cached and computed
|
||||
// values, Promise.all fan-out, and sequential awaits in loops.
|
||||
interface User {
|
||||
id: number;
|
||||
name: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
const cache = new Map<number, User>();
|
||||
async function loadUser(id: number): Promise<User> {
|
||||
const cached = cache.get(id);
|
||||
if (cached !== undefined) return cached;
|
||||
const user = { id, name: "user-" + id, score: (id * 37) % 101 };
|
||||
cache.set(id, user);
|
||||
return user;
|
||||
}
|
||||
async function authorize(user: User): Promise<boolean> {
|
||||
return user.score % 7 !== 0;
|
||||
}
|
||||
async function handle(requestId: number): Promise<number> {
|
||||
const user = await loadUser(requestId % 3000);
|
||||
if (!(await authorize(user))) return 0;
|
||||
const requests: Promise<User>[] = [loadUser((requestId + 1) % 3000), loadUser((requestId + 7) % 3000), loadUser((requestId + 13) % 3000)];
|
||||
const friends = await Promise.all(requests);
|
||||
let sum = user.score;
|
||||
for (const friend of friends) sum += friend.score;
|
||||
return sum;
|
||||
}
|
||||
async function batch(start: number, size: number): Promise<number> {
|
||||
let total = 0;
|
||||
for (let i = 0; i < size; i++) total += await handle(start + i);
|
||||
return total;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const batches = Math.floor(200 * scale);
|
||||
let total = 0;
|
||||
for (let round = 0; round < batches; round += 10) {
|
||||
const work: Promise<number>[] = [];
|
||||
for (let b = 0; b < 10; b++) work.push(batch((round + b) * 100, 100));
|
||||
const results = await Promise.all(work);
|
||||
for (const r of results) total += r;
|
||||
}
|
||||
console.log("handled", batches * 100, "total", total);
|
||||
}
|
||||
void main();
|
||||
@@ -0,0 +1,43 @@
|
||||
// CSV parsing and numeric formatting: split, Number(), parseInt,
|
||||
// toFixed, String(n), and string assembly of a report.
|
||||
let seed = 1001;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
function makeCsv(rows: number): string {
|
||||
const lines: string[] = ["date,store,units,unit_price,discount"];
|
||||
for (let i = 0; i < rows; i++) {
|
||||
const day = 1 + (i % 28);
|
||||
lines.push(
|
||||
`2026-${String(1 + (i % 12)).padStart(2, "0")}-${String(day).padStart(2, "0")},store-${Math.floor(random() * 40)},${Math.floor(random() * 50)},${(random() * 200).toFixed(2)},${(random() * 0.3).toFixed(3)}`,
|
||||
);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const csv = makeCsv(Math.floor(150000 * scale));
|
||||
const totals = new Map<string, number>();
|
||||
const months: number[] = new Array<number>(13).fill(0);
|
||||
let units = 0;
|
||||
let formatted = 0;
|
||||
for (let round = 0; round < 2; round++) {
|
||||
const lines = csv.split("\n");
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const cols = lines[i]!.split(",");
|
||||
const month = parseInt(cols[0]!.slice(5, 7), 10);
|
||||
const count = Number(cols[2]);
|
||||
const price = Number(cols[3]);
|
||||
const discount = Number(cols[4]);
|
||||
const revenue = count * price * (1 - discount);
|
||||
units += count;
|
||||
months[month] = months[month]! + revenue;
|
||||
totals.set(cols[1]!, (totals.get(cols[1]!) ?? 0) + revenue);
|
||||
formatted += (revenue.toFixed(2) + "|" + String(count)).length;
|
||||
}
|
||||
}
|
||||
const stores = [...totals.keys()].sort();
|
||||
const report = stores.map((s) => `${s}: ${totals.get(s)!.toFixed(2)}`).join("\n");
|
||||
console.log(report.split("\n").slice(0, 5).join("\n"));
|
||||
console.log("units", units, "formatted", formatted, "jan", months[1]!.toFixed(2), "dec", months[12]!.toFixed(2));
|
||||
@@ -0,0 +1,56 @@
|
||||
// Functional collection pipelines: map/filter/reduce/flatMap chains,
|
||||
// closures capturing configuration, and small intermediate objects.
|
||||
interface Event {
|
||||
user: number;
|
||||
kind: string;
|
||||
value: number;
|
||||
time: number;
|
||||
}
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
interface Scored {
|
||||
user: number;
|
||||
score: number;
|
||||
}
|
||||
|
||||
let seed = 4242;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
const kinds = ["view", "click", "purchase", "share"];
|
||||
function makeEvents(count: number): Event[] {
|
||||
const events: Event[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
events.push({ user: Math.floor(random() * 5000), kind: kinds[Math.floor(random() * kinds.length)]!, value: Math.floor(random() * 1000), time: i });
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function makeScorer(weight: number, bonus: number): (e: Event) => number {
|
||||
return (e) => (e.kind === "purchase" ? e.value * weight + bonus : e.kind === "click" ? weight : 0);
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const events = makeEvents(Math.floor(200000 * scale));
|
||||
let total = 0;
|
||||
for (let round = 0; round < 5; round++) {
|
||||
const scorer = makeScorer(round + 1, round * 10);
|
||||
const scored: Scored[] = events
|
||||
.filter((e) => e.time % 5 !== round)
|
||||
.map((e) => ({ user: e.user, score: scorer(e) }))
|
||||
.filter((s) => s.score > 0);
|
||||
total += scored.reduce((sum, s) => sum + s.score, 0);
|
||||
const pairs = events.slice(0, 20000).flatMap((e) => (e.kind === "share" ? [e.user, e.value] : []));
|
||||
total += pairs.length;
|
||||
const points: Point[] = events.slice(0, 50000).map((e) => ({ x: e.value / 10, y: e.user / 100 }));
|
||||
const centroid = points.reduce((acc: Point, p: Point): Point => ({ x: acc.x + p.x, y: acc.y + p.y }), { x: 0, y: 0 });
|
||||
total += Math.round(centroid.x + centroid.y);
|
||||
const some = events.some((e) => e.value === 999 - round);
|
||||
const every = events.every((e) => e.value >= 0);
|
||||
const found = events.findIndex((e) => e.user === 4999 - round);
|
||||
total += (some ? 1 : 0) + (every ? 1 : 0) + found;
|
||||
}
|
||||
console.log("total", total);
|
||||
@@ -0,0 +1,102 @@
|
||||
// JSON round trips of typed records: build, stringify, parse back into a
|
||||
// declared shape, aggregate, and stringify the summary.
|
||||
interface LineItem {
|
||||
sku: string;
|
||||
quantity: number;
|
||||
price: number;
|
||||
}
|
||||
interface Order {
|
||||
id: number;
|
||||
customer: string;
|
||||
region: string;
|
||||
priority: boolean;
|
||||
items: LineItem[];
|
||||
tags: string[];
|
||||
}
|
||||
interface Summary {
|
||||
region: string;
|
||||
orders: number;
|
||||
revenue: number;
|
||||
topSku: string;
|
||||
}
|
||||
interface RegionTotals {
|
||||
orders: number;
|
||||
revenue: number;
|
||||
skus: Map<string, number>;
|
||||
}
|
||||
|
||||
let seed = 12345;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
|
||||
const regions = ["north", "south", "east", "west", "central"];
|
||||
function makeOrders(count: number): Order[] {
|
||||
const orders: Order[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const items: LineItem[] = [];
|
||||
const n = 1 + Math.floor(random() * 6);
|
||||
for (let j = 0; j < n; j++) {
|
||||
items.push({
|
||||
sku: "sku-" + Math.floor(random() * 500),
|
||||
quantity: 1 + Math.floor(random() * 9),
|
||||
price: Math.round(random() * 10000) / 100,
|
||||
});
|
||||
}
|
||||
orders.push({
|
||||
id: i,
|
||||
customer: "customer-" + Math.floor(random() * 2000),
|
||||
region: regions[i % regions.length]!,
|
||||
priority: random() < 0.2,
|
||||
items,
|
||||
tags: random() < 0.5 ? ["web"] : ["store", "promo"],
|
||||
});
|
||||
}
|
||||
return orders;
|
||||
}
|
||||
|
||||
function summarize(orders: Order[]): Summary[] {
|
||||
const byRegion = new Map<string, RegionTotals>();
|
||||
for (const order of orders) {
|
||||
let entry = byRegion.get(order.region);
|
||||
if (entry === undefined) {
|
||||
entry = { orders: 0, revenue: 0, skus: new Map<string, number>() };
|
||||
byRegion.set(order.region, entry);
|
||||
}
|
||||
entry.orders++;
|
||||
for (const item of order.items) {
|
||||
entry.revenue += item.quantity * item.price;
|
||||
entry.skus.set(item.sku, (entry.skus.get(item.sku) ?? 0) + item.quantity);
|
||||
}
|
||||
}
|
||||
const result: Summary[] = [];
|
||||
for (const [region, entry] of byRegion) {
|
||||
let topSku = "";
|
||||
let topCount = -1;
|
||||
for (const [sku, count] of entry.skus) {
|
||||
if (count > topCount || (count === topCount && sku < topSku)) {
|
||||
topSku = sku;
|
||||
topCount = count;
|
||||
}
|
||||
}
|
||||
result.push({ region, orders: entry.orders, revenue: Math.round(entry.revenue * 100) / 100, topSku });
|
||||
}
|
||||
result.sort((a, b) => (a.region < b.region ? -1 : a.region > b.region ? 1 : 0));
|
||||
return result;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const orders = makeOrders(Math.floor(4000 * scale));
|
||||
let checksum = 0;
|
||||
let last = "";
|
||||
for (let round = 0; round < 6; round++) {
|
||||
const text = JSON.stringify(orders);
|
||||
checksum += text.length;
|
||||
const parsed = JSON.parse(text) as Order[];
|
||||
const summary = summarize(parsed);
|
||||
last = JSON.stringify(summary);
|
||||
checksum += last.length;
|
||||
}
|
||||
console.log(last);
|
||||
console.log("checksum", checksum);
|
||||
@@ -0,0 +1,74 @@
|
||||
// Integer- and float-heavy kernels: sieve, hashing, matrix multiply on
|
||||
// number[][], histogram on plain arrays, and a Float64Array smoothing pass.
|
||||
function sieve(limit: number): number {
|
||||
const composite: boolean[] = new Array<boolean>(limit + 1).fill(false);
|
||||
let count = 0;
|
||||
for (let i = 2; i <= limit; i++) {
|
||||
if (composite[i]) continue;
|
||||
count++;
|
||||
for (let j = i * i; j <= limit; j += i) composite[j] = true;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
function fnv(values: number[]): number {
|
||||
let h = 2166136261;
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
h ^= values[i]! & 0xff;
|
||||
h = Math.imul(h, 16777619) >>> 0;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
function matmul(a: number[][], b: number[][], n: number): number[][] {
|
||||
const c: number[][] = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const row: number[] = new Array<number>(n).fill(0);
|
||||
const ai = a[i]!;
|
||||
for (let k = 0; k < n; k++) {
|
||||
const aik = ai[k]!;
|
||||
const bk = b[k]!;
|
||||
for (let j = 0; j < n; j++) row[j] = row[j]! + aik * bk[j]!;
|
||||
}
|
||||
c.push(row);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const primes = sieve(Math.floor(2000000 * scale));
|
||||
const data: number[] = [];
|
||||
let x = 1;
|
||||
for (let i = 0; i < Math.floor(1000000 * scale); i++) {
|
||||
x = (Math.imul(x, 48271) + 11) & 0x7fffffff;
|
||||
data.push(x % 1000);
|
||||
}
|
||||
let hash = 0;
|
||||
for (let r = 0; r < 4; r++) hash = (hash + fnv(data)) >>> 0;
|
||||
const histogram: number[] = new Array<number>(100).fill(0);
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const bucket = data[i]! % 100;
|
||||
histogram[bucket] = histogram[bucket]! + 1;
|
||||
}
|
||||
const n = Math.floor(120 * Math.sqrt(scale));
|
||||
const a: number[][] = [];
|
||||
const b: number[][] = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const ra: number[] = [];
|
||||
const rb: number[] = [];
|
||||
for (let j = 0; j < n; j++) {
|
||||
ra.push(((i * j) % 7) - 3);
|
||||
rb.push(((i + j) % 5) - 2);
|
||||
}
|
||||
a.push(ra);
|
||||
b.push(rb);
|
||||
}
|
||||
const c = matmul(a, b, n);
|
||||
let trace = 0;
|
||||
for (let i = 0; i < n; i++) trace += c[i]![i]!;
|
||||
const signal = new Float64Array(data.length);
|
||||
for (let i = 0; i < data.length; i++) signal[i] = data[i]! / 7;
|
||||
let smooth = 0;
|
||||
for (let pass = 0; pass < 3; pass++) {
|
||||
for (let i = 1; i < signal.length - 1; i++) signal[i] = (signal[i - 1]! + signal[i]! * 2 + signal[i + 1]!) / 4;
|
||||
}
|
||||
for (let i = 0; i < signal.length; i += 1000) smooth += signal[i]!;
|
||||
console.log("primes", primes, "hash", hash, "hist", histogram[0], histogram[99], "trace", trace, "smooth", smooth.toFixed(6));
|
||||
@@ -0,0 +1,80 @@
|
||||
// Business-record processing: multi-key sorts with comparator closures,
|
||||
// group-by into Maps, and reductions over the groups.
|
||||
interface Employee {
|
||||
id: number;
|
||||
name: string;
|
||||
department: string;
|
||||
level: number;
|
||||
salary: number;
|
||||
startYear: number;
|
||||
}
|
||||
|
||||
let seed = 2024;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
const departments = ["eng", "sales", "support", "finance", "legal", "design", "ops"];
|
||||
const first = ["ana", "bo", "cy", "dee", "eli", "fay", "gus", "hal", "ivy", "jon"];
|
||||
const last = ["smith", "jones", "lee", "khan", "garcia", "chen", "novak", "silva"];
|
||||
|
||||
function makeEmployees(count: number): Employee[] {
|
||||
const result: Employee[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
result.push({
|
||||
id: i,
|
||||
name: first[Math.floor(random() * first.length)]! + " " + last[Math.floor(random() * last.length)]!,
|
||||
department: departments[Math.floor(random() * departments.length)]!,
|
||||
level: 1 + Math.floor(random() * 7),
|
||||
salary: 40000 + Math.floor(random() * 160000),
|
||||
startYear: 1995 + Math.floor(random() * 30),
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function compareBy(keys: string[]): (a: Employee, b: Employee) => number {
|
||||
return (a, b) => {
|
||||
for (const key of keys) {
|
||||
let d = 0;
|
||||
if (key === "department") d = a.department < b.department ? -1 : a.department > b.department ? 1 : 0;
|
||||
else if (key === "level") d = b.level - a.level;
|
||||
else if (key === "salary") d = b.salary - a.salary;
|
||||
else if (key === "name") d = a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
|
||||
else d = a.id - b.id;
|
||||
if (d !== 0) return d;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const employees = makeEmployees(Math.floor(60000 * scale));
|
||||
let checksum = 0;
|
||||
const orders = [
|
||||
["department", "level", "salary", "id"],
|
||||
["name", "id"],
|
||||
["salary", "id"],
|
||||
["level", "name", "id"],
|
||||
];
|
||||
for (const order of orders) {
|
||||
const sorted = employees.slice().sort(compareBy(order));
|
||||
for (let i = 0; i < sorted.length; i += 997) checksum = (checksum * 31 + sorted[i]!.id) % 1000000007;
|
||||
}
|
||||
const groups = new Map<string, Employee[]>();
|
||||
for (const e of employees) {
|
||||
const key = e.department + "/" + e.level;
|
||||
const group = groups.get(key);
|
||||
if (group === undefined) groups.set(key, [e]);
|
||||
else group.push(e);
|
||||
}
|
||||
const lines: string[] = [];
|
||||
for (const [key, group] of groups) {
|
||||
const total = group.reduce((sum, e) => sum + e.salary, 0);
|
||||
const veterans = group.filter((e) => e.startYear < 2010).length;
|
||||
const top = group.reduce((best, e) => (e.salary > best.salary ? e : best), group[0]!);
|
||||
lines.push(`${key} n=${group.length} avg=${Math.round(total / group.length)} vets=${veterans} top=${top.id}`);
|
||||
}
|
||||
lines.sort();
|
||||
for (const line of lines.slice(0, 12)) console.log(line);
|
||||
console.log("groups", lines.length, "checksum", checksum);
|
||||
@@ -0,0 +1,49 @@
|
||||
// Regex-driven log processing: match with capture groups, test, and replace.
|
||||
let seed = 99;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
|
||||
const methods = ["GET", "POST", "PUT", "DELETE"];
|
||||
const paths = ["/api/users", "/api/orders", "/static/app.js", "/health", "/api/items/search"];
|
||||
function makeLines(count: number): string[] {
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const ip = `10.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}`;
|
||||
const method = methods[Math.floor(random() * methods.length)]!;
|
||||
const path = paths[Math.floor(random() * paths.length)]! + (random() < 0.3 ? "?id=" + Math.floor(random() * 1000) : "");
|
||||
const status = random() < 0.9 ? 200 : random() < 0.5 ? 404 : 500;
|
||||
const ms = Math.floor(random() * 900);
|
||||
const agent = random() < 0.5 ? "Mozilla/5.0 (Macintosh)" : "curl/8.4.0";
|
||||
lines.push(`${ip} - - [07/Oct/2026:12:${String(i % 60).padStart(2, "0")}:00 +0000] "${method} ${path} HTTP/1.1" ${status} ${ms} "${agent}"`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
const linePattern = /^(\d+\.\d+\.\d+\.\d+) - - \[([^\]]+)\] "(\w+) ([^ ?"]+)(?:\?([^ "]*))? HTTP\/1\.1" (\d{3}) (\d+) "([^"]*)"$/;
|
||||
const botPattern = /curl|bot|spider/i;
|
||||
const digits = /\d+/g;
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const lines = makeLines(Math.floor(30000 * scale));
|
||||
const counts = new Map<string, number>();
|
||||
let errors = 0;
|
||||
let bots = 0;
|
||||
let totalMs = 0;
|
||||
let redactedLength = 0;
|
||||
for (let round = 0; round < 3; round++) {
|
||||
for (const line of lines) {
|
||||
const m = linePattern.exec(line);
|
||||
if (m === null) continue;
|
||||
const key = m[3]! + " " + m[4]!;
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
if (m[6] !== "200") errors++;
|
||||
if (botPattern.test(m[8]!)) bots++;
|
||||
totalMs += Number(m[7]);
|
||||
redactedLength += line.replace(digits, "#").length;
|
||||
}
|
||||
}
|
||||
const keys = [...counts.keys()].sort();
|
||||
for (const key of keys) console.log(key, counts.get(key));
|
||||
console.log("errors", errors, "bots", bots, "ms", totalMs, "redacted", redactedLength);
|
||||
@@ -0,0 +1,73 @@
|
||||
// Server-side HTML rendering: template literals, escaping, joins, and
|
||||
// conditional fragments over a list of typed view models.
|
||||
interface Product {
|
||||
id: number;
|
||||
title: string;
|
||||
description: string;
|
||||
price: number;
|
||||
tags: string[];
|
||||
inStock: boolean;
|
||||
rating: number;
|
||||
}
|
||||
|
||||
let seed = 31337;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
const words = ["fast", "durable", "<b>bold</b>", "eco", '"quoted"', "light", "smart", "R&D", "classic", "modern"];
|
||||
function sentence(n: number): string {
|
||||
const parts: string[] = [];
|
||||
for (let i = 0; i < n; i++) parts.push(words[Math.floor(random() * words.length)]!);
|
||||
return parts.join(" ");
|
||||
}
|
||||
function makeProducts(count: number): Product[] {
|
||||
const products: Product[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
products.push({
|
||||
id: i,
|
||||
title: sentence(3),
|
||||
description: sentence(20),
|
||||
price: Math.round(random() * 100000) / 100,
|
||||
tags: [words[i % words.length]!, words[(i * 7) % words.length]!],
|
||||
inStock: random() < 0.8,
|
||||
rating: Math.round(random() * 50) / 10,
|
||||
});
|
||||
}
|
||||
return products;
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
||||
}
|
||||
function stars(rating: number): string {
|
||||
let out = "";
|
||||
for (let i = 1; i <= 5; i++) out += i <= Math.round(rating) ? "*" : "-";
|
||||
return out;
|
||||
}
|
||||
function renderProduct(p: Product): string {
|
||||
const tags = p.tags.map((t) => `<li class="tag">${escapeHtml(t)}</li>`).join("");
|
||||
return `<article id="p-${p.id}" class="${p.inStock ? "in" : "out"}">
|
||||
<h2>${escapeHtml(p.title)}</h2>
|
||||
<p>${escapeHtml(p.description)}</p>
|
||||
<span class="price">$${p.price.toFixed(2)}</span>
|
||||
<span class="rating" title="${p.rating}">${stars(p.rating)}</span>
|
||||
<ul>${tags}</ul>
|
||||
${p.inStock ? "<button>Add to cart</button>" : "<em>Sold out</em>"}
|
||||
</article>`;
|
||||
}
|
||||
function renderPage(products: Product[], page: number): string {
|
||||
const body = products.map(renderProduct).join("\n");
|
||||
return `<!doctype html><html><head><title>Page ${page}</title></head><body>${body}</body></html>`;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const products = makeProducts(Math.floor(2000 * scale));
|
||||
let bytes = 0;
|
||||
let hash = 0;
|
||||
for (let page = 0; page < 10; page++) {
|
||||
const html = renderPage(products, page);
|
||||
bytes += html.length;
|
||||
for (let i = 0; i < html.length; i += 101) hash = (hash * 33 + html.charCodeAt(i)) % 2147483647;
|
||||
}
|
||||
console.log("bytes", bytes, "hash", hash);
|
||||
@@ -0,0 +1,76 @@
|
||||
// Validation layers that throw and catch errors for a fraction of inputs,
|
||||
// with deep call chains of small functions on the success path.
|
||||
class ValidationError extends Error {
|
||||
readonly field: string;
|
||||
constructor(field: string, message: string) {
|
||||
super(message);
|
||||
this.field = field;
|
||||
}
|
||||
}
|
||||
interface Signup {
|
||||
email: string;
|
||||
age: number;
|
||||
name: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
let seed = 8080;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
function makeSignup(i: number): Signup {
|
||||
const bad = random();
|
||||
return {
|
||||
email: bad < 0.05 ? "broken" : `user${i}@example.com`,
|
||||
age: bad > 0.97 ? -1 : 18 + Math.floor(random() * 60),
|
||||
name: bad > 0.95 && bad <= 0.97 ? " " : "Name " + i,
|
||||
password: random() < 0.03 ? "short" : "correct horse battery " + i,
|
||||
};
|
||||
}
|
||||
|
||||
function requireText(field: string, value: string, min: number): string {
|
||||
if (value.length < min) throw new ValidationError(field, `${field} must have at least ${min} characters`);
|
||||
return value;
|
||||
}
|
||||
function requireEmail(value: string): string {
|
||||
const at = value.indexOf("@");
|
||||
if (at <= 0 || value.indexOf(".", at) < 0) throw new ValidationError("email", "invalid email");
|
||||
return value.toLowerCase();
|
||||
}
|
||||
function requireRange(field: string, value: number, min: number, max: number): number {
|
||||
if (value < min || value > max) throw new ValidationError(field, `${field} out of range`);
|
||||
return value;
|
||||
}
|
||||
function normalize(input: Signup): Signup {
|
||||
return {
|
||||
email: requireEmail(input.email),
|
||||
age: requireRange("age", input.age, 13, 120),
|
||||
name: requireText("name", input.name.trim(), 1),
|
||||
password: requireText("password", input.password, 8),
|
||||
};
|
||||
}
|
||||
function score(s: Signup): number {
|
||||
return s.email.length + s.age + s.name.length + s.password.length;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const inputs: Signup[] = [];
|
||||
for (let i = 0; i < Math.floor(200000 * scale); i++) inputs.push(makeSignup(i));
|
||||
const failures = new Map<string, number>();
|
||||
let accepted = 0;
|
||||
let total = 0;
|
||||
for (let round = 0; round < 3; round++) {
|
||||
for (const input of inputs) {
|
||||
try {
|
||||
const s = normalize(input);
|
||||
accepted++;
|
||||
total += score(s);
|
||||
} catch (error) {
|
||||
if (error instanceof ValidationError) failures.set(error.field, (failures.get(error.field) ?? 0) + 1);
|
||||
else throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("accepted", accepted, "total", total);
|
||||
for (const key of [...failures.keys()].sort()) console.log(key, failures.get(key));
|
||||
@@ -0,0 +1,61 @@
|
||||
// Map/Set-heavy text and graph processing: word frequencies, bigrams,
|
||||
// and breadth-first search over a string-keyed adjacency map.
|
||||
let seed = 555;
|
||||
function random(): number {
|
||||
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
||||
return seed / 4294967296;
|
||||
}
|
||||
const syllables = ["ka", "lo", "mi", "ne", "ru", "sa", "to", "vi", "ze", "po", "qu", "ex"];
|
||||
function word(): string {
|
||||
const n = 1 + Math.floor(random() * 3);
|
||||
let w = "";
|
||||
for (let i = 0; i < n; i++) w += syllables[Math.floor(random() * syllables.length)]!;
|
||||
return w;
|
||||
}
|
||||
|
||||
const scale = Number(process.argv[2] ?? "1");
|
||||
const text: string[] = [];
|
||||
for (let i = 0; i < Math.floor(300000 * scale); i++) text.push(word());
|
||||
|
||||
const freq = new Map<string, number>();
|
||||
for (const w of text) freq.set(w, (freq.get(w) ?? 0) + 1);
|
||||
const bigrams = new Map<string, number>();
|
||||
for (let i = 1; i < text.length; i++) {
|
||||
const key = text[i - 1]! + " " + text[i]!;
|
||||
bigrams.set(key, (bigrams.get(key) ?? 0) + 1);
|
||||
}
|
||||
const top = [...freq.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5);
|
||||
console.log("words", freq.size, "bigrams", bigrams.size, "top", top.map((e) => e[0] + ":" + e[1]).join(","));
|
||||
|
||||
const graph = new Map<string, string[]>();
|
||||
for (let i = 1; i < text.length; i += 3) {
|
||||
const from = text[i - 1]!;
|
||||
const to = text[i]!;
|
||||
const edges = graph.get(from);
|
||||
if (edges === undefined) graph.set(from, [to]);
|
||||
else if (edges.length < 8) edges.push(to);
|
||||
}
|
||||
let reachable = 0;
|
||||
let depthSum = 0;
|
||||
const starts = [...graph.keys()].sort().slice(0, 60);
|
||||
for (const start of starts) {
|
||||
const seen = new Set<string>([start]);
|
||||
let frontier = [start];
|
||||
let depth = 0;
|
||||
while (frontier.length > 0) {
|
||||
const next: string[] = [];
|
||||
for (const node of frontier) {
|
||||
for (const neighbor of graph.get(node) ?? []) {
|
||||
if (!seen.has(neighbor)) {
|
||||
seen.add(neighbor);
|
||||
next.push(neighbor);
|
||||
}
|
||||
}
|
||||
}
|
||||
frontier = next;
|
||||
depth++;
|
||||
}
|
||||
reachable += seen.size;
|
||||
depthSum += depth;
|
||||
}
|
||||
console.log("reachable", reachable, "depth", depthSum);
|
||||
@@ -9,6 +9,7 @@
|
||||
"build:fresh": "rm -rf packages/compiler/dist packages/cli/dist node_modules/.cache/scriptc-tsc && pnpm build",
|
||||
"build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-cli.mts",
|
||||
"bench:builds": "node scripts/bench-builds.mjs",
|
||||
"bench:runtime": "node scripts/bench-runtime.mjs",
|
||||
"bench:ast-access": "node scripts/bench-ast-access.mjs",
|
||||
"bench:backend-analysis": "node scripts/bench-backend-analysis.mjs",
|
||||
"test": "vitest run",
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
/* Runtime performance A/B harness for compiled programs.
|
||||
*
|
||||
* Builds every workload in benchmarks/runtime/workloads.json with one or two
|
||||
* scriptc checkouts, verifies each executable against Node byte-for-byte
|
||||
* (stdout, stderr, exit status), then times interleaved launches and reports
|
||||
* per-workload medians with bootstrap confidence intervals for the
|
||||
* candidate/baseline ratio. Executables that diverge from Node are never
|
||||
* timed. A per-host advisory lock serializes measurements so concurrent
|
||||
* agents on one machine do not disturb each other's numbers. */
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
const repoRoot = fileURLToPath(new URL("..", import.meta.url));
|
||||
const suiteRoot = join(repoRoot, "benchmarks/runtime");
|
||||
|
||||
const { values } = parseArgs({
|
||||
allowNegative: true,
|
||||
options: {
|
||||
candidate: { type: "string", default: repoRoot },
|
||||
baseline: { type: "string" },
|
||||
workloads: { type: "string" },
|
||||
runs: { type: "string", default: "15" },
|
||||
warmup: { type: "string", default: "2" },
|
||||
timeout: { type: "string", default: "120" },
|
||||
json: { type: "string" },
|
||||
"no-lock": { type: "boolean", default: false },
|
||||
node: { type: "boolean", default: true },
|
||||
"keep": { type: "boolean", default: false },
|
||||
help: { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
if (values.help) {
|
||||
console.log(`Usage: node scripts/bench-runtime.mjs [--candidate=<checkout>] [--baseline=<checkout>]
|
||||
[--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--json=<file>] [--no-lock] [--no-node] [--keep]
|
||||
|
||||
Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native
|
||||
artifacts for this host. Without --baseline, reports absolute times only. Node is
|
||||
timed in the same interleaved loop as the reference to beat (--no-node skips it).`);
|
||||
process.exit(0);
|
||||
}
|
||||
const runs = Number(values.runs);
|
||||
const warmup = Number(values.warmup);
|
||||
const timeoutMs = Number(values.timeout) * 1000;
|
||||
assert.ok(Number.isInteger(runs) && runs >= 3 && runs <= 200, "--runs must be 3..200");
|
||||
assert.ok(Number.isInteger(warmup) && warmup >= 0 && warmup <= 20, "--warmup must be 0..20");
|
||||
|
||||
const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8"));
|
||||
const selected = values.workloads ? new Set(values.workloads.split(",")) : null;
|
||||
const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name));
|
||||
if (selected) {
|
||||
for (const name of selected)
|
||||
assert.ok(
|
||||
manifest.workloads.some((w) => w.name === name),
|
||||
`unknown workload ${name}`,
|
||||
);
|
||||
}
|
||||
|
||||
const work = mkdtempSync(
|
||||
join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-runtime-"),
|
||||
);
|
||||
|
||||
/* ── inputs for file-processing applications ──────────────────────────── */
|
||||
function generateInput(kind) {
|
||||
const path = join(work, `${kind}.input`);
|
||||
if (kind === "log-summary") {
|
||||
writeFileSync(
|
||||
path,
|
||||
Array.from(
|
||||
{ length: 300_000 },
|
||||
(_, i) =>
|
||||
`/route/${i % 37}?request=${i} ${i % 13 === 0 ? 500 : 200} ${i % 250} ${200 + (i % 4000)}`,
|
||||
).join("\n") + "\n",
|
||||
);
|
||||
} else if (kind === "inventory-report") {
|
||||
writeFileSync(
|
||||
path,
|
||||
Array.from({ length: 300_000 }, (_, i) =>
|
||||
["sku-" + i, "category-" + (i % 17), "product " + i, i % 40, 199 + (i % 5000)].join("\t"),
|
||||
).join("\n") + "\n",
|
||||
);
|
||||
} else throw new Error(`unknown input kind ${kind}`);
|
||||
return path;
|
||||
}
|
||||
|
||||
/* ── advisory lock ─────────────────────────────────────────────────────── */
|
||||
const lockDir = join(
|
||||
process.platform === "win32" ? tmpdir() : "/tmp",
|
||||
"scriptc-bench-runtime.lock",
|
||||
);
|
||||
function acquireLock() {
|
||||
const started = Date.now();
|
||||
let announced = false;
|
||||
for (;;) {
|
||||
try {
|
||||
mkdirSync(lockDir);
|
||||
writeFileSync(join(lockDir, "pid"), String(process.pid));
|
||||
return;
|
||||
} catch (error) {
|
||||
if (error.code !== "EEXIST") throw error;
|
||||
let owner = NaN;
|
||||
try {
|
||||
owner = Number(readFileSync(join(lockDir, "pid"), "utf8"));
|
||||
} catch {}
|
||||
let alive = false;
|
||||
if (Number.isInteger(owner)) {
|
||||
try {
|
||||
process.kill(owner, 0);
|
||||
alive = true;
|
||||
} catch {}
|
||||
}
|
||||
if (!alive && Date.now() - started > 2000) {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
continue;
|
||||
}
|
||||
if (!announced) {
|
||||
process.stderr.write(`waiting for benchmark lock held by pid ${owner}\n`);
|
||||
announced = true;
|
||||
}
|
||||
spawnSync("sleep", ["2"]);
|
||||
}
|
||||
}
|
||||
}
|
||||
function releaseLock() {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/* ── process helpers ───────────────────────────────────────────────────── */
|
||||
function run(command, args, options = {}) {
|
||||
const start = process.hrtime.bigint();
|
||||
const result = spawnSync(command, args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
timeout: options.timeout ?? timeoutMs,
|
||||
env: options.env ?? process.env,
|
||||
cwd: options.cwd,
|
||||
});
|
||||
const ms = Number(process.hrtime.bigint() - start) / 1e6;
|
||||
if (result.error && result.error.code !== "ETIMEDOUT") throw result.error;
|
||||
return {
|
||||
stdout: result.stdout ?? "",
|
||||
stderr: result.stderr ?? "",
|
||||
status: result.status,
|
||||
signal: result.signal,
|
||||
timedOut: result.error?.code === "ETIMEDOUT" || result.signal === "SIGTERM",
|
||||
ms,
|
||||
};
|
||||
}
|
||||
|
||||
function peakRss(binary, args) {
|
||||
if (process.platform !== "darwin" && process.platform !== "linux") return null;
|
||||
if (!existsSync("/usr/bin/time")) return null;
|
||||
const timeArgs =
|
||||
process.platform === "darwin" ? ["-l", binary, ...args] : ["-f", "rss_kib=%M", binary, ...args];
|
||||
const result = run("/usr/bin/time", timeArgs);
|
||||
const match =
|
||||
process.platform === "darwin"
|
||||
? /(\d+)\s+maximum resident set size/.exec(result.stderr)
|
||||
: /rss_kib=(\d+)/.exec(result.stderr);
|
||||
if (!match) return null;
|
||||
return Number(match[1]) * (process.platform === "darwin" ? 1 : 1024);
|
||||
}
|
||||
|
||||
/* ── statistics ────────────────────────────────────────────────────────── */
|
||||
function median(xs) {
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
const m = s.length >> 1;
|
||||
return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2;
|
||||
}
|
||||
function makeRandom(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
|
||||
return state / 4294967296;
|
||||
};
|
||||
}
|
||||
function bootstrapRatio(candidate, baseline, resamples = 2000) {
|
||||
const random = makeRandom(0x5eed);
|
||||
const ratios = [];
|
||||
const pick = (xs) =>
|
||||
Array.from({ length: xs.length }, () => xs[Math.floor(random() * xs.length)]);
|
||||
for (let i = 0; i < resamples; i++) ratios.push(median(pick(candidate)) / median(pick(baseline)));
|
||||
ratios.sort((a, b) => a - b);
|
||||
return {
|
||||
low: ratios[Math.floor(resamples * 0.025)],
|
||||
high: ratios[Math.floor(resamples * 0.975)],
|
||||
};
|
||||
}
|
||||
|
||||
/* ── build ─────────────────────────────────────────────────────────────── */
|
||||
function cliFor(root) {
|
||||
const cli = join(resolve(root), "packages/cli/dist/bootstrap.js");
|
||||
statSync(cli);
|
||||
return cli;
|
||||
}
|
||||
function revision(root) {
|
||||
const r = run("git", ["-C", root, "rev-parse", "--short", "HEAD"], { timeout: 10_000 });
|
||||
const dirty = run("git", ["-C", root, "status", "--porcelain", "--untracked-files=no"], {
|
||||
timeout: 10_000,
|
||||
});
|
||||
return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : "");
|
||||
}
|
||||
function build(label, root, workload) {
|
||||
const cli = cliFor(root);
|
||||
const out = join(work, label, workload.name);
|
||||
mkdirSync(dirname(out), { recursive: true });
|
||||
const cache = join(work, label, ".cache");
|
||||
mkdirSync(cache, { recursive: true });
|
||||
const env = { ...process.env, SCRIPTC_CACHE_DIR: cache };
|
||||
delete env.SCRIPTC_NO_CACHE;
|
||||
delete env.SCRIPTC_TIMING;
|
||||
const entry = join(suiteRoot, workload.entry);
|
||||
const result = run(process.execPath, [cli, "build", entry, "--optimization=release", "-o", out], {
|
||||
env,
|
||||
timeout: 900_000,
|
||||
});
|
||||
if (result.status !== 0)
|
||||
return {
|
||||
ok: false,
|
||||
error: (result.stderr || result.stdout).slice(0, 4000),
|
||||
buildMs: result.ms,
|
||||
};
|
||||
return { ok: true, binary: out, bytes: statSync(out).size, buildMs: result.ms };
|
||||
}
|
||||
|
||||
/* ── main ──────────────────────────────────────────────────────────────── */
|
||||
const contenders = [
|
||||
...(values.baseline ? [{ label: "baseline", root: resolve(values.baseline) }] : []),
|
||||
{ label: "candidate", root: resolve(values.candidate) },
|
||||
];
|
||||
const report = {
|
||||
schema: 1,
|
||||
date: new Date().toISOString(),
|
||||
host: { platform: process.platform, arch: process.arch, node: process.version },
|
||||
runs,
|
||||
contenders: contenders.map((c) => ({ ...c, revision: revision(c.root) })),
|
||||
workloads: [],
|
||||
};
|
||||
|
||||
try {
|
||||
const inputs = new Map();
|
||||
for (const w of workloads)
|
||||
if (w.input && !inputs.has(w.input)) inputs.set(w.input, generateInput(w.input));
|
||||
|
||||
// Builds and the Node oracle run outside the lock: they do not need a
|
||||
// quiet machine, and holding the lock while compiling would serialize
|
||||
// every agent's slowest phase.
|
||||
const prepared = [];
|
||||
for (const w of workloads) {
|
||||
const args = w.args.map((a) => (a === "{input}" ? inputs.get(w.input) : a));
|
||||
// The application workloads are ESM without a package.json "type"; Node's
|
||||
// reparse warning is an oracle artifact, not program output.
|
||||
const oracle = run(process.execPath, [
|
||||
"--disable-warning=MODULE_TYPELESS_PACKAGE_JSON",
|
||||
join(suiteRoot, w.entry),
|
||||
...args,
|
||||
]);
|
||||
const entry = { name: w.name, args, results: {} };
|
||||
if (oracle.status !== 0 || oracle.timedOut) {
|
||||
entry.error = `node oracle failed (status ${oracle.status}): ${oracle.stderr.slice(0, 2000)}`;
|
||||
prepared.push(entry);
|
||||
continue;
|
||||
}
|
||||
for (const c of contenders) {
|
||||
const built = build(c.label, c.root, w);
|
||||
const result = { build_ms: Math.round(built.buildMs) };
|
||||
entry.results[c.label] = result;
|
||||
if (!built.ok) {
|
||||
result.status = "build-failed";
|
||||
result.error = built.error;
|
||||
continue;
|
||||
}
|
||||
result.bytes = built.bytes;
|
||||
result.binary = built.binary;
|
||||
const check = run(built.binary, args);
|
||||
if (check.timedOut) {
|
||||
result.status = "timeout";
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
check.stdout !== oracle.stdout ||
|
||||
check.stderr !== oracle.stderr ||
|
||||
check.status !== oracle.status
|
||||
) {
|
||||
result.status = "mismatch";
|
||||
result.error = `expected status ${oracle.status}, got ${check.status}; stdout ${check.stdout === oracle.stdout ? "matches" : "differs"}; stderr ${check.stderr === oracle.stderr ? "matches" : "differs"}`;
|
||||
continue;
|
||||
}
|
||||
result.status = "ok";
|
||||
result.samples = [];
|
||||
}
|
||||
entry.node_ms = Math.round(oracle.ms);
|
||||
entry.nodeCommand = [
|
||||
"--disable-warning=MODULE_TYPELESS_PACKAGE_JSON",
|
||||
join(suiteRoot, w.entry),
|
||||
...args,
|
||||
];
|
||||
prepared.push(entry);
|
||||
process.stderr.write(`prepared ${w.name}\n`);
|
||||
}
|
||||
|
||||
if (!values["no-lock"]) acquireLock();
|
||||
try {
|
||||
for (const entry of prepared) {
|
||||
const live = contenders.filter((c) => entry.results[c.label]?.status === "ok");
|
||||
if (live.length === 0) continue;
|
||||
for (const c of live)
|
||||
for (let i = 0; i < warmup; i++) run(entry.results[c.label].binary, entry.args);
|
||||
// Node is the reference the project aims to beat, so it is timed in the
|
||||
// same interleaved loop (whole-process wall time, startup included).
|
||||
const nodeSamples = [];
|
||||
if (values.node) run(process.execPath, entry.nodeCommand);
|
||||
const random = makeRandom(entry.name.length * 7919);
|
||||
for (let r = 0; r < runs; r++) {
|
||||
const order = [...live];
|
||||
if (order.length === 2 && random() < 0.5) order.reverse();
|
||||
for (const c of order) {
|
||||
const result = entry.results[c.label];
|
||||
const sample = run(result.binary, entry.args);
|
||||
if (sample.timedOut || sample.status !== 0) {
|
||||
result.status = sample.timedOut ? "timeout" : "unstable-exit";
|
||||
break;
|
||||
}
|
||||
result.samples.push(Math.round(sample.ms * 100) / 100);
|
||||
}
|
||||
if (values.node) nodeSamples.push(run(process.execPath, entry.nodeCommand).ms);
|
||||
}
|
||||
if (nodeSamples.length > 0) entry.node_ms = Math.round(median(nodeSamples) * 100) / 100;
|
||||
const reference = entry.results.candidate;
|
||||
if (reference?.status === "ok" && reference.samples.length > 0 && nodeSamples.length > 0)
|
||||
entry.vs_node = Math.round((median(reference.samples) / entry.node_ms) * 1000) / 1000;
|
||||
for (const c of live) {
|
||||
const result = entry.results[c.label];
|
||||
if (result.status !== "ok") continue;
|
||||
result.median_ms = Math.round(median(result.samples) * 100) / 100;
|
||||
result.peak_rss_bytes = peakRss(result.binary, entry.args);
|
||||
}
|
||||
const base = entry.results.baseline;
|
||||
const cand = entry.results.candidate;
|
||||
if (base?.status === "ok" && cand?.status === "ok") {
|
||||
const ci = bootstrapRatio(cand.samples, base.samples);
|
||||
entry.ratio = Math.round((cand.median_ms / base.median_ms) * 10000) / 10000;
|
||||
entry.ci95 = [Math.round(ci.low * 10000) / 10000, Math.round(ci.high * 10000) / 10000];
|
||||
entry.verdict = ci.high < 1 ? "faster" : ci.low > 1 ? "slower" : "neutral";
|
||||
entry.size_ratio = Math.round((cand.bytes / base.bytes) * 10000) / 10000;
|
||||
}
|
||||
process.stderr.write(`measured ${entry.name}\n`);
|
||||
}
|
||||
} finally {
|
||||
if (!values["no-lock"]) releaseLock();
|
||||
}
|
||||
for (const entry of prepared) {
|
||||
for (const result of Object.values(entry.results)) delete result.binary;
|
||||
delete entry.nodeCommand;
|
||||
report.workloads.push(entry);
|
||||
}
|
||||
const vsNode = report.workloads.filter((w) => w.vs_node !== undefined).map((w) => w.vs_node);
|
||||
if (vsNode.length > 0)
|
||||
report.geomean_vs_node =
|
||||
Math.round(Math.exp(vsNode.reduce((s, r) => s + Math.log(r), 0) / vsNode.length) * 1000) /
|
||||
1000;
|
||||
const ratios = report.workloads.filter((w) => w.ratio !== undefined).map((w) => w.ratio);
|
||||
if (ratios.length > 0)
|
||||
report.geomean_ratio =
|
||||
Math.round(Math.exp(ratios.reduce((s, r) => s + Math.log(r), 0) / ratios.length) * 10000) /
|
||||
10000;
|
||||
} finally {
|
||||
if (!values.keep) rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n");
|
||||
|
||||
/* ── summary table ─────────────────────────────────────────────────────── */
|
||||
const pct = (r) => `${r < 1 ? "" : "+"}${((r - 1) * 100).toFixed(1)}%`;
|
||||
const lines = [];
|
||||
const xNode = (w) => (w.vs_node === undefined ? "-" : `${w.vs_node.toFixed(2)}x`);
|
||||
if (values.baseline) {
|
||||
lines.push(
|
||||
"| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms | candidate/node |",
|
||||
);
|
||||
lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: |");
|
||||
} else {
|
||||
lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms | scriptc/node |");
|
||||
lines.push("| --- | ---: | --- | ---: | ---: | ---: | ---: |");
|
||||
}
|
||||
for (const w of report.workloads) {
|
||||
const b = w.results.baseline;
|
||||
const c = w.results.candidate;
|
||||
if (w.error) {
|
||||
lines.push(`| ${w.name} | ${w.error.split("\n")[0]} |`);
|
||||
} else if (values.baseline) {
|
||||
const show = (r) => (r?.status === "ok" ? r.median_ms.toFixed(1) : (r?.status ?? "-"));
|
||||
lines.push(
|
||||
`| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} | ${xNode(w)} |`,
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
`| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} | ${xNode(w)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (report.geomean_ratio !== undefined)
|
||||
lines.push("", `geomean change: ${pct(report.geomean_ratio)}`);
|
||||
if (report.geomean_vs_node !== undefined)
|
||||
lines.push(
|
||||
`geomean candidate/node: ${report.geomean_vs_node.toFixed(3)}x (lower is better; goal: well below 1)`,
|
||||
);
|
||||
console.log(lines.join("\n"));
|
||||
const broken = report.workloads.some(
|
||||
(w) => w.error || Object.values(w.results).some((r) => r.status !== "ok"),
|
||||
);
|
||||
process.exitCode = broken ? 1 : 0;
|
||||
Reference in New Issue
Block a user