From 10b246d6047d2586f177d40c2ba47e28de64fcd6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 7 Oct 2026 17:55:27 -0700 Subject: [PATCH 1/3] Add a runtime benchmark suite with Node-verified A/B timing Fourteen application-shaped workloads (JSON, regex, parsing and class dispatch, sorting, rendering, collection pipelines, async fan-out, Map/Set graphs, allocation, numeric kernels, exceptions, CSV, and the two multi-module build benchmark apps on large inputs). bench:runtime builds each workload with one or two checkouts, refuses to time any executable whose stdout, stderr, or exit status differs from Node, interleaves baseline and candidate launches, and reports medians with bootstrap 95% confidence intervals, binary size, and peak RSS. --- benchmarks/runtime/README.md | 27 ++ benchmarks/runtime/workloads.json | 108 +++++ benchmarks/runtime/workloads/alloc-trees.ts | 46 +++ benchmarks/runtime/workloads/ast-interp.ts | 199 +++++++++ .../runtime/workloads/async-pipeline.ts | 47 +++ benchmarks/runtime/workloads/csv-numbers.ts | 43 ++ .../runtime/workloads/functional-pipeline.ts | 56 +++ benchmarks/runtime/workloads/json-records.ts | 102 +++++ .../runtime/workloads/numeric-kernels.ts | 74 ++++ benchmarks/runtime/workloads/records-sort.ts | 80 ++++ benchmarks/runtime/workloads/regex-logs.ts | 49 +++ .../runtime/workloads/template-render.ts | 73 ++++ .../runtime/workloads/validate-errors.ts | 76 ++++ benchmarks/runtime/workloads/word-graph.ts | 61 +++ package.json | 1 + scripts/bench-runtime.mjs | 387 ++++++++++++++++++ 16 files changed, 1429 insertions(+) create mode 100644 benchmarks/runtime/README.md create mode 100644 benchmarks/runtime/workloads.json create mode 100644 benchmarks/runtime/workloads/alloc-trees.ts create mode 100644 benchmarks/runtime/workloads/ast-interp.ts create mode 100644 benchmarks/runtime/workloads/async-pipeline.ts create mode 100644 benchmarks/runtime/workloads/csv-numbers.ts create mode 100644 benchmarks/runtime/workloads/functional-pipeline.ts create mode 100644 benchmarks/runtime/workloads/json-records.ts create mode 100644 benchmarks/runtime/workloads/numeric-kernels.ts create mode 100644 benchmarks/runtime/workloads/records-sort.ts create mode 100644 benchmarks/runtime/workloads/regex-logs.ts create mode 100644 benchmarks/runtime/workloads/template-render.ts create mode 100644 benchmarks/runtime/workloads/validate-errors.ts create mode 100644 benchmarks/runtime/workloads/word-graph.ts create mode 100644 scripts/bench-runtime.mjs diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md new file mode 100644 index 000000000..385532c9d --- /dev/null +++ b/benchmarks/runtime/README.md @@ -0,0 +1,27 @@ +# Runtime benchmarks + +Application-shaped workloads for measuring the speed of compiled executables. Each workload is deterministic, generates its own input (or reads a generated file), and prints a summary that must match Node.js byte-for-byte before it is timed. + +| Workload | Exercises | +| --- | --- | +| `json-records` | `JSON.stringify`, typed `JSON.parse`, Map aggregation | +| `regex-logs` | regex `exec` with groups, `test`, global `replace` | +| `ast-interp` | tokenizer, recursive-descent parser, class dispatch | +| `records-sort` | comparator sorts, group-by, `reduce`/`filter` | +| `template-render` | template literals, escaping, `map`/`join` | +| `functional-pipeline` | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `async-pipeline` | `async`/`await`, `Promise.all` fan-out | +| `word-graph` | string-keyed Map/Set, BFS | +| `alloc-trees` | allocation-heavy trees, immutable object-spread updates | +| `numeric-kernels` | sieve, hashing, `number[][]` matmul, `Float64Array` | +| `validate-errors` | throw/catch on a fraction of inputs, small call chains | +| `csv-numbers` | `split`, `Number`, `parseInt`, `toFixed` | +| `log-summary`, `inventory-report` | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | + +Compare two checkouts (each with a built CLI and native artifacts for the host): + +```console +$ pnpm bench:runtime --baseline=../scriptc-main --candidate=. --runs=15 --json=result.json +``` + +Without `--baseline`, the runner reports absolute medians. Baseline and candidate launches are interleaved, and each workload reports the median ratio with a bootstrap 95% confidence interval. `faster`/`slower` verdicts require the whole interval on one side of 1.0. A per-host lock serializes the timed phase so concurrent runs on one machine do not distort each other. Executables that time out or diverge from Node are reported, never timed. diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json new file mode 100644 index 000000000..cba7115d0 --- /dev/null +++ b/benchmarks/runtime/workloads.json @@ -0,0 +1,108 @@ +{ + "workloads": [ + { + "name": "json-records", + "entry": "workloads/json-records.ts", + "args": [ + "4" + ] + }, + { + "name": "regex-logs", + "entry": "workloads/regex-logs.ts", + "args": [ + "1" + ] + }, + { + "name": "ast-interp", + "entry": "workloads/ast-interp.ts", + "args": [ + "5" + ] + }, + { + "name": "records-sort", + "entry": "workloads/records-sort.ts", + "args": [ + "1" + ] + }, + { + "name": "template-render", + "entry": "workloads/template-render.ts", + "args": [ + "1" + ] + }, + { + "name": "functional-pipeline", + "entry": "workloads/functional-pipeline.ts", + "args": [ + "1" + ] + }, + { + "name": "async-pipeline", + "entry": "workloads/async-pipeline.ts", + "args": [ + "1" + ] + }, + { + "name": "word-graph", + "entry": "workloads/word-graph.ts", + "args": [ + "2" + ] + }, + { + "name": "alloc-trees", + "entry": "workloads/alloc-trees.ts", + "args": [ + "1" + ] + }, + { + "name": "numeric-kernels", + "entry": "workloads/numeric-kernels.ts", + "args": [ + "0.5" + ] + }, + { + "name": "validate-errors", + "entry": "workloads/validate-errors.ts", + "args": [ + "1" + ] + }, + { + "name": "csv-numbers", + "entry": "workloads/csv-numbers.ts", + "args": [ + "1" + ] + }, + { + "name": "log-summary", + "entry": "../builds/log-summary/main.ts", + "input": "log-summary", + "args": [ + "{input}", + "--min-status", + "200", + "--limit", + "12" + ] + }, + { + "name": "inventory-report", + "entry": "../builds/inventory-report/main.ts", + "input": "inventory-report", + "args": [ + "{input}" + ] + } + ] +} diff --git a/benchmarks/runtime/workloads/alloc-trees.ts b/benchmarks/runtime/workloads/alloc-trees.ts new file mode 100644 index 000000000..16329a0d5 --- /dev/null +++ b/benchmarks/runtime/workloads/alloc-trees.ts @@ -0,0 +1,46 @@ +// Allocation-heavy code: binary trees built and discarded, plus a +// reducer that produces immutable state updates with object spread. +class TreeNode { + readonly left: TreeNode | null; + readonly right: TreeNode | null; + constructor(left: TreeNode | null, right: TreeNode | null) { + this.left = left; + this.right = right; + } + check(): number { + return 1 + (this.left === null ? 0 : this.left.check()) + (this.right === null ? 0 : this.right.check()); + } +} +function build(depth: number): TreeNode { + return depth === 0 ? new TreeNode(null, null) : new TreeNode(build(depth - 1), build(depth - 1)); +} + +interface State { + count: number; + total: number; + last: string; + history: number[]; +} +interface Action { + type: string; + amount: number; +} +function reduce(state: State, action: Action): State { + if (action.type === "add") return { ...state, count: state.count + 1, total: state.total + action.amount, last: action.type }; + if (action.type === "reset") return { ...state, total: 0, last: action.type, history: [...state.history.slice(-4), state.total] }; + return { ...state, last: action.type }; +} + +const scale = Number(process.argv[2] ?? "1"); +const maxDepth = 14; +let checks = 0; +for (let depth = 4; depth <= maxDepth; depth += 2) { + const iterations = Math.floor((1 << (maxDepth - depth + 4)) * scale); + for (let i = 0; i < iterations; i++) checks += build(depth).check(); +} +const longLived = build(maxDepth); +let state: State = { count: 0, total: 0, last: "", history: [] }; +for (let i = 0; i < Math.floor(300000 * scale); i++) { + state = reduce(state, { type: i % 97 === 0 ? "reset" : i % 3 === 0 ? "noop" : "add", amount: i % 13 }); +} +console.log("checks", checks, "long", longLived.check(), "state", state.count, state.total, state.history.join(",")); diff --git a/benchmarks/runtime/workloads/ast-interp.ts b/benchmarks/runtime/workloads/ast-interp.ts new file mode 100644 index 000000000..fb7db5d28 --- /dev/null +++ b/benchmarks/runtime/workloads/ast-interp.ts @@ -0,0 +1,199 @@ +// A tokenizer, recursive-descent parser, and class-based tree-walking +// evaluator for a small expression language with variables. +abstract class Expr { + abstract evaluate(env: Map): number; +} +class Num extends Expr { + readonly value: number; + constructor(value: number) { + super(); + this.value = value; + } + evaluate(_env: Map): number { + return this.value; + } +} +class Var extends Expr { + readonly name: string; + constructor(name: string) { + super(); + this.name = name; + } + evaluate(env: Map): number { + return env.get(this.name) ?? 0; + } +} +class Binary extends Expr { + readonly op: string; + readonly left: Expr; + readonly right: Expr; + constructor(op: string, left: Expr, right: Expr) { + super(); + this.op = op; + this.left = left; + this.right = right; + } + evaluate(env: Map): number { + const l = this.left.evaluate(env); + const r = this.right.evaluate(env); + switch (this.op) { + case "+": + return l + r; + case "-": + return l - r; + case "*": + return l * r; + case "/": + return r === 0 ? 0 : l / r; + default: + return l % (r === 0 ? 1 : r); + } + } +} +class Call extends Expr { + readonly name: string; + readonly args: Expr[]; + constructor(name: string, args: Expr[]) { + super(); + this.name = name; + this.args = args; + } + evaluate(env: Map): number { + const values = this.args.map((arg) => arg.evaluate(env)); + if (this.name === "max") return Math.max(...values); + if (this.name === "min") return Math.min(...values); + return values.reduce((a, b) => a + b, 0); + } +} + +type TokenKind = "num" | "ident" | "op" | "lparen" | "rparen" | "comma" | "end"; +interface Token { + kind: TokenKind; + text: string; +} + +function isDigit(c: number): boolean { + return (c >= 48 && c <= 57) || c === 46; +} +function isIdent(c: number): boolean { + return (c >= 97 && c <= 122) || c === 95; +} +function tokenize(source: string): Token[] { + const tokens: Token[] = []; + let i = 0; + while (i < source.length) { + const c = source.charCodeAt(i); + if (c === 32) { + i++; + } else if (c >= 48 && c <= 57) { + const start = i; + while (i < source.length && isDigit(source.charCodeAt(i))) i++; + tokens.push({ kind: "num", text: source.slice(start, i) }); + } else if (isIdent(c)) { + const start = i; + while (i < source.length && isIdent(source.charCodeAt(i))) i++; + tokens.push({ kind: "ident", text: source.slice(start, i) }); + } else if (c === 40) { + tokens.push({ kind: "lparen", text: "(" }); + i++; + } else if (c === 41) { + tokens.push({ kind: "rparen", text: ")" }); + i++; + } else if (c === 44) { + tokens.push({ kind: "comma", text: "," }); + i++; + } else { + tokens.push({ kind: "op", text: source[i]! }); + i++; + } + } + tokens.push({ kind: "end", text: "" }); + return tokens; +} + +class Parser { + private pos = 0; + private readonly tokens: Token[]; + constructor(tokens: Token[]) { + this.tokens = tokens; + } + private peek(): Token { + return this.tokens[this.pos]!; + } + private next(): Token { + return this.tokens[this.pos++]!; + } + parseExpr(): Expr { + let left = this.parseTerm(); + while (this.peek().kind === "op" && (this.peek().text === "+" || this.peek().text === "-")) { + const op = this.next().text; + left = new Binary(op, left, this.parseTerm()); + } + return left; + } + private parseTerm(): Expr { + let left = this.parseFactor(); + while (this.peek().kind === "op" && (this.peek().text === "*" || this.peek().text === "/" || this.peek().text === "%")) { + const op = this.next().text; + left = new Binary(op, left, this.parseFactor()); + } + return left; + } + private parseFactor(): Expr { + const token = this.next(); + if (token.kind === "num") return new Num(Number(token.text)); + if (token.kind === "lparen") { + const inner = this.parseExpr(); + this.next(); + return inner; + } + if (this.peek().kind === "lparen") { + this.next(); + const args: Expr[] = []; + while (this.peek().kind !== "rparen") { + args.push(this.parseExpr()); + if (this.peek().kind === "comma") this.next(); + } + this.next(); + return new Call(token.text, args); + } + return new Var(token.text); + } +} + +let seed = 7; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const names = ["alpha", "beta", "gamma", "delta", "x", "y"]; +const ops = ["+", "-", "*", "/", "%"]; +function generate(depth: number): string { + const r = random(); + if (depth === 0 || r < 0.2) return r < 0.1 ? String(Math.floor(random() * 100)) : names[Math.floor(random() * names.length)]!; + if (r < 0.3) return `max(${generate(depth - 1)}, ${generate(depth - 1)}, ${generate(depth - 1)})`; + if (r < 0.4) return `(${generate(depth - 1)})`; + return `${generate(depth - 1)} ${ops[Math.floor(random() * ops.length)]!} ${generate(depth - 1)}`; +} + +const scale = Number(process.argv[2] ?? "1"); +const sources: string[] = []; +for (let i = 0; i < Math.floor(400 * scale); i++) sources.push(generate(7)); +const env = new Map(); +let total = 0; +let chars = 0; +for (const source of sources) { + const tree = new Parser(tokenize(source)).parseExpr(); + chars += source.length; + for (let step = 0; step < 40; step++) { + env.set("alpha", step); + env.set("beta", step * 2 + 1); + env.set("gamma", 3); + env.set("delta", step % 7); + env.set("x", 1.5); + env.set("y", -2); + const value = tree.evaluate(env); + if (Number.isFinite(value)) total += value; + } +} +console.log("sources", sources.length, "chars", chars, "total", total.toFixed(6)); diff --git a/benchmarks/runtime/workloads/async-pipeline.ts b/benchmarks/runtime/workloads/async-pipeline.ts new file mode 100644 index 000000000..97e23857c --- /dev/null +++ b/benchmarks/runtime/workloads/async-pipeline.ts @@ -0,0 +1,47 @@ +// Async application code: request handlers awaiting cached and computed +// values, Promise.all fan-out, and sequential awaits in loops. +interface User { + id: number; + name: string; + score: number; +} + +const cache = new Map(); +async function loadUser(id: number): Promise { + const cached = cache.get(id); + if (cached !== undefined) return cached; + const user = { id, name: "user-" + id, score: (id * 37) % 101 }; + cache.set(id, user); + return user; +} +async function authorize(user: User): Promise { + return user.score % 7 !== 0; +} +async function handle(requestId: number): Promise { + const user = await loadUser(requestId % 3000); + if (!(await authorize(user))) return 0; + const requests: Promise[] = [loadUser((requestId + 1) % 3000), loadUser((requestId + 7) % 3000), loadUser((requestId + 13) % 3000)]; + const friends = await Promise.all(requests); + let sum = user.score; + for (const friend of friends) sum += friend.score; + return sum; +} +async function batch(start: number, size: number): Promise { + let total = 0; + for (let i = 0; i < size; i++) total += await handle(start + i); + return total; +} + +async function main(): Promise { + const scale = Number(process.argv[2] ?? "1"); + const batches = Math.floor(200 * scale); + let total = 0; + for (let round = 0; round < batches; round += 10) { + const work: Promise[] = []; + for (let b = 0; b < 10; b++) work.push(batch((round + b) * 100, 100)); + const results = await Promise.all(work); + for (const r of results) total += r; + } + console.log("handled", batches * 100, "total", total); +} +void main(); diff --git a/benchmarks/runtime/workloads/csv-numbers.ts b/benchmarks/runtime/workloads/csv-numbers.ts new file mode 100644 index 000000000..c4ec05dae --- /dev/null +++ b/benchmarks/runtime/workloads/csv-numbers.ts @@ -0,0 +1,43 @@ +// CSV parsing and numeric formatting: split, Number(), parseInt, +// toFixed, String(n), and string assembly of a report. +let seed = 1001; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function makeCsv(rows: number): string { + const lines: string[] = ["date,store,units,unit_price,discount"]; + for (let i = 0; i < rows; i++) { + const day = 1 + (i % 28); + lines.push( + `2026-${String(1 + (i % 12)).padStart(2, "0")}-${String(day).padStart(2, "0")},store-${Math.floor(random() * 40)},${Math.floor(random() * 50)},${(random() * 200).toFixed(2)},${(random() * 0.3).toFixed(3)}`, + ); + } + return lines.join("\n"); +} + +const scale = Number(process.argv[2] ?? "1"); +const csv = makeCsv(Math.floor(150000 * scale)); +const totals = new Map(); +const months: number[] = new Array(13).fill(0); +let units = 0; +let formatted = 0; +for (let round = 0; round < 2; round++) { + const lines = csv.split("\n"); + for (let i = 1; i < lines.length; i++) { + const cols = lines[i]!.split(","); + const month = parseInt(cols[0]!.slice(5, 7), 10); + const count = Number(cols[2]); + const price = Number(cols[3]); + const discount = Number(cols[4]); + const revenue = count * price * (1 - discount); + units += count; + months[month] = months[month]! + revenue; + totals.set(cols[1]!, (totals.get(cols[1]!) ?? 0) + revenue); + formatted += (revenue.toFixed(2) + "|" + String(count)).length; + } +} +const stores = [...totals.keys()].sort(); +const report = stores.map((s) => `${s}: ${totals.get(s)!.toFixed(2)}`).join("\n"); +console.log(report.split("\n").slice(0, 5).join("\n")); +console.log("units", units, "formatted", formatted, "jan", months[1]!.toFixed(2), "dec", months[12]!.toFixed(2)); diff --git a/benchmarks/runtime/workloads/functional-pipeline.ts b/benchmarks/runtime/workloads/functional-pipeline.ts new file mode 100644 index 000000000..1b246081e --- /dev/null +++ b/benchmarks/runtime/workloads/functional-pipeline.ts @@ -0,0 +1,56 @@ +// Functional collection pipelines: map/filter/reduce/flatMap chains, +// closures capturing configuration, and small intermediate objects. +interface Event { + user: number; + kind: string; + value: number; + time: number; +} +interface Point { + x: number; + y: number; +} +interface Scored { + user: number; + score: number; +} + +let seed = 4242; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const kinds = ["view", "click", "purchase", "share"]; +function makeEvents(count: number): Event[] { + const events: Event[] = []; + for (let i = 0; i < count; i++) { + events.push({ user: Math.floor(random() * 5000), kind: kinds[Math.floor(random() * kinds.length)]!, value: Math.floor(random() * 1000), time: i }); + } + return events; +} + +function makeScorer(weight: number, bonus: number): (e: Event) => number { + return (e) => (e.kind === "purchase" ? e.value * weight + bonus : e.kind === "click" ? weight : 0); +} + +const scale = Number(process.argv[2] ?? "1"); +const events = makeEvents(Math.floor(200000 * scale)); +let total = 0; +for (let round = 0; round < 5; round++) { + const scorer = makeScorer(round + 1, round * 10); + const scored: Scored[] = events + .filter((e) => e.time % 5 !== round) + .map((e) => ({ user: e.user, score: scorer(e) })) + .filter((s) => s.score > 0); + total += scored.reduce((sum, s) => sum + s.score, 0); + const pairs = events.slice(0, 20000).flatMap((e) => (e.kind === "share" ? [e.user, e.value] : [])); + total += pairs.length; + const points: Point[] = events.slice(0, 50000).map((e) => ({ x: e.value / 10, y: e.user / 100 })); + const centroid = points.reduce((acc: Point, p: Point): Point => ({ x: acc.x + p.x, y: acc.y + p.y }), { x: 0, y: 0 }); + total += Math.round(centroid.x + centroid.y); + const some = events.some((e) => e.value === 999 - round); + const every = events.every((e) => e.value >= 0); + const found = events.findIndex((e) => e.user === 4999 - round); + total += (some ? 1 : 0) + (every ? 1 : 0) + found; +} +console.log("total", total); diff --git a/benchmarks/runtime/workloads/json-records.ts b/benchmarks/runtime/workloads/json-records.ts new file mode 100644 index 000000000..c1e6bb866 --- /dev/null +++ b/benchmarks/runtime/workloads/json-records.ts @@ -0,0 +1,102 @@ +// JSON round trips of typed records: build, stringify, parse back into a +// declared shape, aggregate, and stringify the summary. +interface LineItem { + sku: string; + quantity: number; + price: number; +} +interface Order { + id: number; + customer: string; + region: string; + priority: boolean; + items: LineItem[]; + tags: string[]; +} +interface Summary { + region: string; + orders: number; + revenue: number; + topSku: string; +} +interface RegionTotals { + orders: number; + revenue: number; + skus: Map; +} + +let seed = 12345; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +const regions = ["north", "south", "east", "west", "central"]; +function makeOrders(count: number): Order[] { + const orders: Order[] = []; + for (let i = 0; i < count; i++) { + const items: LineItem[] = []; + const n = 1 + Math.floor(random() * 6); + for (let j = 0; j < n; j++) { + items.push({ + sku: "sku-" + Math.floor(random() * 500), + quantity: 1 + Math.floor(random() * 9), + price: Math.round(random() * 10000) / 100, + }); + } + orders.push({ + id: i, + customer: "customer-" + Math.floor(random() * 2000), + region: regions[i % regions.length]!, + priority: random() < 0.2, + items, + tags: random() < 0.5 ? ["web"] : ["store", "promo"], + }); + } + return orders; +} + +function summarize(orders: Order[]): Summary[] { + const byRegion = new Map(); + for (const order of orders) { + let entry = byRegion.get(order.region); + if (entry === undefined) { + entry = { orders: 0, revenue: 0, skus: new Map() }; + byRegion.set(order.region, entry); + } + entry.orders++; + for (const item of order.items) { + entry.revenue += item.quantity * item.price; + entry.skus.set(item.sku, (entry.skus.get(item.sku) ?? 0) + item.quantity); + } + } + const result: Summary[] = []; + for (const [region, entry] of byRegion) { + let topSku = ""; + let topCount = -1; + for (const [sku, count] of entry.skus) { + if (count > topCount || (count === topCount && sku < topSku)) { + topSku = sku; + topCount = count; + } + } + result.push({ region, orders: entry.orders, revenue: Math.round(entry.revenue * 100) / 100, topSku }); + } + result.sort((a, b) => (a.region < b.region ? -1 : a.region > b.region ? 1 : 0)); + return result; +} + +const scale = Number(process.argv[2] ?? "1"); +const orders = makeOrders(Math.floor(4000 * scale)); +let checksum = 0; +let last = ""; +for (let round = 0; round < 6; round++) { + const text = JSON.stringify(orders); + checksum += text.length; + const parsed = JSON.parse(text) as Order[]; + const summary = summarize(parsed); + last = JSON.stringify(summary); + checksum += last.length; +} +console.log(last); +console.log("checksum", checksum); diff --git a/benchmarks/runtime/workloads/numeric-kernels.ts b/benchmarks/runtime/workloads/numeric-kernels.ts new file mode 100644 index 000000000..b2b1db3cd --- /dev/null +++ b/benchmarks/runtime/workloads/numeric-kernels.ts @@ -0,0 +1,74 @@ +// Integer- and float-heavy kernels: sieve, hashing, matrix multiply on +// number[][], histogram on plain arrays, and a Float64Array smoothing pass. +function sieve(limit: number): number { + const composite: boolean[] = new Array(limit + 1).fill(false); + let count = 0; + for (let i = 2; i <= limit; i++) { + if (composite[i]) continue; + count++; + for (let j = i * i; j <= limit; j += i) composite[j] = true; + } + return count; +} +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) { + h ^= values[i]! & 0xff; + h = Math.imul(h, 16777619) >>> 0; + } + return h; +} +function matmul(a: number[][], b: number[][], n: number): number[][] { + const c: number[][] = []; + for (let i = 0; i < n; i++) { + const row: number[] = new Array(n).fill(0); + const ai = a[i]!; + for (let k = 0; k < n; k++) { + const aik = ai[k]!; + const bk = b[k]!; + for (let j = 0; j < n; j++) row[j] = row[j]! + aik * bk[j]!; + } + c.push(row); + } + return c; +} + +const scale = Number(process.argv[2] ?? "1"); +const primes = sieve(Math.floor(2000000 * scale)); +const data: number[] = []; +let x = 1; +for (let i = 0; i < Math.floor(1000000 * scale); i++) { + x = (Math.imul(x, 48271) + 11) & 0x7fffffff; + data.push(x % 1000); +} +let hash = 0; +for (let r = 0; r < 4; r++) hash = (hash + fnv(data)) >>> 0; +const histogram: number[] = new Array(100).fill(0); +for (let i = 0; i < data.length; i++) { + const bucket = data[i]! % 100; + histogram[bucket] = histogram[bucket]! + 1; +} +const n = Math.floor(120 * Math.sqrt(scale)); +const a: number[][] = []; +const b: number[][] = []; +for (let i = 0; i < n; i++) { + const ra: number[] = []; + const rb: number[] = []; + for (let j = 0; j < n; j++) { + ra.push(((i * j) % 7) - 3); + rb.push(((i + j) % 5) - 2); + } + a.push(ra); + b.push(rb); +} +const c = matmul(a, b, n); +let trace = 0; +for (let i = 0; i < n; i++) trace += c[i]![i]!; +const signal = new Float64Array(data.length); +for (let i = 0; i < data.length; i++) signal[i] = data[i]! / 7; +let smooth = 0; +for (let pass = 0; pass < 3; pass++) { + for (let i = 1; i < signal.length - 1; i++) signal[i] = (signal[i - 1]! + signal[i]! * 2 + signal[i + 1]!) / 4; +} +for (let i = 0; i < signal.length; i += 1000) smooth += signal[i]!; +console.log("primes", primes, "hash", hash, "hist", histogram[0], histogram[99], "trace", trace, "smooth", smooth.toFixed(6)); diff --git a/benchmarks/runtime/workloads/records-sort.ts b/benchmarks/runtime/workloads/records-sort.ts new file mode 100644 index 000000000..d58d9bdb7 --- /dev/null +++ b/benchmarks/runtime/workloads/records-sort.ts @@ -0,0 +1,80 @@ +// Business-record processing: multi-key sorts with comparator closures, +// group-by into Maps, and reductions over the groups. +interface Employee { + id: number; + name: string; + department: string; + level: number; + salary: number; + startYear: number; +} + +let seed = 2024; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const departments = ["eng", "sales", "support", "finance", "legal", "design", "ops"]; +const first = ["ana", "bo", "cy", "dee", "eli", "fay", "gus", "hal", "ivy", "jon"]; +const last = ["smith", "jones", "lee", "khan", "garcia", "chen", "novak", "silva"]; + +function makeEmployees(count: number): Employee[] { + const result: Employee[] = []; + for (let i = 0; i < count; i++) { + result.push({ + id: i, + name: first[Math.floor(random() * first.length)]! + " " + last[Math.floor(random() * last.length)]!, + department: departments[Math.floor(random() * departments.length)]!, + level: 1 + Math.floor(random() * 7), + salary: 40000 + Math.floor(random() * 160000), + startYear: 1995 + Math.floor(random() * 30), + }); + } + return result; +} + +function compareBy(keys: string[]): (a: Employee, b: Employee) => number { + return (a, b) => { + for (const key of keys) { + let d = 0; + if (key === "department") d = a.department < b.department ? -1 : a.department > b.department ? 1 : 0; + else if (key === "level") d = b.level - a.level; + else if (key === "salary") d = b.salary - a.salary; + else if (key === "name") d = a.name < b.name ? -1 : a.name > b.name ? 1 : 0; + else d = a.id - b.id; + if (d !== 0) return d; + } + return 0; + }; +} + +const scale = Number(process.argv[2] ?? "1"); +const employees = makeEmployees(Math.floor(60000 * scale)); +let checksum = 0; +const orders = [ + ["department", "level", "salary", "id"], + ["name", "id"], + ["salary", "id"], + ["level", "name", "id"], +]; +for (const order of orders) { + const sorted = employees.slice().sort(compareBy(order)); + for (let i = 0; i < sorted.length; i += 997) checksum = (checksum * 31 + sorted[i]!.id) % 1000000007; +} +const groups = new Map(); +for (const e of employees) { + const key = e.department + "/" + e.level; + const group = groups.get(key); + if (group === undefined) groups.set(key, [e]); + else group.push(e); +} +const lines: string[] = []; +for (const [key, group] of groups) { + const total = group.reduce((sum, e) => sum + e.salary, 0); + const veterans = group.filter((e) => e.startYear < 2010).length; + const top = group.reduce((best, e) => (e.salary > best.salary ? e : best), group[0]!); + lines.push(`${key} n=${group.length} avg=${Math.round(total / group.length)} vets=${veterans} top=${top.id}`); +} +lines.sort(); +for (const line of lines.slice(0, 12)) console.log(line); +console.log("groups", lines.length, "checksum", checksum); diff --git a/benchmarks/runtime/workloads/regex-logs.ts b/benchmarks/runtime/workloads/regex-logs.ts new file mode 100644 index 000000000..729639706 --- /dev/null +++ b/benchmarks/runtime/workloads/regex-logs.ts @@ -0,0 +1,49 @@ +// Regex-driven log processing: match with capture groups, test, and replace. +let seed = 99; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +const methods = ["GET", "POST", "PUT", "DELETE"]; +const paths = ["/api/users", "/api/orders", "/static/app.js", "/health", "/api/items/search"]; +function makeLines(count: number): string[] { + const lines: string[] = []; + for (let i = 0; i < count; i++) { + const ip = `10.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}`; + const method = methods[Math.floor(random() * methods.length)]!; + const path = paths[Math.floor(random() * paths.length)]! + (random() < 0.3 ? "?id=" + Math.floor(random() * 1000) : ""); + const status = random() < 0.9 ? 200 : random() < 0.5 ? 404 : 500; + const ms = Math.floor(random() * 900); + const agent = random() < 0.5 ? "Mozilla/5.0 (Macintosh)" : "curl/8.4.0"; + lines.push(`${ip} - - [07/Oct/2026:12:${String(i % 60).padStart(2, "0")}:00 +0000] "${method} ${path} HTTP/1.1" ${status} ${ms} "${agent}"`); + } + return lines; +} + +const linePattern = /^(\d+\.\d+\.\d+\.\d+) - - \[([^\]]+)\] "(\w+) ([^ ?"]+)(?:\?([^ "]*))? HTTP\/1\.1" (\d{3}) (\d+) "([^"]*)"$/; +const botPattern = /curl|bot|spider/i; +const digits = /\d+/g; + +const scale = Number(process.argv[2] ?? "1"); +const lines = makeLines(Math.floor(30000 * scale)); +const counts = new Map(); +let errors = 0; +let bots = 0; +let totalMs = 0; +let redactedLength = 0; +for (let round = 0; round < 3; round++) { + for (const line of lines) { + const m = linePattern.exec(line); + if (m === null) continue; + const key = m[3]! + " " + m[4]!; + counts.set(key, (counts.get(key) ?? 0) + 1); + if (m[6] !== "200") errors++; + if (botPattern.test(m[8]!)) bots++; + totalMs += Number(m[7]); + redactedLength += line.replace(digits, "#").length; + } +} +const keys = [...counts.keys()].sort(); +for (const key of keys) console.log(key, counts.get(key)); +console.log("errors", errors, "bots", bots, "ms", totalMs, "redacted", redactedLength); diff --git a/benchmarks/runtime/workloads/template-render.ts b/benchmarks/runtime/workloads/template-render.ts new file mode 100644 index 000000000..d59fbc15b --- /dev/null +++ b/benchmarks/runtime/workloads/template-render.ts @@ -0,0 +1,73 @@ +// Server-side HTML rendering: template literals, escaping, joins, and +// conditional fragments over a list of typed view models. +interface Product { + id: number; + title: string; + description: string; + price: number; + tags: string[]; + inStock: boolean; + rating: number; +} + +let seed = 31337; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const words = ["fast", "durable", "bold", "eco", '"quoted"', "light", "smart", "R&D", "classic", "modern"]; +function sentence(n: number): string { + const parts: string[] = []; + for (let i = 0; i < n; i++) parts.push(words[Math.floor(random() * words.length)]!); + return parts.join(" "); +} +function makeProducts(count: number): Product[] { + const products: Product[] = []; + for (let i = 0; i < count; i++) { + products.push({ + id: i, + title: sentence(3), + description: sentence(20), + price: Math.round(random() * 100000) / 100, + tags: [words[i % words.length]!, words[(i * 7) % words.length]!], + inStock: random() < 0.8, + rating: Math.round(random() * 50) / 10, + }); + } + return products; +} + +function escapeHtml(text: string): string { + return text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} +function stars(rating: number): string { + let out = ""; + for (let i = 1; i <= 5; i++) out += i <= Math.round(rating) ? "*" : "-"; + return out; +} +function renderProduct(p: Product): string { + const tags = p.tags.map((t) => `
  • ${escapeHtml(t)}
    • `).join(""); + return `
    +

    ${escapeHtml(p.title)}

    +

    ${escapeHtml(p.description)}

    + $${p.price.toFixed(2)} + ${stars(p.rating)} +
      ${tags}
    + ${p.inStock ? "" : "Sold out"} +
    `; +} +function renderPage(products: Product[], page: number): string { + const body = products.map(renderProduct).join("\n"); + return `Page ${page}${body}`; +} + +const scale = Number(process.argv[2] ?? "1"); +const products = makeProducts(Math.floor(2000 * scale)); +let bytes = 0; +let hash = 0; +for (let page = 0; page < 10; page++) { + const html = renderPage(products, page); + bytes += html.length; + for (let i = 0; i < html.length; i += 101) hash = (hash * 33 + html.charCodeAt(i)) % 2147483647; +} +console.log("bytes", bytes, "hash", hash); diff --git a/benchmarks/runtime/workloads/validate-errors.ts b/benchmarks/runtime/workloads/validate-errors.ts new file mode 100644 index 000000000..387014435 --- /dev/null +++ b/benchmarks/runtime/workloads/validate-errors.ts @@ -0,0 +1,76 @@ +// Validation layers that throw and catch errors for a fraction of inputs, +// with deep call chains of small functions on the success path. +class ValidationError extends Error { + readonly field: string; + constructor(field: string, message: string) { + super(message); + this.field = field; + } +} +interface Signup { + email: string; + age: number; + name: string; + password: string; +} + +let seed = 8080; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function makeSignup(i: number): Signup { + const bad = random(); + return { + email: bad < 0.05 ? "broken" : `user${i}@example.com`, + age: bad > 0.97 ? -1 : 18 + Math.floor(random() * 60), + name: bad > 0.95 && bad <= 0.97 ? " " : "Name " + i, + password: random() < 0.03 ? "short" : "correct horse battery " + i, + }; +} + +function requireText(field: string, value: string, min: number): string { + if (value.length < min) throw new ValidationError(field, `${field} must have at least ${min} characters`); + return value; +} +function requireEmail(value: string): string { + const at = value.indexOf("@"); + if (at <= 0 || value.indexOf(".", at) < 0) throw new ValidationError("email", "invalid email"); + return value.toLowerCase(); +} +function requireRange(field: string, value: number, min: number, max: number): number { + if (value < min || value > max) throw new ValidationError(field, `${field} out of range`); + return value; +} +function normalize(input: Signup): Signup { + return { + email: requireEmail(input.email), + age: requireRange("age", input.age, 13, 120), + name: requireText("name", input.name.trim(), 1), + password: requireText("password", input.password, 8), + }; +} +function score(s: Signup): number { + return s.email.length + s.age + s.name.length + s.password.length; +} + +const scale = Number(process.argv[2] ?? "1"); +const inputs: Signup[] = []; +for (let i = 0; i < Math.floor(200000 * scale); i++) inputs.push(makeSignup(i)); +const failures = new Map(); +let accepted = 0; +let total = 0; +for (let round = 0; round < 3; round++) { + for (const input of inputs) { + try { + const s = normalize(input); + accepted++; + total += score(s); + } catch (error) { + if (error instanceof ValidationError) failures.set(error.field, (failures.get(error.field) ?? 0) + 1); + else throw error; + } + } +} +console.log("accepted", accepted, "total", total); +for (const key of [...failures.keys()].sort()) console.log(key, failures.get(key)); diff --git a/benchmarks/runtime/workloads/word-graph.ts b/benchmarks/runtime/workloads/word-graph.ts new file mode 100644 index 000000000..e40e9e638 --- /dev/null +++ b/benchmarks/runtime/workloads/word-graph.ts @@ -0,0 +1,61 @@ +// Map/Set-heavy text and graph processing: word frequencies, bigrams, +// and breadth-first search over a string-keyed adjacency map. +let seed = 555; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const syllables = ["ka", "lo", "mi", "ne", "ru", "sa", "to", "vi", "ze", "po", "qu", "ex"]; +function word(): string { + const n = 1 + Math.floor(random() * 3); + let w = ""; + for (let i = 0; i < n; i++) w += syllables[Math.floor(random() * syllables.length)]!; + return w; +} + +const scale = Number(process.argv[2] ?? "1"); +const text: string[] = []; +for (let i = 0; i < Math.floor(300000 * scale); i++) text.push(word()); + +const freq = new Map(); +for (const w of text) freq.set(w, (freq.get(w) ?? 0) + 1); +const bigrams = new Map(); +for (let i = 1; i < text.length; i++) { + const key = text[i - 1]! + " " + text[i]!; + bigrams.set(key, (bigrams.get(key) ?? 0) + 1); +} +const top = [...freq.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5); +console.log("words", freq.size, "bigrams", bigrams.size, "top", top.map((e) => e[0] + ":" + e[1]).join(",")); + +const graph = new Map(); +for (let i = 1; i < text.length; i += 3) { + const from = text[i - 1]!; + const to = text[i]!; + const edges = graph.get(from); + if (edges === undefined) graph.set(from, [to]); + else if (edges.length < 8) edges.push(to); +} +let reachable = 0; +let depthSum = 0; +const starts = [...graph.keys()].sort().slice(0, 60); +for (const start of starts) { + const seen = new Set([start]); + let frontier = [start]; + let depth = 0; + while (frontier.length > 0) { + const next: string[] = []; + for (const node of frontier) { + for (const neighbor of graph.get(node) ?? []) { + if (!seen.has(neighbor)) { + seen.add(neighbor); + next.push(neighbor); + } + } + } + frontier = next; + depth++; + } + reachable += seen.size; + depthSum += depth; +} +console.log("reachable", reachable, "depth", depthSum); diff --git a/package.json b/package.json index 0efe1a8a0..813302758 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "build:fresh": "rm -rf packages/compiler/dist packages/cli/dist node_modules/.cache/scriptc-tsc && pnpm build", "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-cli.mts", "bench:builds": "node scripts/bench-builds.mjs", + "bench:runtime": "node scripts/bench-runtime.mjs", "bench:ast-access": "node scripts/bench-ast-access.mjs", "bench:backend-analysis": "node scripts/bench-backend-analysis.mjs", "test": "vitest run", diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs new file mode 100644 index 000000000..99bf150d8 --- /dev/null +++ b/scripts/bench-runtime.mjs @@ -0,0 +1,387 @@ +/* Runtime performance A/B harness for compiled programs. + * + * Builds every workload in benchmarks/runtime/workloads.json with one or two + * scriptc checkouts, verifies each executable against Node byte-for-byte + * (stdout, stderr, exit status), then times interleaved launches and reports + * per-workload medians with bootstrap confidence intervals for the + * candidate/baseline ratio. Executables that diverge from Node are never + * timed. A per-host advisory lock serializes measurements so concurrent + * agents on one machine do not disturb each other's numbers. */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const suiteRoot = join(repoRoot, "benchmarks/runtime"); + +const { values } = parseArgs({ + options: { + candidate: { type: "string", default: repoRoot }, + baseline: { type: "string" }, + workloads: { type: "string" }, + runs: { type: "string", default: "15" }, + warmup: { type: "string", default: "2" }, + timeout: { type: "string", default: "120" }, + json: { type: "string" }, + "no-lock": { type: "boolean", default: false }, + "keep": { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help) { + console.log(`Usage: node scripts/bench-runtime.mjs [--candidate=] [--baseline=] + [--workloads=a,b] [--runs=11] [--warmup=2] [--timeout=120] [--json=] [--no-lock] [--keep] + +Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native +artifacts for this host. Without --baseline, reports absolute times only.`); + process.exit(0); +} +const runs = Number(values.runs); +const warmup = Number(values.warmup); +const timeoutMs = Number(values.timeout) * 1000; +assert.ok(Number.isInteger(runs) && runs >= 3 && runs <= 200, "--runs must be 3..200"); +assert.ok(Number.isInteger(warmup) && warmup >= 0 && warmup <= 20, "--warmup must be 0..20"); + +const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8")); +const selected = values.workloads ? new Set(values.workloads.split(",")) : null; +const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name)); +if (selected) { + for (const name of selected) + assert.ok( + manifest.workloads.some((w) => w.name === name), + `unknown workload ${name}`, + ); +} + +const work = mkdtempSync( + join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-runtime-"), +); + +/* ── inputs for file-processing applications ──────────────────────────── */ +function generateInput(kind) { + const path = join(work, `${kind}.input`); + if (kind === "log-summary") { + writeFileSync( + path, + Array.from( + { length: 300_000 }, + (_, i) => + `/route/${i % 37}?request=${i} ${i % 13 === 0 ? 500 : 200} ${i % 250} ${200 + (i % 4000)}`, + ).join("\n") + "\n", + ); + } else if (kind === "inventory-report") { + writeFileSync( + path, + Array.from({ length: 300_000 }, (_, i) => + ["sku-" + i, "category-" + (i % 17), "product " + i, i % 40, 199 + (i % 5000)].join("\t"), + ).join("\n") + "\n", + ); + } else throw new Error(`unknown input kind ${kind}`); + return path; +} + +/* ── advisory lock ─────────────────────────────────────────────────────── */ +const lockDir = join( + process.platform === "win32" ? tmpdir() : "/tmp", + "scriptc-bench-runtime.lock", +); +function acquireLock() { + const started = Date.now(); + let announced = false; + for (;;) { + try { + mkdirSync(lockDir); + writeFileSync(join(lockDir, "pid"), String(process.pid)); + return; + } catch (error) { + if (error.code !== "EEXIST") throw error; + let owner = NaN; + try { + owner = Number(readFileSync(join(lockDir, "pid"), "utf8")); + } catch {} + let alive = false; + if (Number.isInteger(owner)) { + try { + process.kill(owner, 0); + alive = true; + } catch {} + } + if (!alive && Date.now() - started > 2000) { + rmSync(lockDir, { recursive: true, force: true }); + continue; + } + if (!announced) { + process.stderr.write(`waiting for benchmark lock held by pid ${owner}\n`); + announced = true; + } + spawnSync("sleep", ["2"]); + } + } +} +function releaseLock() { + rmSync(lockDir, { recursive: true, force: true }); +} + +/* ── process helpers ───────────────────────────────────────────────────── */ +function run(command, args, options = {}) { + const start = process.hrtime.bigint(); + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + timeout: options.timeout ?? timeoutMs, + env: options.env ?? process.env, + cwd: options.cwd, + }); + const ms = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error && result.error.code !== "ETIMEDOUT") throw result.error; + return { + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + status: result.status, + signal: result.signal, + timedOut: result.error?.code === "ETIMEDOUT" || result.signal === "SIGTERM", + ms, + }; +} + +function peakRss(binary, args) { + if (process.platform !== "darwin" && process.platform !== "linux") return null; + const timeArgs = + process.platform === "darwin" ? ["-l", binary, ...args] : ["-f", "rss_kib=%M", binary, ...args]; + const result = run("/usr/bin/time", timeArgs); + const match = + process.platform === "darwin" + ? /(\d+)\s+maximum resident set size/.exec(result.stderr) + : /rss_kib=(\d+)/.exec(result.stderr); + if (!match) return null; + return Number(match[1]) * (process.platform === "darwin" ? 1 : 1024); +} + +/* ── statistics ────────────────────────────────────────────────────────── */ +function median(xs) { + const s = [...xs].sort((a, b) => a - b); + const m = s.length >> 1; + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} +function makeRandom(seed) { + let state = seed >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; +} +function bootstrapRatio(candidate, baseline, resamples = 2000) { + const random = makeRandom(0x5eed); + const ratios = []; + const pick = (xs) => + Array.from({ length: xs.length }, () => xs[Math.floor(random() * xs.length)]); + for (let i = 0; i < resamples; i++) ratios.push(median(pick(candidate)) / median(pick(baseline))); + ratios.sort((a, b) => a - b); + return { + low: ratios[Math.floor(resamples * 0.025)], + high: ratios[Math.floor(resamples * 0.975)], + }; +} + +/* ── build ─────────────────────────────────────────────────────────────── */ +function cliFor(root) { + const cli = join(resolve(root), "packages/cli/dist/bootstrap.js"); + statSync(cli); + return cli; +} +function revision(root) { + const r = run("git", ["-C", root, "rev-parse", "--short", "HEAD"], { timeout: 10_000 }); + const dirty = run("git", ["-C", root, "status", "--porcelain", "--untracked-files=no"], { + timeout: 10_000, + }); + return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : ""); +} +function build(label, root, workload) { + const cli = cliFor(root); + const out = join(work, label, workload.name); + mkdirSync(dirname(out), { recursive: true }); + const cache = join(work, label, ".cache"); + mkdirSync(cache, { recursive: true }); + const env = { ...process.env, SCRIPTC_CACHE_DIR: cache }; + delete env.SCRIPTC_NO_CACHE; + delete env.SCRIPTC_TIMING; + const entry = join(suiteRoot, workload.entry); + const result = run(process.execPath, [cli, "build", entry, "--optimization=release", "-o", out], { + env, + timeout: 900_000, + }); + if (result.status !== 0) + return { + ok: false, + error: (result.stderr || result.stdout).slice(0, 4000), + buildMs: result.ms, + }; + return { ok: true, binary: out, bytes: statSync(out).size, buildMs: result.ms }; +} + +/* ── main ──────────────────────────────────────────────────────────────── */ +const contenders = [ + ...(values.baseline ? [{ label: "baseline", root: resolve(values.baseline) }] : []), + { label: "candidate", root: resolve(values.candidate) }, +]; +const report = { + schema: 1, + date: new Date().toISOString(), + host: { platform: process.platform, arch: process.arch, node: process.version }, + runs, + contenders: contenders.map((c) => ({ ...c, revision: revision(c.root) })), + workloads: [], +}; + +try { + const inputs = new Map(); + for (const w of workloads) + if (w.input && !inputs.has(w.input)) inputs.set(w.input, generateInput(w.input)); + + // Builds and the Node oracle run outside the lock: they do not need a + // quiet machine, and holding the lock while compiling would serialize + // every agent's slowest phase. + const prepared = []; + for (const w of workloads) { + const args = w.args.map((a) => (a === "{input}" ? inputs.get(w.input) : a)); + // The application workloads are ESM without a package.json "type"; Node's + // reparse warning is an oracle artifact, not program output. + const oracle = run(process.execPath, [ + "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", + join(suiteRoot, w.entry), + ...args, + ]); + const entry = { name: w.name, args, results: {} }; + if (oracle.status !== 0 || oracle.timedOut) { + entry.error = `node oracle failed (status ${oracle.status}): ${oracle.stderr.slice(0, 2000)}`; + prepared.push(entry); + continue; + } + for (const c of contenders) { + const built = build(c.label, c.root, w); + const result = { build_ms: Math.round(built.buildMs) }; + entry.results[c.label] = result; + if (!built.ok) { + result.status = "build-failed"; + result.error = built.error; + continue; + } + result.bytes = built.bytes; + result.binary = built.binary; + const check = run(built.binary, args); + if (check.timedOut) { + result.status = "timeout"; + continue; + } + if ( + check.stdout !== oracle.stdout || + check.stderr !== oracle.stderr || + check.status !== oracle.status + ) { + result.status = "mismatch"; + result.error = `expected status ${oracle.status}, got ${check.status}; stdout ${check.stdout === oracle.stdout ? "matches" : "differs"}; stderr ${check.stderr === oracle.stderr ? "matches" : "differs"}`; + continue; + } + result.status = "ok"; + result.samples = []; + } + entry.node_ms = Math.round(oracle.ms); + prepared.push(entry); + process.stderr.write(`prepared ${w.name}\n`); + } + + if (!values["no-lock"]) acquireLock(); + try { + for (const entry of prepared) { + const live = contenders.filter((c) => entry.results[c.label]?.status === "ok"); + if (live.length === 0) continue; + for (const c of live) + for (let i = 0; i < warmup; i++) run(entry.results[c.label].binary, entry.args); + const random = makeRandom(entry.name.length * 7919); + for (let r = 0; r < runs; r++) { + const order = [...live]; + if (order.length === 2 && random() < 0.5) order.reverse(); + for (const c of order) { + const result = entry.results[c.label]; + const sample = run(result.binary, entry.args); + if (sample.timedOut || sample.status !== 0) { + result.status = sample.timedOut ? "timeout" : "unstable-exit"; + break; + } + result.samples.push(Math.round(sample.ms * 100) / 100); + } + } + for (const c of live) { + const result = entry.results[c.label]; + if (result.status !== "ok") continue; + result.median_ms = Math.round(median(result.samples) * 100) / 100; + result.peak_rss_bytes = peakRss(result.binary, entry.args); + } + const base = entry.results.baseline; + const cand = entry.results.candidate; + if (base?.status === "ok" && cand?.status === "ok") { + const ci = bootstrapRatio(cand.samples, base.samples); + entry.ratio = Math.round((cand.median_ms / base.median_ms) * 10000) / 10000; + entry.ci95 = [Math.round(ci.low * 10000) / 10000, Math.round(ci.high * 10000) / 10000]; + entry.verdict = ci.high < 1 ? "faster" : ci.low > 1 ? "slower" : "neutral"; + entry.size_ratio = Math.round((cand.bytes / base.bytes) * 10000) / 10000; + } + process.stderr.write(`measured ${entry.name}\n`); + } + } finally { + if (!values["no-lock"]) releaseLock(); + } + for (const entry of prepared) { + for (const result of Object.values(entry.results)) delete result.binary; + report.workloads.push(entry); + } + const ratios = report.workloads.filter((w) => w.ratio !== undefined).map((w) => w.ratio); + if (ratios.length > 0) + report.geomean_ratio = + Math.round(Math.exp(ratios.reduce((s, r) => s + Math.log(r), 0) / ratios.length) * 10000) / + 10000; +} finally { + if (!values.keep) rmSync(work, { recursive: true, force: true }); +} + +if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n"); + +/* ── summary table ─────────────────────────────────────────────────────── */ +const pct = (r) => `${r < 1 ? "" : "+"}${((r - 1) * 100).toFixed(1)}%`; +const lines = []; +if (values.baseline) { + lines.push( + "| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms |", + ); + lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: |"); +} else { + lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms |"); + lines.push("| --- | ---: | --- | ---: | ---: | ---: |"); +} +for (const w of report.workloads) { + const b = w.results.baseline; + const c = w.results.candidate; + if (w.error) { + lines.push(`| ${w.name} | ${w.error.split("\n")[0]} |`); + } else if (values.baseline) { + const show = (r) => (r?.status === "ok" ? r.median_ms.toFixed(1) : (r?.status ?? "-")); + lines.push( + `| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} |`, + ); + } else { + lines.push( + `| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} |`, + ); + } +} +if (report.geomean_ratio !== undefined) + lines.push("", `geomean change: ${pct(report.geomean_ratio)}`); +console.log(lines.join("\n")); +const broken = report.workloads.some( + (w) => w.error || Object.values(w.results).some((r) => r.status !== "ok"), +); +process.exitCode = broken ? 1 : 0; From 523a8271f8365be994c3ce0673e1d8b899f8e8e9 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 7 Oct 2026 18:02:01 -0700 Subject: [PATCH 2/3] Skip peak RSS when /usr/bin/time is unavailable --- scripts/bench-runtime.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index 99bf150d8..faa3677d2 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -9,7 +9,15 @@ * agents on one machine do not disturb each other's numbers. */ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -150,6 +158,7 @@ function run(command, args, options = {}) { function peakRss(binary, args) { if (process.platform !== "darwin" && process.platform !== "linux") return null; + if (!existsSync("/usr/bin/time")) return null; const timeArgs = process.platform === "darwin" ? ["-l", binary, ...args] : ["-f", "rss_kib=%M", binary, ...args]; const result = run("/usr/bin/time", timeArgs); From 1dc9e04fc5ade8065b5830fd2d649d7dd28c473d Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 7 Oct 2026 18:37:21 -0700 Subject: [PATCH 3/3] Time Node in the interleaved loop and report candidate/Node ratios Node is the reference the project aims to beat. A single cold oracle launch overstated its time; it is now warmed and sampled in the same interleaved loop as the compiled executables, and the report adds a per-workload candidate/Node ratio and its geomean. --no-node skips it. --- scripts/bench-runtime.mjs | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index faa3677d2..3753473c1 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -27,6 +27,7 @@ const repoRoot = fileURLToPath(new URL("..", import.meta.url)); const suiteRoot = join(repoRoot, "benchmarks/runtime"); const { values } = parseArgs({ + allowNegative: true, options: { candidate: { type: "string", default: repoRoot }, baseline: { type: "string" }, @@ -36,16 +37,18 @@ const { values } = parseArgs({ timeout: { type: "string", default: "120" }, json: { type: "string" }, "no-lock": { type: "boolean", default: false }, + node: { type: "boolean", default: true }, "keep": { type: "boolean", default: false }, help: { type: "boolean", default: false }, }, }); if (values.help) { console.log(`Usage: node scripts/bench-runtime.mjs [--candidate=] [--baseline=] - [--workloads=a,b] [--runs=11] [--warmup=2] [--timeout=120] [--json=] [--no-lock] [--keep] + [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--json=] [--no-lock] [--no-node] [--keep] Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native -artifacts for this host. Without --baseline, reports absolute times only.`); +artifacts for this host. Without --baseline, reports absolute times only. Node is +timed in the same interleaved loop as the reference to beat (--no-node skips it).`); process.exit(0); } const runs = Number(values.runs); @@ -299,6 +302,11 @@ try { result.samples = []; } entry.node_ms = Math.round(oracle.ms); + entry.nodeCommand = [ + "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", + join(suiteRoot, w.entry), + ...args, + ]; prepared.push(entry); process.stderr.write(`prepared ${w.name}\n`); } @@ -310,6 +318,10 @@ try { if (live.length === 0) continue; for (const c of live) for (let i = 0; i < warmup; i++) run(entry.results[c.label].binary, entry.args); + // Node is the reference the project aims to beat, so it is timed in the + // same interleaved loop (whole-process wall time, startup included). + const nodeSamples = []; + if (values.node) run(process.execPath, entry.nodeCommand); const random = makeRandom(entry.name.length * 7919); for (let r = 0; r < runs; r++) { const order = [...live]; @@ -323,7 +335,12 @@ try { } result.samples.push(Math.round(sample.ms * 100) / 100); } + if (values.node) nodeSamples.push(run(process.execPath, entry.nodeCommand).ms); } + if (nodeSamples.length > 0) entry.node_ms = Math.round(median(nodeSamples) * 100) / 100; + const reference = entry.results.candidate; + if (reference?.status === "ok" && reference.samples.length > 0 && nodeSamples.length > 0) + entry.vs_node = Math.round((median(reference.samples) / entry.node_ms) * 1000) / 1000; for (const c of live) { const result = entry.results[c.label]; if (result.status !== "ok") continue; @@ -346,8 +363,14 @@ try { } for (const entry of prepared) { for (const result of Object.values(entry.results)) delete result.binary; + delete entry.nodeCommand; report.workloads.push(entry); } + const vsNode = report.workloads.filter((w) => w.vs_node !== undefined).map((w) => w.vs_node); + if (vsNode.length > 0) + report.geomean_vs_node = + Math.round(Math.exp(vsNode.reduce((s, r) => s + Math.log(r), 0) / vsNode.length) * 1000) / + 1000; const ratios = report.workloads.filter((w) => w.ratio !== undefined).map((w) => w.ratio); if (ratios.length > 0) report.geomean_ratio = @@ -362,14 +385,15 @@ if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\ /* ── summary table ─────────────────────────────────────────────────────── */ const pct = (r) => `${r < 1 ? "" : "+"}${((r - 1) * 100).toFixed(1)}%`; const lines = []; +const xNode = (w) => (w.vs_node === undefined ? "-" : `${w.vs_node.toFixed(2)}x`); if (values.baseline) { lines.push( - "| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms |", + "| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms | candidate/node |", ); - lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: |"); + lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: |"); } else { - lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms |"); - lines.push("| --- | ---: | --- | ---: | ---: | ---: |"); + lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms | scriptc/node |"); + lines.push("| --- | ---: | --- | ---: | ---: | ---: | ---: |"); } for (const w of report.workloads) { const b = w.results.baseline; @@ -379,16 +403,20 @@ for (const w of report.workloads) { } else if (values.baseline) { const show = (r) => (r?.status === "ok" ? r.median_ms.toFixed(1) : (r?.status ?? "-")); lines.push( - `| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} |`, + `| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} | ${xNode(w)} |`, ); } else { lines.push( - `| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} |`, + `| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} | ${xNode(w)} |`, ); } } if (report.geomean_ratio !== undefined) lines.push("", `geomean change: ${pct(report.geomean_ratio)}`); +if (report.geomean_vs_node !== undefined) + lines.push( + `geomean candidate/node: ${report.geomean_vs_node.toFixed(3)}x (lower is better; goal: well below 1)`, + ); console.log(lines.join("\n")); const broken = report.workloads.some( (w) => w.error || Object.values(w.results).some((r) => r.status !== "ok"),