diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md new file mode 100644 index 000000000..385532c9d --- /dev/null +++ b/benchmarks/runtime/README.md @@ -0,0 +1,27 @@ +# Runtime benchmarks + +Application-shaped workloads for measuring the speed of compiled executables. Each workload is deterministic, generates its own input (or reads a generated file), and prints a summary that must match Node.js byte-for-byte before it is timed. + +| Workload | Exercises | +| --- | --- | +| `json-records` | `JSON.stringify`, typed `JSON.parse`, Map aggregation | +| `regex-logs` | regex `exec` with groups, `test`, global `replace` | +| `ast-interp` | tokenizer, recursive-descent parser, class dispatch | +| `records-sort` | comparator sorts, group-by, `reduce`/`filter` | +| `template-render` | template literals, escaping, `map`/`join` | +| `functional-pipeline` | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `async-pipeline` | `async`/`await`, `Promise.all` fan-out | +| `word-graph` | string-keyed Map/Set, BFS | +| `alloc-trees` | allocation-heavy trees, immutable object-spread updates | +| `numeric-kernels` | sieve, hashing, `number[][]` matmul, `Float64Array` | +| `validate-errors` | throw/catch on a fraction of inputs, small call chains | +| `csv-numbers` | `split`, `Number`, `parseInt`, `toFixed` | +| `log-summary`, `inventory-report` | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | + +Compare two checkouts (each with a built CLI and native artifacts for the host): + +```console +$ pnpm bench:runtime --baseline=../scriptc-main --candidate=. --runs=15 --json=result.json +``` + +Without `--baseline`, the runner reports absolute medians. Baseline and candidate launches are interleaved, and each workload reports the median ratio with a bootstrap 95% confidence interval. `faster`/`slower` verdicts require the whole interval on one side of 1.0. A per-host lock serializes the timed phase so concurrent runs on one machine do not distort each other. Executables that time out or diverge from Node are reported, never timed. diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json new file mode 100644 index 000000000..cba7115d0 --- /dev/null +++ b/benchmarks/runtime/workloads.json @@ -0,0 +1,108 @@ +{ + "workloads": [ + { + "name": "json-records", + "entry": "workloads/json-records.ts", + "args": [ + "4" + ] + }, + { + "name": "regex-logs", + "entry": "workloads/regex-logs.ts", + "args": [ + "1" + ] + }, + { + "name": "ast-interp", + "entry": "workloads/ast-interp.ts", + "args": [ + "5" + ] + }, + { + "name": "records-sort", + "entry": "workloads/records-sort.ts", + "args": [ + "1" + ] + }, + { + "name": "template-render", + "entry": "workloads/template-render.ts", + "args": [ + "1" + ] + }, + { + "name": "functional-pipeline", + "entry": "workloads/functional-pipeline.ts", + "args": [ + "1" + ] + }, + { + "name": "async-pipeline", + "entry": "workloads/async-pipeline.ts", + "args": [ + "1" + ] + }, + { + "name": "word-graph", + "entry": "workloads/word-graph.ts", + "args": [ + "2" + ] + }, + { + "name": "alloc-trees", + "entry": "workloads/alloc-trees.ts", + "args": [ + "1" + ] + }, + { + "name": "numeric-kernels", + "entry": "workloads/numeric-kernels.ts", + "args": [ + "0.5" + ] + }, + { + "name": "validate-errors", + "entry": "workloads/validate-errors.ts", + "args": [ + "1" + ] + }, + { + "name": "csv-numbers", + "entry": "workloads/csv-numbers.ts", + "args": [ + "1" + ] + }, + { + "name": "log-summary", + "entry": "../builds/log-summary/main.ts", + "input": "log-summary", + "args": [ + "{input}", + "--min-status", + "200", + "--limit", + "12" + ] + }, + { + "name": "inventory-report", + "entry": "../builds/inventory-report/main.ts", + "input": "inventory-report", + "args": [ + "{input}" + ] + } + ] +} diff --git a/benchmarks/runtime/workloads/alloc-trees.ts b/benchmarks/runtime/workloads/alloc-trees.ts new file mode 100644 index 000000000..16329a0d5 --- /dev/null +++ b/benchmarks/runtime/workloads/alloc-trees.ts @@ -0,0 +1,46 @@ +// Allocation-heavy code: binary trees built and discarded, plus a +// reducer that produces immutable state updates with object spread. +class TreeNode { + readonly left: TreeNode | null; + readonly right: TreeNode | null; + constructor(left: TreeNode | null, right: TreeNode | null) { + this.left = left; + this.right = right; + } + check(): number { + return 1 + (this.left === null ? 0 : this.left.check()) + (this.right === null ? 0 : this.right.check()); + } +} +function build(depth: number): TreeNode { + return depth === 0 ? new TreeNode(null, null) : new TreeNode(build(depth - 1), build(depth - 1)); +} + +interface State { + count: number; + total: number; + last: string; + history: number[]; +} +interface Action { + type: string; + amount: number; +} +function reduce(state: State, action: Action): State { + if (action.type === "add") return { ...state, count: state.count + 1, total: state.total + action.amount, last: action.type }; + if (action.type === "reset") return { ...state, total: 0, last: action.type, history: [...state.history.slice(-4), state.total] }; + return { ...state, last: action.type }; +} + +const scale = Number(process.argv[2] ?? "1"); +const maxDepth = 14; +let checks = 0; +for (let depth = 4; depth <= maxDepth; depth += 2) { + const iterations = Math.floor((1 << (maxDepth - depth + 4)) * scale); + for (let i = 0; i < iterations; i++) checks += build(depth).check(); +} +const longLived = build(maxDepth); +let state: State = { count: 0, total: 0, last: "", history: [] }; +for (let i = 0; i < Math.floor(300000 * scale); i++) { + state = reduce(state, { type: i % 97 === 0 ? "reset" : i % 3 === 0 ? "noop" : "add", amount: i % 13 }); +} +console.log("checks", checks, "long", longLived.check(), "state", state.count, state.total, state.history.join(",")); diff --git a/benchmarks/runtime/workloads/ast-interp.ts b/benchmarks/runtime/workloads/ast-interp.ts new file mode 100644 index 000000000..fb7db5d28 --- /dev/null +++ b/benchmarks/runtime/workloads/ast-interp.ts @@ -0,0 +1,199 @@ +// A tokenizer, recursive-descent parser, and class-based tree-walking +// evaluator for a small expression language with variables. +abstract class Expr { + abstract evaluate(env: Map): number; +} +class Num extends Expr { + readonly value: number; + constructor(value: number) { + super(); + this.value = value; + } + evaluate(_env: Map): number { + return this.value; + } +} +class Var extends Expr { + readonly name: string; + constructor(name: string) { + super(); + this.name = name; + } + evaluate(env: Map): number { + return env.get(this.name) ?? 0; + } +} +class Binary extends Expr { + readonly op: string; + readonly left: Expr; + readonly right: Expr; + constructor(op: string, left: Expr, right: Expr) { + super(); + this.op = op; + this.left = left; + this.right = right; + } + evaluate(env: Map): number { + const l = this.left.evaluate(env); + const r = this.right.evaluate(env); + switch (this.op) { + case "+": + return l + r; + case "-": + return l - r; + case "*": + return l * r; + case "/": + return r === 0 ? 0 : l / r; + default: + return l % (r === 0 ? 1 : r); + } + } +} +class Call extends Expr { + readonly name: string; + readonly args: Expr[]; + constructor(name: string, args: Expr[]) { + super(); + this.name = name; + this.args = args; + } + evaluate(env: Map): number { + const values = this.args.map((arg) => arg.evaluate(env)); + if (this.name === "max") return Math.max(...values); + if (this.name === "min") return Math.min(...values); + return values.reduce((a, b) => a + b, 0); + } +} + +type TokenKind = "num" | "ident" | "op" | "lparen" | "rparen" | "comma" | "end"; +interface Token { + kind: TokenKind; + text: string; +} + +function isDigit(c: number): boolean { + return (c >= 48 && c <= 57) || c === 46; +} +function isIdent(c: number): boolean { + return (c >= 97 && c <= 122) || c === 95; +} +function tokenize(source: string): Token[] { + const tokens: Token[] = []; + let i = 0; + while (i < source.length) { + const c = source.charCodeAt(i); + if (c === 32) { + i++; + } else if (c >= 48 && c <= 57) { + const start = i; + while (i < source.length && isDigit(source.charCodeAt(i))) i++; + tokens.push({ kind: "num", text: source.slice(start, i) }); + } else if (isIdent(c)) { + const start = i; + while (i < source.length && isIdent(source.charCodeAt(i))) i++; + tokens.push({ kind: "ident", text: source.slice(start, i) }); + } else if (c === 40) { + tokens.push({ kind: "lparen", text: "(" }); + i++; + } else if (c === 41) { + tokens.push({ kind: "rparen", text: ")" }); + i++; + } else if (c === 44) { + tokens.push({ kind: "comma", text: "," }); + i++; + } else { + tokens.push({ kind: "op", text: source[i]! }); + i++; + } + } + tokens.push({ kind: "end", text: "" }); + return tokens; +} + +class Parser { + private pos = 0; + private readonly tokens: Token[]; + constructor(tokens: Token[]) { + this.tokens = tokens; + } + private peek(): Token { + return this.tokens[this.pos]!; + } + private next(): Token { + return this.tokens[this.pos++]!; + } + parseExpr(): Expr { + let left = this.parseTerm(); + while (this.peek().kind === "op" && (this.peek().text === "+" || this.peek().text === "-")) { + const op = this.next().text; + left = new Binary(op, left, this.parseTerm()); + } + return left; + } + private parseTerm(): Expr { + let left = this.parseFactor(); + while (this.peek().kind === "op" && (this.peek().text === "*" || this.peek().text === "/" || this.peek().text === "%")) { + const op = this.next().text; + left = new Binary(op, left, this.parseFactor()); + } + return left; + } + private parseFactor(): Expr { + const token = this.next(); + if (token.kind === "num") return new Num(Number(token.text)); + if (token.kind === "lparen") { + const inner = this.parseExpr(); + this.next(); + return inner; + } + if (this.peek().kind === "lparen") { + this.next(); + const args: Expr[] = []; + while (this.peek().kind !== "rparen") { + args.push(this.parseExpr()); + if (this.peek().kind === "comma") this.next(); + } + this.next(); + return new Call(token.text, args); + } + return new Var(token.text); + } +} + +let seed = 7; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const names = ["alpha", "beta", "gamma", "delta", "x", "y"]; +const ops = ["+", "-", "*", "/", "%"]; +function generate(depth: number): string { + const r = random(); + if (depth === 0 || r < 0.2) return r < 0.1 ? String(Math.floor(random() * 100)) : names[Math.floor(random() * names.length)]!; + if (r < 0.3) return `max(${generate(depth - 1)}, ${generate(depth - 1)}, ${generate(depth - 1)})`; + if (r < 0.4) return `(${generate(depth - 1)})`; + return `${generate(depth - 1)} ${ops[Math.floor(random() * ops.length)]!} ${generate(depth - 1)}`; +} + +const scale = Number(process.argv[2] ?? "1"); +const sources: string[] = []; +for (let i = 0; i < Math.floor(400 * scale); i++) sources.push(generate(7)); +const env = new Map(); +let total = 0; +let chars = 0; +for (const source of sources) { + const tree = new Parser(tokenize(source)).parseExpr(); + chars += source.length; + for (let step = 0; step < 40; step++) { + env.set("alpha", step); + env.set("beta", step * 2 + 1); + env.set("gamma", 3); + env.set("delta", step % 7); + env.set("x", 1.5); + env.set("y", -2); + const value = tree.evaluate(env); + if (Number.isFinite(value)) total += value; + } +} +console.log("sources", sources.length, "chars", chars, "total", total.toFixed(6)); diff --git a/benchmarks/runtime/workloads/async-pipeline.ts b/benchmarks/runtime/workloads/async-pipeline.ts new file mode 100644 index 000000000..97e23857c --- /dev/null +++ b/benchmarks/runtime/workloads/async-pipeline.ts @@ -0,0 +1,47 @@ +// Async application code: request handlers awaiting cached and computed +// values, Promise.all fan-out, and sequential awaits in loops. +interface User { + id: number; + name: string; + score: number; +} + +const cache = new Map(); +async function loadUser(id: number): Promise { + const cached = cache.get(id); + if (cached !== undefined) return cached; + const user = { id, name: "user-" + id, score: (id * 37) % 101 }; + cache.set(id, user); + return user; +} +async function authorize(user: User): Promise { + return user.score % 7 !== 0; +} +async function handle(requestId: number): Promise { + const user = await loadUser(requestId % 3000); + if (!(await authorize(user))) return 0; + const requests: Promise[] = [loadUser((requestId + 1) % 3000), loadUser((requestId + 7) % 3000), loadUser((requestId + 13) % 3000)]; + const friends = await Promise.all(requests); + let sum = user.score; + for (const friend of friends) sum += friend.score; + return sum; +} +async function batch(start: number, size: number): Promise { + let total = 0; + for (let i = 0; i < size; i++) total += await handle(start + i); + return total; +} + +async function main(): Promise { + const scale = Number(process.argv[2] ?? "1"); + const batches = Math.floor(200 * scale); + let total = 0; + for (let round = 0; round < batches; round += 10) { + const work: Promise[] = []; + for (let b = 0; b < 10; b++) work.push(batch((round + b) * 100, 100)); + const results = await Promise.all(work); + for (const r of results) total += r; + } + console.log("handled", batches * 100, "total", total); +} +void main(); diff --git a/benchmarks/runtime/workloads/csv-numbers.ts b/benchmarks/runtime/workloads/csv-numbers.ts new file mode 100644 index 000000000..c4ec05dae --- /dev/null +++ b/benchmarks/runtime/workloads/csv-numbers.ts @@ -0,0 +1,43 @@ +// CSV parsing and numeric formatting: split, Number(), parseInt, +// toFixed, String(n), and string assembly of a report. +let seed = 1001; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function makeCsv(rows: number): string { + const lines: string[] = ["date,store,units,unit_price,discount"]; + for (let i = 0; i < rows; i++) { + const day = 1 + (i % 28); + lines.push( + `2026-${String(1 + (i % 12)).padStart(2, "0")}-${String(day).padStart(2, "0")},store-${Math.floor(random() * 40)},${Math.floor(random() * 50)},${(random() * 200).toFixed(2)},${(random() * 0.3).toFixed(3)}`, + ); + } + return lines.join("\n"); +} + +const scale = Number(process.argv[2] ?? "1"); +const csv = makeCsv(Math.floor(150000 * scale)); +const totals = new Map(); +const months: number[] = new Array(13).fill(0); +let units = 0; +let formatted = 0; +for (let round = 0; round < 2; round++) { + const lines = csv.split("\n"); + for (let i = 1; i < lines.length; i++) { + const cols = lines[i]!.split(","); + const month = parseInt(cols[0]!.slice(5, 7), 10); + const count = Number(cols[2]); + const price = Number(cols[3]); + const discount = Number(cols[4]); + const revenue = count * price * (1 - discount); + units += count; + months[month] = months[month]! + revenue; + totals.set(cols[1]!, (totals.get(cols[1]!) ?? 0) + revenue); + formatted += (revenue.toFixed(2) + "|" + String(count)).length; + } +} +const stores = [...totals.keys()].sort(); +const report = stores.map((s) => `${s}: ${totals.get(s)!.toFixed(2)}`).join("\n"); +console.log(report.split("\n").slice(0, 5).join("\n")); +console.log("units", units, "formatted", formatted, "jan", months[1]!.toFixed(2), "dec", months[12]!.toFixed(2)); diff --git a/benchmarks/runtime/workloads/functional-pipeline.ts b/benchmarks/runtime/workloads/functional-pipeline.ts new file mode 100644 index 000000000..1b246081e --- /dev/null +++ b/benchmarks/runtime/workloads/functional-pipeline.ts @@ -0,0 +1,56 @@ +// Functional collection pipelines: map/filter/reduce/flatMap chains, +// closures capturing configuration, and small intermediate objects. +interface Event { + user: number; + kind: string; + value: number; + time: number; +} +interface Point { + x: number; + y: number; +} +interface Scored { + user: number; + score: number; +} + +let seed = 4242; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const kinds = ["view", "click", "purchase", "share"]; +function makeEvents(count: number): Event[] { + const events: Event[] = []; + for (let i = 0; i < count; i++) { + events.push({ user: Math.floor(random() * 5000), kind: kinds[Math.floor(random() * kinds.length)]!, value: Math.floor(random() * 1000), time: i }); + } + return events; +} + +function makeScorer(weight: number, bonus: number): (e: Event) => number { + return (e) => (e.kind === "purchase" ? e.value * weight + bonus : e.kind === "click" ? weight : 0); +} + +const scale = Number(process.argv[2] ?? "1"); +const events = makeEvents(Math.floor(200000 * scale)); +let total = 0; +for (let round = 0; round < 5; round++) { + const scorer = makeScorer(round + 1, round * 10); + const scored: Scored[] = events + .filter((e) => e.time % 5 !== round) + .map((e) => ({ user: e.user, score: scorer(e) })) + .filter((s) => s.score > 0); + total += scored.reduce((sum, s) => sum + s.score, 0); + const pairs = events.slice(0, 20000).flatMap((e) => (e.kind === "share" ? [e.user, e.value] : [])); + total += pairs.length; + const points: Point[] = events.slice(0, 50000).map((e) => ({ x: e.value / 10, y: e.user / 100 })); + const centroid = points.reduce((acc: Point, p: Point): Point => ({ x: acc.x + p.x, y: acc.y + p.y }), { x: 0, y: 0 }); + total += Math.round(centroid.x + centroid.y); + const some = events.some((e) => e.value === 999 - round); + const every = events.every((e) => e.value >= 0); + const found = events.findIndex((e) => e.user === 4999 - round); + total += (some ? 1 : 0) + (every ? 1 : 0) + found; +} +console.log("total", total); diff --git a/benchmarks/runtime/workloads/json-records.ts b/benchmarks/runtime/workloads/json-records.ts new file mode 100644 index 000000000..c1e6bb866 --- /dev/null +++ b/benchmarks/runtime/workloads/json-records.ts @@ -0,0 +1,102 @@ +// JSON round trips of typed records: build, stringify, parse back into a +// declared shape, aggregate, and stringify the summary. +interface LineItem { + sku: string; + quantity: number; + price: number; +} +interface Order { + id: number; + customer: string; + region: string; + priority: boolean; + items: LineItem[]; + tags: string[]; +} +interface Summary { + region: string; + orders: number; + revenue: number; + topSku: string; +} +interface RegionTotals { + orders: number; + revenue: number; + skus: Map; +} + +let seed = 12345; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +const regions = ["north", "south", "east", "west", "central"]; +function makeOrders(count: number): Order[] { + const orders: Order[] = []; + for (let i = 0; i < count; i++) { + const items: LineItem[] = []; + const n = 1 + Math.floor(random() * 6); + for (let j = 0; j < n; j++) { + items.push({ + sku: "sku-" + Math.floor(random() * 500), + quantity: 1 + Math.floor(random() * 9), + price: Math.round(random() * 10000) / 100, + }); + } + orders.push({ + id: i, + customer: "customer-" + Math.floor(random() * 2000), + region: regions[i % regions.length]!, + priority: random() < 0.2, + items, + tags: random() < 0.5 ? ["web"] : ["store", "promo"], + }); + } + return orders; +} + +function summarize(orders: Order[]): Summary[] { + const byRegion = new Map(); + for (const order of orders) { + let entry = byRegion.get(order.region); + if (entry === undefined) { + entry = { orders: 0, revenue: 0, skus: new Map() }; + byRegion.set(order.region, entry); + } + entry.orders++; + for (const item of order.items) { + entry.revenue += item.quantity * item.price; + entry.skus.set(item.sku, (entry.skus.get(item.sku) ?? 0) + item.quantity); + } + } + const result: Summary[] = []; + for (const [region, entry] of byRegion) { + let topSku = ""; + let topCount = -1; + for (const [sku, count] of entry.skus) { + if (count > topCount || (count === topCount && sku < topSku)) { + topSku = sku; + topCount = count; + } + } + result.push({ region, orders: entry.orders, revenue: Math.round(entry.revenue * 100) / 100, topSku }); + } + result.sort((a, b) => (a.region < b.region ? -1 : a.region > b.region ? 1 : 0)); + return result; +} + +const scale = Number(process.argv[2] ?? "1"); +const orders = makeOrders(Math.floor(4000 * scale)); +let checksum = 0; +let last = ""; +for (let round = 0; round < 6; round++) { + const text = JSON.stringify(orders); + checksum += text.length; + const parsed = JSON.parse(text) as Order[]; + const summary = summarize(parsed); + last = JSON.stringify(summary); + checksum += last.length; +} +console.log(last); +console.log("checksum", checksum); diff --git a/benchmarks/runtime/workloads/numeric-kernels.ts b/benchmarks/runtime/workloads/numeric-kernels.ts new file mode 100644 index 000000000..b2b1db3cd --- /dev/null +++ b/benchmarks/runtime/workloads/numeric-kernels.ts @@ -0,0 +1,74 @@ +// Integer- and float-heavy kernels: sieve, hashing, matrix multiply on +// number[][], histogram on plain arrays, and a Float64Array smoothing pass. +function sieve(limit: number): number { + const composite: boolean[] = new Array(limit + 1).fill(false); + let count = 0; + for (let i = 2; i <= limit; i++) { + if (composite[i]) continue; + count++; + for (let j = i * i; j <= limit; j += i) composite[j] = true; + } + return count; +} +function fnv(values: number[]): number { + let h = 2166136261; + for (let i = 0; i < values.length; i++) { + h ^= values[i]! & 0xff; + h = Math.imul(h, 16777619) >>> 0; + } + return h; +} +function matmul(a: number[][], b: number[][], n: number): number[][] { + const c: number[][] = []; + for (let i = 0; i < n; i++) { + const row: number[] = new Array(n).fill(0); + const ai = a[i]!; + for (let k = 0; k < n; k++) { + const aik = ai[k]!; + const bk = b[k]!; + for (let j = 0; j < n; j++) row[j] = row[j]! + aik * bk[j]!; + } + c.push(row); + } + return c; +} + +const scale = Number(process.argv[2] ?? "1"); +const primes = sieve(Math.floor(2000000 * scale)); +const data: number[] = []; +let x = 1; +for (let i = 0; i < Math.floor(1000000 * scale); i++) { + x = (Math.imul(x, 48271) + 11) & 0x7fffffff; + data.push(x % 1000); +} +let hash = 0; +for (let r = 0; r < 4; r++) hash = (hash + fnv(data)) >>> 0; +const histogram: number[] = new Array(100).fill(0); +for (let i = 0; i < data.length; i++) { + const bucket = data[i]! % 100; + histogram[bucket] = histogram[bucket]! + 1; +} +const n = Math.floor(120 * Math.sqrt(scale)); +const a: number[][] = []; +const b: number[][] = []; +for (let i = 0; i < n; i++) { + const ra: number[] = []; + const rb: number[] = []; + for (let j = 0; j < n; j++) { + ra.push(((i * j) % 7) - 3); + rb.push(((i + j) % 5) - 2); + } + a.push(ra); + b.push(rb); +} +const c = matmul(a, b, n); +let trace = 0; +for (let i = 0; i < n; i++) trace += c[i]![i]!; +const signal = new Float64Array(data.length); +for (let i = 0; i < data.length; i++) signal[i] = data[i]! / 7; +let smooth = 0; +for (let pass = 0; pass < 3; pass++) { + for (let i = 1; i < signal.length - 1; i++) signal[i] = (signal[i - 1]! + signal[i]! * 2 + signal[i + 1]!) / 4; +} +for (let i = 0; i < signal.length; i += 1000) smooth += signal[i]!; +console.log("primes", primes, "hash", hash, "hist", histogram[0], histogram[99], "trace", trace, "smooth", smooth.toFixed(6)); diff --git a/benchmarks/runtime/workloads/records-sort.ts b/benchmarks/runtime/workloads/records-sort.ts new file mode 100644 index 000000000..d58d9bdb7 --- /dev/null +++ b/benchmarks/runtime/workloads/records-sort.ts @@ -0,0 +1,80 @@ +// Business-record processing: multi-key sorts with comparator closures, +// group-by into Maps, and reductions over the groups. +interface Employee { + id: number; + name: string; + department: string; + level: number; + salary: number; + startYear: number; +} + +let seed = 2024; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const departments = ["eng", "sales", "support", "finance", "legal", "design", "ops"]; +const first = ["ana", "bo", "cy", "dee", "eli", "fay", "gus", "hal", "ivy", "jon"]; +const last = ["smith", "jones", "lee", "khan", "garcia", "chen", "novak", "silva"]; + +function makeEmployees(count: number): Employee[] { + const result: Employee[] = []; + for (let i = 0; i < count; i++) { + result.push({ + id: i, + name: first[Math.floor(random() * first.length)]! + " " + last[Math.floor(random() * last.length)]!, + department: departments[Math.floor(random() * departments.length)]!, + level: 1 + Math.floor(random() * 7), + salary: 40000 + Math.floor(random() * 160000), + startYear: 1995 + Math.floor(random() * 30), + }); + } + return result; +} + +function compareBy(keys: string[]): (a: Employee, b: Employee) => number { + return (a, b) => { + for (const key of keys) { + let d = 0; + if (key === "department") d = a.department < b.department ? -1 : a.department > b.department ? 1 : 0; + else if (key === "level") d = b.level - a.level; + else if (key === "salary") d = b.salary - a.salary; + else if (key === "name") d = a.name < b.name ? -1 : a.name > b.name ? 1 : 0; + else d = a.id - b.id; + if (d !== 0) return d; + } + return 0; + }; +} + +const scale = Number(process.argv[2] ?? "1"); +const employees = makeEmployees(Math.floor(60000 * scale)); +let checksum = 0; +const orders = [ + ["department", "level", "salary", "id"], + ["name", "id"], + ["salary", "id"], + ["level", "name", "id"], +]; +for (const order of orders) { + const sorted = employees.slice().sort(compareBy(order)); + for (let i = 0; i < sorted.length; i += 997) checksum = (checksum * 31 + sorted[i]!.id) % 1000000007; +} +const groups = new Map(); +for (const e of employees) { + const key = e.department + "/" + e.level; + const group = groups.get(key); + if (group === undefined) groups.set(key, [e]); + else group.push(e); +} +const lines: string[] = []; +for (const [key, group] of groups) { + const total = group.reduce((sum, e) => sum + e.salary, 0); + const veterans = group.filter((e) => e.startYear < 2010).length; + const top = group.reduce((best, e) => (e.salary > best.salary ? e : best), group[0]!); + lines.push(`${key} n=${group.length} avg=${Math.round(total / group.length)} vets=${veterans} top=${top.id}`); +} +lines.sort(); +for (const line of lines.slice(0, 12)) console.log(line); +console.log("groups", lines.length, "checksum", checksum); diff --git a/benchmarks/runtime/workloads/regex-logs.ts b/benchmarks/runtime/workloads/regex-logs.ts new file mode 100644 index 000000000..729639706 --- /dev/null +++ b/benchmarks/runtime/workloads/regex-logs.ts @@ -0,0 +1,49 @@ +// Regex-driven log processing: match with capture groups, test, and replace. +let seed = 99; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} + +const methods = ["GET", "POST", "PUT", "DELETE"]; +const paths = ["/api/users", "/api/orders", "/static/app.js", "/health", "/api/items/search"]; +function makeLines(count: number): string[] { + const lines: string[] = []; + for (let i = 0; i < count; i++) { + const ip = `10.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}.${Math.floor(random() * 255)}`; + const method = methods[Math.floor(random() * methods.length)]!; + const path = paths[Math.floor(random() * paths.length)]! + (random() < 0.3 ? "?id=" + Math.floor(random() * 1000) : ""); + const status = random() < 0.9 ? 200 : random() < 0.5 ? 404 : 500; + const ms = Math.floor(random() * 900); + const agent = random() < 0.5 ? "Mozilla/5.0 (Macintosh)" : "curl/8.4.0"; + lines.push(`${ip} - - [07/Oct/2026:12:${String(i % 60).padStart(2, "0")}:00 +0000] "${method} ${path} HTTP/1.1" ${status} ${ms} "${agent}"`); + } + return lines; +} + +const linePattern = /^(\d+\.\d+\.\d+\.\d+) - - \[([^\]]+)\] "(\w+) ([^ ?"]+)(?:\?([^ "]*))? HTTP\/1\.1" (\d{3}) (\d+) "([^"]*)"$/; +const botPattern = /curl|bot|spider/i; +const digits = /\d+/g; + +const scale = Number(process.argv[2] ?? "1"); +const lines = makeLines(Math.floor(30000 * scale)); +const counts = new Map(); +let errors = 0; +let bots = 0; +let totalMs = 0; +let redactedLength = 0; +for (let round = 0; round < 3; round++) { + for (const line of lines) { + const m = linePattern.exec(line); + if (m === null) continue; + const key = m[3]! + " " + m[4]!; + counts.set(key, (counts.get(key) ?? 0) + 1); + if (m[6] !== "200") errors++; + if (botPattern.test(m[8]!)) bots++; + totalMs += Number(m[7]); + redactedLength += line.replace(digits, "#").length; + } +} +const keys = [...counts.keys()].sort(); +for (const key of keys) console.log(key, counts.get(key)); +console.log("errors", errors, "bots", bots, "ms", totalMs, "redacted", redactedLength); diff --git a/benchmarks/runtime/workloads/template-render.ts b/benchmarks/runtime/workloads/template-render.ts new file mode 100644 index 000000000..d59fbc15b --- /dev/null +++ b/benchmarks/runtime/workloads/template-render.ts @@ -0,0 +1,73 @@ +// Server-side HTML rendering: template literals, escaping, joins, and +// conditional fragments over a list of typed view models. +interface Product { + id: number; + title: string; + description: string; + price: number; + tags: string[]; + inStock: boolean; + rating: number; +} + +let seed = 31337; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const words = ["fast", "durable", "bold", "eco", '"quoted"', "light", "smart", "R&D", "classic", "modern"]; +function sentence(n: number): string { + const parts: string[] = []; + for (let i = 0; i < n; i++) parts.push(words[Math.floor(random() * words.length)]!); + return parts.join(" "); +} +function makeProducts(count: number): Product[] { + const products: Product[] = []; + for (let i = 0; i < count; i++) { + products.push({ + id: i, + title: sentence(3), + description: sentence(20), + price: Math.round(random() * 100000) / 100, + tags: [words[i % words.length]!, words[(i * 7) % words.length]!], + inStock: random() < 0.8, + rating: Math.round(random() * 50) / 10, + }); + } + return products; +} + +function escapeHtml(text: string): string { + return text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} +function stars(rating: number): string { + let out = ""; + for (let i = 1; i <= 5; i++) out += i <= Math.round(rating) ? "*" : "-"; + return out; +} +function renderProduct(p: Product): string { + const tags = p.tags.map((t) => `
  • ${escapeHtml(t)}
  • `).join(""); + return `
    +

    ${escapeHtml(p.title)}

    +

    ${escapeHtml(p.description)}

    + $${p.price.toFixed(2)} + ${stars(p.rating)} +
      ${tags}
    + ${p.inStock ? "" : "Sold out"} +
    `; +} +function renderPage(products: Product[], page: number): string { + const body = products.map(renderProduct).join("\n"); + return `Page ${page}${body}`; +} + +const scale = Number(process.argv[2] ?? "1"); +const products = makeProducts(Math.floor(2000 * scale)); +let bytes = 0; +let hash = 0; +for (let page = 0; page < 10; page++) { + const html = renderPage(products, page); + bytes += html.length; + for (let i = 0; i < html.length; i += 101) hash = (hash * 33 + html.charCodeAt(i)) % 2147483647; +} +console.log("bytes", bytes, "hash", hash); diff --git a/benchmarks/runtime/workloads/validate-errors.ts b/benchmarks/runtime/workloads/validate-errors.ts new file mode 100644 index 000000000..387014435 --- /dev/null +++ b/benchmarks/runtime/workloads/validate-errors.ts @@ -0,0 +1,76 @@ +// Validation layers that throw and catch errors for a fraction of inputs, +// with deep call chains of small functions on the success path. +class ValidationError extends Error { + readonly field: string; + constructor(field: string, message: string) { + super(message); + this.field = field; + } +} +interface Signup { + email: string; + age: number; + name: string; + password: string; +} + +let seed = 8080; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function makeSignup(i: number): Signup { + const bad = random(); + return { + email: bad < 0.05 ? "broken" : `user${i}@example.com`, + age: bad > 0.97 ? -1 : 18 + Math.floor(random() * 60), + name: bad > 0.95 && bad <= 0.97 ? " " : "Name " + i, + password: random() < 0.03 ? "short" : "correct horse battery " + i, + }; +} + +function requireText(field: string, value: string, min: number): string { + if (value.length < min) throw new ValidationError(field, `${field} must have at least ${min} characters`); + return value; +} +function requireEmail(value: string): string { + const at = value.indexOf("@"); + if (at <= 0 || value.indexOf(".", at) < 0) throw new ValidationError("email", "invalid email"); + return value.toLowerCase(); +} +function requireRange(field: string, value: number, min: number, max: number): number { + if (value < min || value > max) throw new ValidationError(field, `${field} out of range`); + return value; +} +function normalize(input: Signup): Signup { + return { + email: requireEmail(input.email), + age: requireRange("age", input.age, 13, 120), + name: requireText("name", input.name.trim(), 1), + password: requireText("password", input.password, 8), + }; +} +function score(s: Signup): number { + return s.email.length + s.age + s.name.length + s.password.length; +} + +const scale = Number(process.argv[2] ?? "1"); +const inputs: Signup[] = []; +for (let i = 0; i < Math.floor(200000 * scale); i++) inputs.push(makeSignup(i)); +const failures = new Map(); +let accepted = 0; +let total = 0; +for (let round = 0; round < 3; round++) { + for (const input of inputs) { + try { + const s = normalize(input); + accepted++; + total += score(s); + } catch (error) { + if (error instanceof ValidationError) failures.set(error.field, (failures.get(error.field) ?? 0) + 1); + else throw error; + } + } +} +console.log("accepted", accepted, "total", total); +for (const key of [...failures.keys()].sort()) console.log(key, failures.get(key)); diff --git a/benchmarks/runtime/workloads/word-graph.ts b/benchmarks/runtime/workloads/word-graph.ts new file mode 100644 index 000000000..e40e9e638 --- /dev/null +++ b/benchmarks/runtime/workloads/word-graph.ts @@ -0,0 +1,61 @@ +// Map/Set-heavy text and graph processing: word frequencies, bigrams, +// and breadth-first search over a string-keyed adjacency map. +let seed = 555; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const syllables = ["ka", "lo", "mi", "ne", "ru", "sa", "to", "vi", "ze", "po", "qu", "ex"]; +function word(): string { + const n = 1 + Math.floor(random() * 3); + let w = ""; + for (let i = 0; i < n; i++) w += syllables[Math.floor(random() * syllables.length)]!; + return w; +} + +const scale = Number(process.argv[2] ?? "1"); +const text: string[] = []; +for (let i = 0; i < Math.floor(300000 * scale); i++) text.push(word()); + +const freq = new Map(); +for (const w of text) freq.set(w, (freq.get(w) ?? 0) + 1); +const bigrams = new Map(); +for (let i = 1; i < text.length; i++) { + const key = text[i - 1]! + " " + text[i]!; + bigrams.set(key, (bigrams.get(key) ?? 0) + 1); +} +const top = [...freq.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5); +console.log("words", freq.size, "bigrams", bigrams.size, "top", top.map((e) => e[0] + ":" + e[1]).join(",")); + +const graph = new Map(); +for (let i = 1; i < text.length; i += 3) { + const from = text[i - 1]!; + const to = text[i]!; + const edges = graph.get(from); + if (edges === undefined) graph.set(from, [to]); + else if (edges.length < 8) edges.push(to); +} +let reachable = 0; +let depthSum = 0; +const starts = [...graph.keys()].sort().slice(0, 60); +for (const start of starts) { + const seen = new Set([start]); + let frontier = [start]; + let depth = 0; + while (frontier.length > 0) { + const next: string[] = []; + for (const node of frontier) { + for (const neighbor of graph.get(node) ?? []) { + if (!seen.has(neighbor)) { + seen.add(neighbor); + next.push(neighbor); + } + } + } + frontier = next; + depth++; + } + reachable += seen.size; + depthSum += depth; +} +console.log("reachable", reachable, "depth", depthSum); diff --git a/package.json b/package.json index 0efe1a8a0..813302758 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "build:fresh": "rm -rf packages/compiler/dist packages/cli/dist node_modules/.cache/scriptc-tsc && pnpm build", "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-cli.mts", "bench:builds": "node scripts/bench-builds.mjs", + "bench:runtime": "node scripts/bench-runtime.mjs", "bench:ast-access": "node scripts/bench-ast-access.mjs", "bench:backend-analysis": "node scripts/bench-backend-analysis.mjs", "test": "vitest run", diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs new file mode 100644 index 000000000..3753473c1 --- /dev/null +++ b/scripts/bench-runtime.mjs @@ -0,0 +1,424 @@ +/* Runtime performance A/B harness for compiled programs. + * + * Builds every workload in benchmarks/runtime/workloads.json with one or two + * scriptc checkouts, verifies each executable against Node byte-for-byte + * (stdout, stderr, exit status), then times interleaved launches and reports + * per-workload medians with bootstrap confidence intervals for the + * candidate/baseline ratio. Executables that diverge from Node are never + * timed. A per-host advisory lock serializes measurements so concurrent + * agents on one machine do not disturb each other's numbers. */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const suiteRoot = join(repoRoot, "benchmarks/runtime"); + +const { values } = parseArgs({ + allowNegative: true, + options: { + candidate: { type: "string", default: repoRoot }, + baseline: { type: "string" }, + workloads: { type: "string" }, + runs: { type: "string", default: "15" }, + warmup: { type: "string", default: "2" }, + timeout: { type: "string", default: "120" }, + json: { type: "string" }, + "no-lock": { type: "boolean", default: false }, + node: { type: "boolean", default: true }, + "keep": { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help) { + console.log(`Usage: node scripts/bench-runtime.mjs [--candidate=] [--baseline=] + [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--json=] [--no-lock] [--no-node] [--keep] + +Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native +artifacts for this host. Without --baseline, reports absolute times only. Node is +timed in the same interleaved loop as the reference to beat (--no-node skips it).`); + process.exit(0); +} +const runs = Number(values.runs); +const warmup = Number(values.warmup); +const timeoutMs = Number(values.timeout) * 1000; +assert.ok(Number.isInteger(runs) && runs >= 3 && runs <= 200, "--runs must be 3..200"); +assert.ok(Number.isInteger(warmup) && warmup >= 0 && warmup <= 20, "--warmup must be 0..20"); + +const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8")); +const selected = values.workloads ? new Set(values.workloads.split(",")) : null; +const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name)); +if (selected) { + for (const name of selected) + assert.ok( + manifest.workloads.some((w) => w.name === name), + `unknown workload ${name}`, + ); +} + +const work = mkdtempSync( + join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-runtime-"), +); + +/* ── inputs for file-processing applications ──────────────────────────── */ +function generateInput(kind) { + const path = join(work, `${kind}.input`); + if (kind === "log-summary") { + writeFileSync( + path, + Array.from( + { length: 300_000 }, + (_, i) => + `/route/${i % 37}?request=${i} ${i % 13 === 0 ? 500 : 200} ${i % 250} ${200 + (i % 4000)}`, + ).join("\n") + "\n", + ); + } else if (kind === "inventory-report") { + writeFileSync( + path, + Array.from({ length: 300_000 }, (_, i) => + ["sku-" + i, "category-" + (i % 17), "product " + i, i % 40, 199 + (i % 5000)].join("\t"), + ).join("\n") + "\n", + ); + } else throw new Error(`unknown input kind ${kind}`); + return path; +} + +/* ── advisory lock ─────────────────────────────────────────────────────── */ +const lockDir = join( + process.platform === "win32" ? tmpdir() : "/tmp", + "scriptc-bench-runtime.lock", +); +function acquireLock() { + const started = Date.now(); + let announced = false; + for (;;) { + try { + mkdirSync(lockDir); + writeFileSync(join(lockDir, "pid"), String(process.pid)); + return; + } catch (error) { + if (error.code !== "EEXIST") throw error; + let owner = NaN; + try { + owner = Number(readFileSync(join(lockDir, "pid"), "utf8")); + } catch {} + let alive = false; + if (Number.isInteger(owner)) { + try { + process.kill(owner, 0); + alive = true; + } catch {} + } + if (!alive && Date.now() - started > 2000) { + rmSync(lockDir, { recursive: true, force: true }); + continue; + } + if (!announced) { + process.stderr.write(`waiting for benchmark lock held by pid ${owner}\n`); + announced = true; + } + spawnSync("sleep", ["2"]); + } + } +} +function releaseLock() { + rmSync(lockDir, { recursive: true, force: true }); +} + +/* ── process helpers ───────────────────────────────────────────────────── */ +function run(command, args, options = {}) { + const start = process.hrtime.bigint(); + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + timeout: options.timeout ?? timeoutMs, + env: options.env ?? process.env, + cwd: options.cwd, + }); + const ms = Number(process.hrtime.bigint() - start) / 1e6; + if (result.error && result.error.code !== "ETIMEDOUT") throw result.error; + return { + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + status: result.status, + signal: result.signal, + timedOut: result.error?.code === "ETIMEDOUT" || result.signal === "SIGTERM", + ms, + }; +} + +function peakRss(binary, args) { + if (process.platform !== "darwin" && process.platform !== "linux") return null; + if (!existsSync("/usr/bin/time")) return null; + const timeArgs = + process.platform === "darwin" ? ["-l", binary, ...args] : ["-f", "rss_kib=%M", binary, ...args]; + const result = run("/usr/bin/time", timeArgs); + const match = + process.platform === "darwin" + ? /(\d+)\s+maximum resident set size/.exec(result.stderr) + : /rss_kib=(\d+)/.exec(result.stderr); + if (!match) return null; + return Number(match[1]) * (process.platform === "darwin" ? 1 : 1024); +} + +/* ── statistics ────────────────────────────────────────────────────────── */ +function median(xs) { + const s = [...xs].sort((a, b) => a - b); + const m = s.length >> 1; + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} +function makeRandom(seed) { + let state = seed >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; +} +function bootstrapRatio(candidate, baseline, resamples = 2000) { + const random = makeRandom(0x5eed); + const ratios = []; + const pick = (xs) => + Array.from({ length: xs.length }, () => xs[Math.floor(random() * xs.length)]); + for (let i = 0; i < resamples; i++) ratios.push(median(pick(candidate)) / median(pick(baseline))); + ratios.sort((a, b) => a - b); + return { + low: ratios[Math.floor(resamples * 0.025)], + high: ratios[Math.floor(resamples * 0.975)], + }; +} + +/* ── build ─────────────────────────────────────────────────────────────── */ +function cliFor(root) { + const cli = join(resolve(root), "packages/cli/dist/bootstrap.js"); + statSync(cli); + return cli; +} +function revision(root) { + const r = run("git", ["-C", root, "rev-parse", "--short", "HEAD"], { timeout: 10_000 }); + const dirty = run("git", ["-C", root, "status", "--porcelain", "--untracked-files=no"], { + timeout: 10_000, + }); + return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : ""); +} +function build(label, root, workload) { + const cli = cliFor(root); + const out = join(work, label, workload.name); + mkdirSync(dirname(out), { recursive: true }); + const cache = join(work, label, ".cache"); + mkdirSync(cache, { recursive: true }); + const env = { ...process.env, SCRIPTC_CACHE_DIR: cache }; + delete env.SCRIPTC_NO_CACHE; + delete env.SCRIPTC_TIMING; + const entry = join(suiteRoot, workload.entry); + const result = run(process.execPath, [cli, "build", entry, "--optimization=release", "-o", out], { + env, + timeout: 900_000, + }); + if (result.status !== 0) + return { + ok: false, + error: (result.stderr || result.stdout).slice(0, 4000), + buildMs: result.ms, + }; + return { ok: true, binary: out, bytes: statSync(out).size, buildMs: result.ms }; +} + +/* ── main ──────────────────────────────────────────────────────────────── */ +const contenders = [ + ...(values.baseline ? [{ label: "baseline", root: resolve(values.baseline) }] : []), + { label: "candidate", root: resolve(values.candidate) }, +]; +const report = { + schema: 1, + date: new Date().toISOString(), + host: { platform: process.platform, arch: process.arch, node: process.version }, + runs, + contenders: contenders.map((c) => ({ ...c, revision: revision(c.root) })), + workloads: [], +}; + +try { + const inputs = new Map(); + for (const w of workloads) + if (w.input && !inputs.has(w.input)) inputs.set(w.input, generateInput(w.input)); + + // Builds and the Node oracle run outside the lock: they do not need a + // quiet machine, and holding the lock while compiling would serialize + // every agent's slowest phase. + const prepared = []; + for (const w of workloads) { + const args = w.args.map((a) => (a === "{input}" ? inputs.get(w.input) : a)); + // The application workloads are ESM without a package.json "type"; Node's + // reparse warning is an oracle artifact, not program output. + const oracle = run(process.execPath, [ + "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", + join(suiteRoot, w.entry), + ...args, + ]); + const entry = { name: w.name, args, results: {} }; + if (oracle.status !== 0 || oracle.timedOut) { + entry.error = `node oracle failed (status ${oracle.status}): ${oracle.stderr.slice(0, 2000)}`; + prepared.push(entry); + continue; + } + for (const c of contenders) { + const built = build(c.label, c.root, w); + const result = { build_ms: Math.round(built.buildMs) }; + entry.results[c.label] = result; + if (!built.ok) { + result.status = "build-failed"; + result.error = built.error; + continue; + } + result.bytes = built.bytes; + result.binary = built.binary; + const check = run(built.binary, args); + if (check.timedOut) { + result.status = "timeout"; + continue; + } + if ( + check.stdout !== oracle.stdout || + check.stderr !== oracle.stderr || + check.status !== oracle.status + ) { + result.status = "mismatch"; + result.error = `expected status ${oracle.status}, got ${check.status}; stdout ${check.stdout === oracle.stdout ? "matches" : "differs"}; stderr ${check.stderr === oracle.stderr ? "matches" : "differs"}`; + continue; + } + result.status = "ok"; + result.samples = []; + } + entry.node_ms = Math.round(oracle.ms); + entry.nodeCommand = [ + "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON", + join(suiteRoot, w.entry), + ...args, + ]; + prepared.push(entry); + process.stderr.write(`prepared ${w.name}\n`); + } + + if (!values["no-lock"]) acquireLock(); + try { + for (const entry of prepared) { + const live = contenders.filter((c) => entry.results[c.label]?.status === "ok"); + if (live.length === 0) continue; + for (const c of live) + for (let i = 0; i < warmup; i++) run(entry.results[c.label].binary, entry.args); + // Node is the reference the project aims to beat, so it is timed in the + // same interleaved loop (whole-process wall time, startup included). + const nodeSamples = []; + if (values.node) run(process.execPath, entry.nodeCommand); + const random = makeRandom(entry.name.length * 7919); + for (let r = 0; r < runs; r++) { + const order = [...live]; + if (order.length === 2 && random() < 0.5) order.reverse(); + for (const c of order) { + const result = entry.results[c.label]; + const sample = run(result.binary, entry.args); + if (sample.timedOut || sample.status !== 0) { + result.status = sample.timedOut ? "timeout" : "unstable-exit"; + break; + } + result.samples.push(Math.round(sample.ms * 100) / 100); + } + if (values.node) nodeSamples.push(run(process.execPath, entry.nodeCommand).ms); + } + if (nodeSamples.length > 0) entry.node_ms = Math.round(median(nodeSamples) * 100) / 100; + const reference = entry.results.candidate; + if (reference?.status === "ok" && reference.samples.length > 0 && nodeSamples.length > 0) + entry.vs_node = Math.round((median(reference.samples) / entry.node_ms) * 1000) / 1000; + for (const c of live) { + const result = entry.results[c.label]; + if (result.status !== "ok") continue; + result.median_ms = Math.round(median(result.samples) * 100) / 100; + result.peak_rss_bytes = peakRss(result.binary, entry.args); + } + const base = entry.results.baseline; + const cand = entry.results.candidate; + if (base?.status === "ok" && cand?.status === "ok") { + const ci = bootstrapRatio(cand.samples, base.samples); + entry.ratio = Math.round((cand.median_ms / base.median_ms) * 10000) / 10000; + entry.ci95 = [Math.round(ci.low * 10000) / 10000, Math.round(ci.high * 10000) / 10000]; + entry.verdict = ci.high < 1 ? "faster" : ci.low > 1 ? "slower" : "neutral"; + entry.size_ratio = Math.round((cand.bytes / base.bytes) * 10000) / 10000; + } + process.stderr.write(`measured ${entry.name}\n`); + } + } finally { + if (!values["no-lock"]) releaseLock(); + } + for (const entry of prepared) { + for (const result of Object.values(entry.results)) delete result.binary; + delete entry.nodeCommand; + report.workloads.push(entry); + } + const vsNode = report.workloads.filter((w) => w.vs_node !== undefined).map((w) => w.vs_node); + if (vsNode.length > 0) + report.geomean_vs_node = + Math.round(Math.exp(vsNode.reduce((s, r) => s + Math.log(r), 0) / vsNode.length) * 1000) / + 1000; + const ratios = report.workloads.filter((w) => w.ratio !== undefined).map((w) => w.ratio); + if (ratios.length > 0) + report.geomean_ratio = + Math.round(Math.exp(ratios.reduce((s, r) => s + Math.log(r), 0) / ratios.length) * 10000) / + 10000; +} finally { + if (!values.keep) rmSync(work, { recursive: true, force: true }); +} + +if (values.json) writeFileSync(values.json, JSON.stringify(report, null, 2) + "\n"); + +/* ── summary table ─────────────────────────────────────────────────────── */ +const pct = (r) => `${r < 1 ? "" : "+"}${((r - 1) * 100).toFixed(1)}%`; +const lines = []; +const xNode = (w) => (w.vs_node === undefined ? "-" : `${w.vs_node.toFixed(2)}x`); +if (values.baseline) { + lines.push( + "| workload | baseline ms | candidate ms | change | 95% CI | verdict | size | node ms | candidate/node |", + ); + lines.push("| --- | ---: | ---: | ---: | --- | --- | ---: | ---: | ---: |"); +} else { + lines.push("| workload | median ms | status | bytes | peak RSS MiB | node ms | scriptc/node |"); + lines.push("| --- | ---: | --- | ---: | ---: | ---: | ---: |"); +} +for (const w of report.workloads) { + const b = w.results.baseline; + const c = w.results.candidate; + if (w.error) { + lines.push(`| ${w.name} | ${w.error.split("\n")[0]} |`); + } else if (values.baseline) { + const show = (r) => (r?.status === "ok" ? r.median_ms.toFixed(1) : (r?.status ?? "-")); + lines.push( + `| ${w.name} | ${show(b)} | ${show(c)} | ${w.ratio ? pct(w.ratio) : "-"} | ${w.ci95 ? `${pct(w.ci95[0])} .. ${pct(w.ci95[1])}` : "-"} | ${w.verdict ?? "-"} | ${w.size_ratio ? pct(w.size_ratio) : "-"} | ${w.node_ms} | ${xNode(w)} |`, + ); + } else { + lines.push( + `| ${w.name} | ${c?.median_ms?.toFixed(1) ?? "-"} | ${c?.status ?? "-"} | ${c?.bytes ?? "-"} | ${c?.peak_rss_bytes ? (c.peak_rss_bytes / 1048576).toFixed(1) : "-"} | ${w.node_ms} | ${xNode(w)} |`, + ); + } +} +if (report.geomean_ratio !== undefined) + lines.push("", `geomean change: ${pct(report.geomean_ratio)}`); +if (report.geomean_vs_node !== undefined) + lines.push( + `geomean candidate/node: ${report.geomean_vs_node.toFixed(3)}x (lower is better; goal: well below 1)`, + ); +console.log(lines.join("\n")); +const broken = report.workloads.some( + (w) => w.error || Object.values(w.results).some((r) => r.status !== "ok"), +); +process.exitCode = broken ? 1 : 0;