From fd34d72e00af54632c756a78fb8dc13177e8ded7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 20 Oct 2026 18:08:10 -0500 Subject: [PATCH] Keep each runtime benchmark workload in its own file workloads.json listed every workload in one array, and its README table listed them in one hand order, so changes that each added a workload appended at the same place and conflicted. Each workload is now a JSON file in benchmarks/runtime/workloads/ named after it, read through benchmarks/runtime/workloads.mjs by the bench runner, the zero-cost gate, the strace script and the dynamic-tier test, ordered by category and then name. The loader still reads workloads.json in a checkout from before this change, so the gate can compare against one. The README table follows the same order. --- benchmarks/runtime/README.md | 50 +-- benchmarks/runtime/workloads.json | 290 ------------------ benchmarks/runtime/workloads.mjs | 33 ++ benchmarks/runtime/workloads/alloc-trees.json | 6 + benchmarks/runtime/workloads/ast-interp.json | 6 + .../runtime/workloads/async-pipeline.json | 6 + .../runtime/workloads/atomics-contended.json | 6 + .../runtime/workloads/channel-messages.json | 6 + benchmarks/runtime/workloads/cli-config.json | 12 + benchmarks/runtime/workloads/csv-numbers.json | 6 + .../runtime/workloads/cyclic-graphs.json | 6 + .../workloads/functional-pipeline.json | 6 + benchmarks/runtime/workloads/http-api.json | 14 + .../runtime/workloads/intern-table-1.json | 8 + .../runtime/workloads/intern-table-map-1.json | 8 + .../runtime/workloads/intern-table-map.json | 8 + .../runtime/workloads/intern-table.json | 8 + .../runtime/workloads/inventory-report.json | 7 + .../runtime/workloads/json-records.json | 6 + benchmarks/runtime/workloads/log-lines.json | 6 + benchmarks/runtime/workloads/log-summary.json | 11 + benchmarks/runtime/workloads/number-maps.json | 6 + .../runtime/workloads/numeric-kernels.json | 6 + .../runtime/workloads/optional-fields.json | 6 + .../runtime/workloads/published-graphs.json | 6 + .../runtime/workloads/record-boundaries.json | 6 + .../runtime/workloads/records-sort.json | 6 + benchmarks/runtime/workloads/regex-logs.json | 6 + .../runtime/workloads/region-builds.json | 6 + .../runtime/workloads/template-render.json | 6 + .../runtime/workloads/thread-slots-cache.json | 9 + .../runtime/workloads/thread-slots.json | 9 + .../workloads/typed-array-scan-workers.json | 6 + .../runtime/workloads/typed-array-scan.json | 6 + .../runtime/workloads/unicode-lexer.json | 6 + .../runtime/workloads/validate-errors.json | 6 + .../runtime/workloads/value-objects.json | 6 + .../runtime/workloads/value-records.json | 6 + benchmarks/runtime/workloads/word-graph.json | 6 + benchmarks/scaling/strace-workloads.mjs | 3 +- scripts/bench-runtime.mjs | 15 +- scripts/zero-cost-gate.mjs | 4 +- tests/harness/bench-dynamic-tier.test.ts | 8 +- 43 files changed, 322 insertions(+), 331 deletions(-) delete mode 100644 benchmarks/runtime/workloads.json create mode 100644 benchmarks/runtime/workloads.mjs create mode 100644 benchmarks/runtime/workloads/alloc-trees.json create mode 100644 benchmarks/runtime/workloads/ast-interp.json create mode 100644 benchmarks/runtime/workloads/async-pipeline.json create mode 100644 benchmarks/runtime/workloads/atomics-contended.json create mode 100644 benchmarks/runtime/workloads/channel-messages.json create mode 100644 benchmarks/runtime/workloads/cli-config.json create mode 100644 benchmarks/runtime/workloads/csv-numbers.json create mode 100644 benchmarks/runtime/workloads/cyclic-graphs.json create mode 100644 benchmarks/runtime/workloads/functional-pipeline.json create mode 100644 benchmarks/runtime/workloads/http-api.json create mode 100644 benchmarks/runtime/workloads/intern-table-1.json create mode 100644 benchmarks/runtime/workloads/intern-table-map-1.json create mode 100644 benchmarks/runtime/workloads/intern-table-map.json create mode 100644 benchmarks/runtime/workloads/intern-table.json create mode 100644 benchmarks/runtime/workloads/inventory-report.json create mode 100644 benchmarks/runtime/workloads/json-records.json create mode 100644 benchmarks/runtime/workloads/log-lines.json create mode 100644 benchmarks/runtime/workloads/log-summary.json create mode 100644 benchmarks/runtime/workloads/number-maps.json create mode 100644 benchmarks/runtime/workloads/numeric-kernels.json create mode 100644 benchmarks/runtime/workloads/optional-fields.json create mode 100644 benchmarks/runtime/workloads/published-graphs.json create mode 100644 benchmarks/runtime/workloads/record-boundaries.json create mode 100644 benchmarks/runtime/workloads/records-sort.json create mode 100644 benchmarks/runtime/workloads/regex-logs.json create mode 100644 benchmarks/runtime/workloads/region-builds.json create mode 100644 benchmarks/runtime/workloads/template-render.json create mode 100644 benchmarks/runtime/workloads/thread-slots-cache.json create mode 100644 benchmarks/runtime/workloads/thread-slots.json create mode 100644 benchmarks/runtime/workloads/typed-array-scan-workers.json create mode 100644 benchmarks/runtime/workloads/typed-array-scan.json create mode 100644 benchmarks/runtime/workloads/unicode-lexer.json create mode 100644 benchmarks/runtime/workloads/validate-errors.json create mode 100644 benchmarks/runtime/workloads/value-objects.json create mode 100644 benchmarks/runtime/workloads/value-records.json create mode 100644 benchmarks/runtime/workloads/word-graph.json diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 7ab829bd5..92eb099db 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -4,34 +4,34 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | Workload | Category | Exercises | | --- | --- | --- | -| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | -| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | -| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch | -| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | -| `template-render` | cpu | template literals, escaping, `map`/`join` | -| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | -| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out | -| `word-graph` | cpu | string-keyed Map/Set, BFS | | `alloc-trees` | cpu | allocation-heavy trees, immutable object-spread updates | -| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | -| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` | -| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage | -| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | -| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread | -| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | +| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch | +| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out | | `atomics-contended` | cpu | four worker threads updating `SharedArrayBuffer` counters with `Atomics`: one shared counter, one counter per worker, a `compareExchange` loop, and `Atomics.wait`/`notify` handoffs | +| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | +| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | +| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | +| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | +| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | +| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | +| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | +| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | +| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` | +| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | +| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read | +| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | +| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | +| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | +| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | +| `template-render` | cpu | template literals, escaping, `map`/`join` | +| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread | +| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage | +| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts | +| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | | `value-objects` | cpu | small immutable records and class instances: vector helpers, method chains, particle updates | | `value-records` | cpu | records that hold strings, spread state updates and destructured tuples: token helpers, an immutable reducer, tuple returns | -| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | -| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | -| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | -| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | -| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read | -| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | -| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | -| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | -| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts | -| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | +| `word-graph` | cpu | string-keyed Map/Set, BFS | | `cli-config` | startup | a `deploy-plan` CLI: read a ~110 KB JSON manifest, validate, interpolate env values, order 80 services into dependency waves, print a summary | | `http-api` | server | a `node:http` JSON API: routing, query parsing, a seeded catalog, per-cart state, typed JSON bodies, `ETag`/`If-None-Match`, auth and request-id headers | @@ -47,7 +47,7 @@ Node is timed in the same interleaved loop and every row reports candidate/Node; ## Workload kinds -Entries in `workloads.json` are process workloads unless they set `kind`. +Each workload is a JSON file in `workloads/`, named after the workload, that holds its `entry`, `args` and options; adding a workload adds its file and its row in the table above, kept in category and then name order. Workloads are process workloads unless they set `kind`. Process workloads time whole launches (wall time, startup included). An optional `launches` count makes each sample the median of that many sequential launches; `cli-config` uses 20 so that a startup-dominated command (a few milliseconds compiled, tens of milliseconds under Node) is measured with low noise. Its table row is labeled `(per launch)`. On a heavily loaded macOS host, whole batches of launches can run several times slower (samples become bimodal), so judge startup changes on the Linux lane or a quiet machine. diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json deleted file mode 100644 index 80beefab4..000000000 --- a/benchmarks/runtime/workloads.json +++ /dev/null @@ -1,290 +0,0 @@ -{ - "workloads": [ - { - "name": "json-records", - "entry": "workloads/json-records.ts", - "args": [ - "4" - ] - }, - { - "name": "regex-logs", - "entry": "workloads/regex-logs.ts", - "args": [ - "1" - ] - }, - { - "name": "ast-interp", - "entry": "workloads/ast-interp.ts", - "args": [ - "5" - ] - }, - { - "name": "records-sort", - "entry": "workloads/records-sort.ts", - "args": [ - "1" - ] - }, - { - "name": "template-render", - "entry": "workloads/template-render.ts", - "args": [ - "1" - ] - }, - { - "name": "functional-pipeline", - "entry": "workloads/functional-pipeline.ts", - "args": [ - "1" - ] - }, - { - "name": "async-pipeline", - "entry": "workloads/async-pipeline.ts", - "args": [ - "1" - ] - }, - { - "name": "word-graph", - "entry": "workloads/word-graph.ts", - "args": [ - "2" - ] - }, - { - "name": "alloc-trees", - "entry": "workloads/alloc-trees.ts", - "args": [ - "1" - ] - }, - { - "name": "cyclic-graphs", - "entry": "workloads/cyclic-graphs.ts", - "args": [ - "3" - ] - }, - { - "name": "numeric-kernels", - "entry": "workloads/numeric-kernels.ts", - "args": [ - "0.5" - ] - }, - { - "name": "typed-array-scan", - "entry": "workloads/typed-array-scan.ts", - "args": [ - "3" - ] - }, - { - "name": "typed-array-scan-workers", - "entry": "workloads/typed-array-scan-workers.ts", - "args": [ - "3" - ] - }, - { - "name": "intern-table", - "entry": "workloads/intern-table.ts", - "args": [ - "table", - "8", - "100000" - ] - }, - { - "name": "intern-table-map", - "entry": "workloads/intern-table.ts", - "args": [ - "map", - "8", - "100000" - ] - }, - { - "name": "intern-table-1", - "entry": "workloads/intern-table.ts", - "args": [ - "table", - "1", - "100000" - ] - }, - { - "name": "intern-table-map-1", - "entry": "workloads/intern-table.ts", - "args": [ - "map", - "1", - "100000" - ] - }, - { - "name": "thread-slots", - "entry": "workloads/thread-slots.ts", - "args": [ - "slots", - "8", - "50000", - "20" - ] - }, - { - "name": "thread-slots-cache", - "entry": "workloads/thread-slots.ts", - "args": [ - "cache", - "8", - "50000", - "20" - ] - }, - { - "name": "channel-messages", - "entry": "workloads/channel-messages.ts", - "args": [ - "200000" - ] - }, - { - "name": "atomics-contended", - "entry": "workloads/atomics-contended.ts", - "args": [ - "2" - ] - }, - { - "name": "value-objects", - "entry": "workloads/value-objects.ts", - "args": [ - "2" - ] - }, - { - "name": "value-records", - "entry": "workloads/value-records.ts", - "args": [ - "1" - ] - }, - { - "name": "region-builds", - "entry": "workloads/region-builds.ts", - "args": [ - "1" - ] - }, - { - "name": "optional-fields", - "entry": "workloads/optional-fields.ts", - "args": [ - "10" - ] - }, - { - "name": "record-boundaries", - "entry": "workloads/record-boundaries.ts", - "args": [ - "8" - ] - }, - { - "name": "number-maps", - "entry": "workloads/number-maps.ts", - "args": [ - "4" - ] - }, - { - "name": "published-graphs", - "entry": "workloads/published-graphs.ts", - "args": [ - "1" - ] - }, - { - "name": "validate-errors", - "entry": "workloads/validate-errors.ts", - "args": [ - "1" - ] - }, - { - "name": "csv-numbers", - "entry": "workloads/csv-numbers.ts", - "args": [ - "1" - ] - }, - { - "name": "log-lines", - "entry": "workloads/log-lines.ts", - "args": [ - "4" - ] - }, - { - "name": "unicode-lexer", - "entry": "workloads/unicode-lexer.ts", - "args": [ - "1" - ] - }, - { - "name": "log-summary", - "entry": "../builds/log-summary/main.ts", - "input": "log-summary", - "args": [ - "{input}", - "--min-status", - "200", - "--limit", - "12" - ] - }, - { - "name": "inventory-report", - "entry": "../builds/inventory-report/main.ts", - "input": "inventory-report", - "args": [ - "{input}" - ] - }, - { - "name": "cli-config", - "entry": "workloads/cli-config.ts", - "input": "cli-config", - "launches": 20, - "args": [ - "{input}", - "--env", - "production", - "--max-wave", - "12" - ] - }, - { - "name": "http-api", - "kind": "server", - "entry": "workloads/http-api.ts", - "args": [], - "client": { - "script": "clients/http-load.mjs", - "args": [ - "--connections=64", - "--threads=2", - "--warmup=4000", - "--requests=40000" - ] - } - } - ] -} diff --git a/benchmarks/runtime/workloads.mjs b/benchmarks/runtime/workloads.mjs new file mode 100644 index 000000000..e63ee98eb --- /dev/null +++ b/benchmarks/runtime/workloads.mjs @@ -0,0 +1,33 @@ +/* The runtime benchmark workloads: one JSON file per workload in + * workloads/, named after the workload, next to the entry programs. A file + * holds the workload's entry, args and options; changes that add + * different workloads add different files. */ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const CATEGORIES = ["cpu", "startup", "server"]; + +/* Server workloads run a listening program; startup-bound ones time + * several sequential launches per sample. */ +export const workloadCategory = (w) => + w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu"; + +/* The workloads of the suite at `suite` (a benchmarks/runtime directory), + * by category and then name. A checkout from before the per-workload files + * lists them in workloads.json, in its own order. */ +export function readWorkloads(suite) { + const legacy = join(suite, "workloads.json"); + if (existsSync(legacy)) return JSON.parse(readFileSync(legacy, "utf8")).workloads; + const dir = join(suite, "workloads"); + return readdirSync(dir) + .filter((file) => file.endsWith(".json")) + .map((file) => ({ + name: file.slice(0, -".json".length), + ...JSON.parse(readFileSync(join(dir, file), "utf8")), + })) + .sort( + (a, b) => + CATEGORIES.indexOf(workloadCategory(a)) - CATEGORIES.indexOf(workloadCategory(b)) || + (a.name < b.name ? -1 : a.name > b.name ? 1 : 0), + ); +} diff --git a/benchmarks/runtime/workloads/alloc-trees.json b/benchmarks/runtime/workloads/alloc-trees.json new file mode 100644 index 000000000..d4ae2ae72 --- /dev/null +++ b/benchmarks/runtime/workloads/alloc-trees.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/alloc-trees.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/ast-interp.json b/benchmarks/runtime/workloads/ast-interp.json new file mode 100644 index 000000000..1817541ac --- /dev/null +++ b/benchmarks/runtime/workloads/ast-interp.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/ast-interp.ts", + "args": [ + "5" + ] +} diff --git a/benchmarks/runtime/workloads/async-pipeline.json b/benchmarks/runtime/workloads/async-pipeline.json new file mode 100644 index 000000000..610f62793 --- /dev/null +++ b/benchmarks/runtime/workloads/async-pipeline.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/async-pipeline.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/atomics-contended.json b/benchmarks/runtime/workloads/atomics-contended.json new file mode 100644 index 000000000..7f78d265b --- /dev/null +++ b/benchmarks/runtime/workloads/atomics-contended.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/atomics-contended.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/runtime/workloads/channel-messages.json b/benchmarks/runtime/workloads/channel-messages.json new file mode 100644 index 000000000..56b84dda4 --- /dev/null +++ b/benchmarks/runtime/workloads/channel-messages.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/channel-messages.ts", + "args": [ + "200000" + ] +} diff --git a/benchmarks/runtime/workloads/cli-config.json b/benchmarks/runtime/workloads/cli-config.json new file mode 100644 index 000000000..db7a4c645 --- /dev/null +++ b/benchmarks/runtime/workloads/cli-config.json @@ -0,0 +1,12 @@ +{ + "entry": "workloads/cli-config.ts", + "input": "cli-config", + "launches": 20, + "args": [ + "{input}", + "--env", + "production", + "--max-wave", + "12" + ] +} diff --git a/benchmarks/runtime/workloads/csv-numbers.json b/benchmarks/runtime/workloads/csv-numbers.json new file mode 100644 index 000000000..acb3c21d7 --- /dev/null +++ b/benchmarks/runtime/workloads/csv-numbers.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/csv-numbers.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/cyclic-graphs.json b/benchmarks/runtime/workloads/cyclic-graphs.json new file mode 100644 index 000000000..2e62f7fdf --- /dev/null +++ b/benchmarks/runtime/workloads/cyclic-graphs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/cyclic-graphs.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/functional-pipeline.json b/benchmarks/runtime/workloads/functional-pipeline.json new file mode 100644 index 000000000..5ee3af902 --- /dev/null +++ b/benchmarks/runtime/workloads/functional-pipeline.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/functional-pipeline.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/http-api.json b/benchmarks/runtime/workloads/http-api.json new file mode 100644 index 000000000..145166945 --- /dev/null +++ b/benchmarks/runtime/workloads/http-api.json @@ -0,0 +1,14 @@ +{ + "kind": "server", + "entry": "workloads/http-api.ts", + "args": [], + "client": { + "script": "clients/http-load.mjs", + "args": [ + "--connections=64", + "--threads=2", + "--warmup=4000", + "--requests=40000" + ] + } +} diff --git a/benchmarks/runtime/workloads/intern-table-1.json b/benchmarks/runtime/workloads/intern-table-1.json new file mode 100644 index 000000000..8a2b253f6 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-1.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "table", + "1", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table-map-1.json b/benchmarks/runtime/workloads/intern-table-map-1.json new file mode 100644 index 000000000..fefafac67 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-map-1.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "map", + "1", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table-map.json b/benchmarks/runtime/workloads/intern-table-map.json new file mode 100644 index 000000000..136c6c31e --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-map.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "map", + "8", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table.json b/benchmarks/runtime/workloads/intern-table.json new file mode 100644 index 000000000..86fa11413 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "table", + "8", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/inventory-report.json b/benchmarks/runtime/workloads/inventory-report.json new file mode 100644 index 000000000..e75f5e3e5 --- /dev/null +++ b/benchmarks/runtime/workloads/inventory-report.json @@ -0,0 +1,7 @@ +{ + "entry": "../builds/inventory-report/main.ts", + "input": "inventory-report", + "args": [ + "{input}" + ] +} diff --git a/benchmarks/runtime/workloads/json-records.json b/benchmarks/runtime/workloads/json-records.json new file mode 100644 index 000000000..3263ac068 --- /dev/null +++ b/benchmarks/runtime/workloads/json-records.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/json-records.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/log-lines.json b/benchmarks/runtime/workloads/log-lines.json new file mode 100644 index 000000000..635bcc49f --- /dev/null +++ b/benchmarks/runtime/workloads/log-lines.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/log-lines.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/log-summary.json b/benchmarks/runtime/workloads/log-summary.json new file mode 100644 index 000000000..274a1cff6 --- /dev/null +++ b/benchmarks/runtime/workloads/log-summary.json @@ -0,0 +1,11 @@ +{ + "entry": "../builds/log-summary/main.ts", + "input": "log-summary", + "args": [ + "{input}", + "--min-status", + "200", + "--limit", + "12" + ] +} diff --git a/benchmarks/runtime/workloads/number-maps.json b/benchmarks/runtime/workloads/number-maps.json new file mode 100644 index 000000000..2adfbcc47 --- /dev/null +++ b/benchmarks/runtime/workloads/number-maps.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/number-maps.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/numeric-kernels.json b/benchmarks/runtime/workloads/numeric-kernels.json new file mode 100644 index 000000000..04718346b --- /dev/null +++ b/benchmarks/runtime/workloads/numeric-kernels.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/numeric-kernels.ts", + "args": [ + "0.5" + ] +} diff --git a/benchmarks/runtime/workloads/optional-fields.json b/benchmarks/runtime/workloads/optional-fields.json new file mode 100644 index 000000000..c54d87437 --- /dev/null +++ b/benchmarks/runtime/workloads/optional-fields.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/optional-fields.ts", + "args": [ + "10" + ] +} diff --git a/benchmarks/runtime/workloads/published-graphs.json b/benchmarks/runtime/workloads/published-graphs.json new file mode 100644 index 000000000..b22f5a0db --- /dev/null +++ b/benchmarks/runtime/workloads/published-graphs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/published-graphs.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/record-boundaries.json b/benchmarks/runtime/workloads/record-boundaries.json new file mode 100644 index 000000000..a35c346eb --- /dev/null +++ b/benchmarks/runtime/workloads/record-boundaries.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/record-boundaries.ts", + "args": [ + "8" + ] +} diff --git a/benchmarks/runtime/workloads/records-sort.json b/benchmarks/runtime/workloads/records-sort.json new file mode 100644 index 000000000..fd0097fe2 --- /dev/null +++ b/benchmarks/runtime/workloads/records-sort.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/records-sort.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/regex-logs.json b/benchmarks/runtime/workloads/regex-logs.json new file mode 100644 index 000000000..636db124e --- /dev/null +++ b/benchmarks/runtime/workloads/regex-logs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/regex-logs.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/region-builds.json b/benchmarks/runtime/workloads/region-builds.json new file mode 100644 index 000000000..dc53dbc36 --- /dev/null +++ b/benchmarks/runtime/workloads/region-builds.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/region-builds.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/template-render.json b/benchmarks/runtime/workloads/template-render.json new file mode 100644 index 000000000..9897494f6 --- /dev/null +++ b/benchmarks/runtime/workloads/template-render.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/template-render.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/thread-slots-cache.json b/benchmarks/runtime/workloads/thread-slots-cache.json new file mode 100644 index 000000000..0eb6e11ce --- /dev/null +++ b/benchmarks/runtime/workloads/thread-slots-cache.json @@ -0,0 +1,9 @@ +{ + "entry": "workloads/thread-slots.ts", + "args": [ + "cache", + "8", + "50000", + "20" + ] +} diff --git a/benchmarks/runtime/workloads/thread-slots.json b/benchmarks/runtime/workloads/thread-slots.json new file mode 100644 index 000000000..b8ed5b140 --- /dev/null +++ b/benchmarks/runtime/workloads/thread-slots.json @@ -0,0 +1,9 @@ +{ + "entry": "workloads/thread-slots.ts", + "args": [ + "slots", + "8", + "50000", + "20" + ] +} diff --git a/benchmarks/runtime/workloads/typed-array-scan-workers.json b/benchmarks/runtime/workloads/typed-array-scan-workers.json new file mode 100644 index 000000000..c01893a2c --- /dev/null +++ b/benchmarks/runtime/workloads/typed-array-scan-workers.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/typed-array-scan-workers.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/typed-array-scan.json b/benchmarks/runtime/workloads/typed-array-scan.json new file mode 100644 index 000000000..0b2ba0592 --- /dev/null +++ b/benchmarks/runtime/workloads/typed-array-scan.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/typed-array-scan.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/unicode-lexer.json b/benchmarks/runtime/workloads/unicode-lexer.json new file mode 100644 index 000000000..91ccab087 --- /dev/null +++ b/benchmarks/runtime/workloads/unicode-lexer.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/unicode-lexer.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/validate-errors.json b/benchmarks/runtime/workloads/validate-errors.json new file mode 100644 index 000000000..b8d457897 --- /dev/null +++ b/benchmarks/runtime/workloads/validate-errors.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/validate-errors.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/value-objects.json b/benchmarks/runtime/workloads/value-objects.json new file mode 100644 index 000000000..276fa4ab8 --- /dev/null +++ b/benchmarks/runtime/workloads/value-objects.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/value-objects.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/runtime/workloads/value-records.json b/benchmarks/runtime/workloads/value-records.json new file mode 100644 index 000000000..84c0e12dd --- /dev/null +++ b/benchmarks/runtime/workloads/value-records.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/value-records.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/word-graph.json b/benchmarks/runtime/workloads/word-graph.json new file mode 100644 index 000000000..95ffb27e5 --- /dev/null +++ b/benchmarks/runtime/workloads/word-graph.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/word-graph.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/scaling/strace-workloads.mjs b/benchmarks/scaling/strace-workloads.mjs index b4db5af19..f0644cf13 100644 --- a/benchmarks/scaling/strace-workloads.mjs +++ b/benchmarks/scaling/strace-workloads.mjs @@ -7,6 +7,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads } from "../runtime/workloads.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const suite = resolve(here, "../runtime"); @@ -18,7 +19,7 @@ const { values } = parseArgs({ }, }); mkdirSync(values.out, { recursive: true }); -const { workloads } = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")); +const workloads = readWorkloads(suite); const cli = join(values.scriptc, "packages/cli/dist/bootstrap.js"); // Same inputs as scripts/bench-runtime.mjs generates. function input(kind) { diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index 4e4f2c245..61134fa4c 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -1,6 +1,6 @@ /* Runtime performance A/B harness for compiled programs. * - * Builds every workload in benchmarks/runtime/workloads.json with one or two + * Builds every workload in benchmarks/runtime/workloads/ with one or two * scriptc checkouts, verifies that each executable's output is identical to * Node's (stdout, stderr, exit status), then times interleaved launches and reports * per-workload medians with bootstrap confidence intervals for the @@ -23,6 +23,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads, workloadCategory } from "../benchmarks/runtime/workloads.mjs"; const repoRoot = fileURLToPath(new URL("..", import.meta.url)); const suiteRoot = join(repoRoot, "benchmarks/runtime"); @@ -62,7 +63,7 @@ evenly over those layouts, so verdicts cover code-placement luck instead of one placement. Use it for small codegen changes (e.g. --layouts=4 --runs=16). -Workload names are listed in benchmarks/runtime/workloads.json; for example +Workload names are the file names in benchmarks/runtime/workloads/*.json; for example --workloads=cli-config,http-api runs only the startup and server workloads. --baseline-optimization/--candidate-optimization pick each contender's @@ -88,13 +89,13 @@ assert.ok( "--layouts needs macOS or Linux", ); -const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8")); +const allWorkloads = readWorkloads(suiteRoot); const selected = values.workloads ? new Set(values.workloads.split(",")) : null; -const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name)); +const workloads = allWorkloads.filter((w) => selected === null || selected.has(w.name)); if (selected) { for (const name of selected) assert.ok( - manifest.workloads.some((w) => w.name === name), + allWorkloads.some((w) => w.name === name), `unknown workload ${name}`, ); } @@ -488,8 +489,6 @@ const report = { /* Workload kinds: "process" (default) times whole launches; `launches` > 1 * makes each sample the median of that many sequential launches (startup-bound * CLIs). "server" times the measured phase of a load-client session. */ -const categoryOf = (w) => - w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu"; for (const w of workloads) { assert.ok((w.kind ?? "process") === "process" || w.kind === "server", `${w.name}: unknown kind`); if (w.kind === "server") assert.ok(w.client?.script, `${w.name}: server workloads need a client`); @@ -569,7 +568,7 @@ try { const entry = { name: w.name, args, results: {} }; if (w.kind === "server") entry.kind = "server"; if ((w.launches ?? 1) > 1) entry.launches = w.launches; - entry.category = categoryOf(w); + entry.category = workloadCategory(w); const oracleFailure = server ? serverFailure(oracle) : oracle.timedOut ? "timed out" : ""; if (oracle.status !== 0 || oracleFailure) { entry.error = `node oracle failed (status ${oracle.status}${oracleFailure ? `, ${oracleFailure}` : ""}): ${oracle.stderr.slice(0, 2000)}`; diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index de3c0d0fc..1575fc5ee 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -27,6 +27,7 @@ import { availableParallelism } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads } from "../benchmarks/runtime/workloads.mjs"; import { cachegrindCounts, compareArtifact, @@ -173,9 +174,8 @@ function goodCodeSet() { const programs = []; if (sets.has("bench")) { const suite = join(programsRoot, "benchmarks/runtime"); - const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")); const seen = new Set(); - for (const w of manifest.workloads) { + for (const w of readWorkloads(suite)) { const entry = join(suite, w.entry); if (seen.has(entry)) continue; seen.add(entry); diff --git a/tests/harness/bench-dynamic-tier.test.ts b/tests/harness/bench-dynamic-tier.test.ts index de19f2b21..9276221ab 100644 --- a/tests/harness/bench-dynamic-tier.test.ts +++ b/tests/harness/bench-dynamic-tier.test.ts @@ -7,7 +7,7 @@ * release binary the gate measures, so other hosts and the sanitized lane * skip the builds. */ import { execFile } from "node:child_process"; -import { mkdirSync, readFileSync } from "node:fs"; +import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { promisify } from "node:util"; import { expect, test } from "vitest"; @@ -17,6 +17,7 @@ import { definedFunctionNames, dynamicTierSymbols, } from "../../scripts/zero-cost-compare.mjs"; +import { readWorkloads } from "../../benchmarks/runtime/workloads.mjs"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -31,11 +32,8 @@ interface Workload { // One program per entry file, named after its first workload, as the gate // names them. -const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")) as { - workloads: Workload[]; -}; const programs = new Map(); -for (const w of manifest.workloads) +for (const w of readWorkloads(suite) as Workload[]) if (![...programs.values()].includes(w.entry)) programs.set(`bench/${w.name}`, w.entry); test("every workload the gate lists as dynamic is a bench program", () => {