diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 7ab829bd5..92eb099db 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -4,34 +4,34 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | Workload | Category | Exercises | | --- | --- | --- | -| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | -| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | -| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch | -| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | -| `template-render` | cpu | template literals, escaping, `map`/`join` | -| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | -| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out | -| `word-graph` | cpu | string-keyed Map/Set, BFS | | `alloc-trees` | cpu | allocation-heavy trees, immutable object-spread updates | -| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | -| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` | -| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage | -| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | -| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread | -| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | +| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch | +| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out | | `atomics-contended` | cpu | four worker threads updating `SharedArrayBuffer` counters with `Atomics`: one shared counter, one counter per worker, a `compareExchange` loop, and `Atomics.wait`/`notify` handoffs | +| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any | +| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | +| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner | +| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records | +| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables | +| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation | +| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | +| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | +| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | +| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` | +| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | +| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read | +| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | +| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | +| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | +| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | +| `template-render` | cpu | template literals, escaping, `map`/`join` | +| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread | +| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage | +| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts | +| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | | `value-objects` | cpu | small immutable records and class instances: vector helpers, method chains, particle updates | | `value-records` | cpu | records that hold strings, spread state updates and destructured tuples: token helpers, an immutable reducer, tuple returns | -| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | -| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | -| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | -| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | -| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read | -| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | -| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | -| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | -| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts | -| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs | +| `word-graph` | cpu | string-keyed Map/Set, BFS | | `cli-config` | startup | a `deploy-plan` CLI: read a ~110 KB JSON manifest, validate, interpolate env values, order 80 services into dependency waves, print a summary | | `http-api` | server | a `node:http` JSON API: routing, query parsing, a seeded catalog, per-cart state, typed JSON bodies, `ETag`/`If-None-Match`, auth and request-id headers | @@ -47,7 +47,7 @@ Node is timed in the same interleaved loop and every row reports candidate/Node; ## Workload kinds -Entries in `workloads.json` are process workloads unless they set `kind`. +Each workload is a JSON file in `workloads/`, named after the workload, that holds its `entry`, `args` and options; adding a workload adds its file and its row in the table above, kept in category and then name order. Workloads are process workloads unless they set `kind`. Process workloads time whole launches (wall time, startup included). An optional `launches` count makes each sample the median of that many sequential launches; `cli-config` uses 20 so that a startup-dominated command (a few milliseconds compiled, tens of milliseconds under Node) is measured with low noise. Its table row is labeled `(per launch)`. On a heavily loaded macOS host, whole batches of launches can run several times slower (samples become bimodal), so judge startup changes on the Linux lane or a quiet machine. diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json deleted file mode 100644 index 80beefab4..000000000 --- a/benchmarks/runtime/workloads.json +++ /dev/null @@ -1,290 +0,0 @@ -{ - "workloads": [ - { - "name": "json-records", - "entry": "workloads/json-records.ts", - "args": [ - "4" - ] - }, - { - "name": "regex-logs", - "entry": "workloads/regex-logs.ts", - "args": [ - "1" - ] - }, - { - "name": "ast-interp", - "entry": "workloads/ast-interp.ts", - "args": [ - "5" - ] - }, - { - "name": "records-sort", - "entry": "workloads/records-sort.ts", - "args": [ - "1" - ] - }, - { - "name": "template-render", - "entry": "workloads/template-render.ts", - "args": [ - "1" - ] - }, - { - "name": "functional-pipeline", - "entry": "workloads/functional-pipeline.ts", - "args": [ - "1" - ] - }, - { - "name": "async-pipeline", - "entry": "workloads/async-pipeline.ts", - "args": [ - "1" - ] - }, - { - "name": "word-graph", - "entry": "workloads/word-graph.ts", - "args": [ - "2" - ] - }, - { - "name": "alloc-trees", - "entry": "workloads/alloc-trees.ts", - "args": [ - "1" - ] - }, - { - "name": "cyclic-graphs", - "entry": "workloads/cyclic-graphs.ts", - "args": [ - "3" - ] - }, - { - "name": "numeric-kernels", - "entry": "workloads/numeric-kernels.ts", - "args": [ - "0.5" - ] - }, - { - "name": "typed-array-scan", - "entry": "workloads/typed-array-scan.ts", - "args": [ - "3" - ] - }, - { - "name": "typed-array-scan-workers", - "entry": "workloads/typed-array-scan-workers.ts", - "args": [ - "3" - ] - }, - { - "name": "intern-table", - "entry": "workloads/intern-table.ts", - "args": [ - "table", - "8", - "100000" - ] - }, - { - "name": "intern-table-map", - "entry": "workloads/intern-table.ts", - "args": [ - "map", - "8", - "100000" - ] - }, - { - "name": "intern-table-1", - "entry": "workloads/intern-table.ts", - "args": [ - "table", - "1", - "100000" - ] - }, - { - "name": "intern-table-map-1", - "entry": "workloads/intern-table.ts", - "args": [ - "map", - "1", - "100000" - ] - }, - { - "name": "thread-slots", - "entry": "workloads/thread-slots.ts", - "args": [ - "slots", - "8", - "50000", - "20" - ] - }, - { - "name": "thread-slots-cache", - "entry": "workloads/thread-slots.ts", - "args": [ - "cache", - "8", - "50000", - "20" - ] - }, - { - "name": "channel-messages", - "entry": "workloads/channel-messages.ts", - "args": [ - "200000" - ] - }, - { - "name": "atomics-contended", - "entry": "workloads/atomics-contended.ts", - "args": [ - "2" - ] - }, - { - "name": "value-objects", - "entry": "workloads/value-objects.ts", - "args": [ - "2" - ] - }, - { - "name": "value-records", - "entry": "workloads/value-records.ts", - "args": [ - "1" - ] - }, - { - "name": "region-builds", - "entry": "workloads/region-builds.ts", - "args": [ - "1" - ] - }, - { - "name": "optional-fields", - "entry": "workloads/optional-fields.ts", - "args": [ - "10" - ] - }, - { - "name": "record-boundaries", - "entry": "workloads/record-boundaries.ts", - "args": [ - "8" - ] - }, - { - "name": "number-maps", - "entry": "workloads/number-maps.ts", - "args": [ - "4" - ] - }, - { - "name": "published-graphs", - "entry": "workloads/published-graphs.ts", - "args": [ - "1" - ] - }, - { - "name": "validate-errors", - "entry": "workloads/validate-errors.ts", - "args": [ - "1" - ] - }, - { - "name": "csv-numbers", - "entry": "workloads/csv-numbers.ts", - "args": [ - "1" - ] - }, - { - "name": "log-lines", - "entry": "workloads/log-lines.ts", - "args": [ - "4" - ] - }, - { - "name": "unicode-lexer", - "entry": "workloads/unicode-lexer.ts", - "args": [ - "1" - ] - }, - { - "name": "log-summary", - "entry": "../builds/log-summary/main.ts", - "input": "log-summary", - "args": [ - "{input}", - "--min-status", - "200", - "--limit", - "12" - ] - }, - { - "name": "inventory-report", - "entry": "../builds/inventory-report/main.ts", - "input": "inventory-report", - "args": [ - "{input}" - ] - }, - { - "name": "cli-config", - "entry": "workloads/cli-config.ts", - "input": "cli-config", - "launches": 20, - "args": [ - "{input}", - "--env", - "production", - "--max-wave", - "12" - ] - }, - { - "name": "http-api", - "kind": "server", - "entry": "workloads/http-api.ts", - "args": [], - "client": { - "script": "clients/http-load.mjs", - "args": [ - "--connections=64", - "--threads=2", - "--warmup=4000", - "--requests=40000" - ] - } - } - ] -} diff --git a/benchmarks/runtime/workloads.mjs b/benchmarks/runtime/workloads.mjs new file mode 100644 index 000000000..e63ee98eb --- /dev/null +++ b/benchmarks/runtime/workloads.mjs @@ -0,0 +1,33 @@ +/* The runtime benchmark workloads: one JSON file per workload in + * workloads/, named after the workload, next to the entry programs. A file + * holds the workload's entry, args and options; changes that add + * different workloads add different files. */ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const CATEGORIES = ["cpu", "startup", "server"]; + +/* Server workloads run a listening program; startup-bound ones time + * several sequential launches per sample. */ +export const workloadCategory = (w) => + w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu"; + +/* The workloads of the suite at `suite` (a benchmarks/runtime directory), + * by category and then name. A checkout from before the per-workload files + * lists them in workloads.json, in its own order. */ +export function readWorkloads(suite) { + const legacy = join(suite, "workloads.json"); + if (existsSync(legacy)) return JSON.parse(readFileSync(legacy, "utf8")).workloads; + const dir = join(suite, "workloads"); + return readdirSync(dir) + .filter((file) => file.endsWith(".json")) + .map((file) => ({ + name: file.slice(0, -".json".length), + ...JSON.parse(readFileSync(join(dir, file), "utf8")), + })) + .sort( + (a, b) => + CATEGORIES.indexOf(workloadCategory(a)) - CATEGORIES.indexOf(workloadCategory(b)) || + (a.name < b.name ? -1 : a.name > b.name ? 1 : 0), + ); +} diff --git a/benchmarks/runtime/workloads/alloc-trees.json b/benchmarks/runtime/workloads/alloc-trees.json new file mode 100644 index 000000000..d4ae2ae72 --- /dev/null +++ b/benchmarks/runtime/workloads/alloc-trees.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/alloc-trees.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/ast-interp.json b/benchmarks/runtime/workloads/ast-interp.json new file mode 100644 index 000000000..1817541ac --- /dev/null +++ b/benchmarks/runtime/workloads/ast-interp.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/ast-interp.ts", + "args": [ + "5" + ] +} diff --git a/benchmarks/runtime/workloads/async-pipeline.json b/benchmarks/runtime/workloads/async-pipeline.json new file mode 100644 index 000000000..610f62793 --- /dev/null +++ b/benchmarks/runtime/workloads/async-pipeline.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/async-pipeline.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/atomics-contended.json b/benchmarks/runtime/workloads/atomics-contended.json new file mode 100644 index 000000000..7f78d265b --- /dev/null +++ b/benchmarks/runtime/workloads/atomics-contended.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/atomics-contended.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/runtime/workloads/channel-messages.json b/benchmarks/runtime/workloads/channel-messages.json new file mode 100644 index 000000000..56b84dda4 --- /dev/null +++ b/benchmarks/runtime/workloads/channel-messages.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/channel-messages.ts", + "args": [ + "200000" + ] +} diff --git a/benchmarks/runtime/workloads/cli-config.json b/benchmarks/runtime/workloads/cli-config.json new file mode 100644 index 000000000..db7a4c645 --- /dev/null +++ b/benchmarks/runtime/workloads/cli-config.json @@ -0,0 +1,12 @@ +{ + "entry": "workloads/cli-config.ts", + "input": "cli-config", + "launches": 20, + "args": [ + "{input}", + "--env", + "production", + "--max-wave", + "12" + ] +} diff --git a/benchmarks/runtime/workloads/csv-numbers.json b/benchmarks/runtime/workloads/csv-numbers.json new file mode 100644 index 000000000..acb3c21d7 --- /dev/null +++ b/benchmarks/runtime/workloads/csv-numbers.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/csv-numbers.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/cyclic-graphs.json b/benchmarks/runtime/workloads/cyclic-graphs.json new file mode 100644 index 000000000..2e62f7fdf --- /dev/null +++ b/benchmarks/runtime/workloads/cyclic-graphs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/cyclic-graphs.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/functional-pipeline.json b/benchmarks/runtime/workloads/functional-pipeline.json new file mode 100644 index 000000000..5ee3af902 --- /dev/null +++ b/benchmarks/runtime/workloads/functional-pipeline.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/functional-pipeline.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/http-api.json b/benchmarks/runtime/workloads/http-api.json new file mode 100644 index 000000000..145166945 --- /dev/null +++ b/benchmarks/runtime/workloads/http-api.json @@ -0,0 +1,14 @@ +{ + "kind": "server", + "entry": "workloads/http-api.ts", + "args": [], + "client": { + "script": "clients/http-load.mjs", + "args": [ + "--connections=64", + "--threads=2", + "--warmup=4000", + "--requests=40000" + ] + } +} diff --git a/benchmarks/runtime/workloads/intern-table-1.json b/benchmarks/runtime/workloads/intern-table-1.json new file mode 100644 index 000000000..8a2b253f6 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-1.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "table", + "1", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table-map-1.json b/benchmarks/runtime/workloads/intern-table-map-1.json new file mode 100644 index 000000000..fefafac67 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-map-1.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "map", + "1", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table-map.json b/benchmarks/runtime/workloads/intern-table-map.json new file mode 100644 index 000000000..136c6c31e --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table-map.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "map", + "8", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/intern-table.json b/benchmarks/runtime/workloads/intern-table.json new file mode 100644 index 000000000..86fa11413 --- /dev/null +++ b/benchmarks/runtime/workloads/intern-table.json @@ -0,0 +1,8 @@ +{ + "entry": "workloads/intern-table.ts", + "args": [ + "table", + "8", + "100000" + ] +} diff --git a/benchmarks/runtime/workloads/inventory-report.json b/benchmarks/runtime/workloads/inventory-report.json new file mode 100644 index 000000000..e75f5e3e5 --- /dev/null +++ b/benchmarks/runtime/workloads/inventory-report.json @@ -0,0 +1,7 @@ +{ + "entry": "../builds/inventory-report/main.ts", + "input": "inventory-report", + "args": [ + "{input}" + ] +} diff --git a/benchmarks/runtime/workloads/json-records.json b/benchmarks/runtime/workloads/json-records.json new file mode 100644 index 000000000..3263ac068 --- /dev/null +++ b/benchmarks/runtime/workloads/json-records.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/json-records.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/log-lines.json b/benchmarks/runtime/workloads/log-lines.json new file mode 100644 index 000000000..635bcc49f --- /dev/null +++ b/benchmarks/runtime/workloads/log-lines.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/log-lines.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/log-summary.json b/benchmarks/runtime/workloads/log-summary.json new file mode 100644 index 000000000..274a1cff6 --- /dev/null +++ b/benchmarks/runtime/workloads/log-summary.json @@ -0,0 +1,11 @@ +{ + "entry": "../builds/log-summary/main.ts", + "input": "log-summary", + "args": [ + "{input}", + "--min-status", + "200", + "--limit", + "12" + ] +} diff --git a/benchmarks/runtime/workloads/number-maps.json b/benchmarks/runtime/workloads/number-maps.json new file mode 100644 index 000000000..2adfbcc47 --- /dev/null +++ b/benchmarks/runtime/workloads/number-maps.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/number-maps.ts", + "args": [ + "4" + ] +} diff --git a/benchmarks/runtime/workloads/numeric-kernels.json b/benchmarks/runtime/workloads/numeric-kernels.json new file mode 100644 index 000000000..04718346b --- /dev/null +++ b/benchmarks/runtime/workloads/numeric-kernels.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/numeric-kernels.ts", + "args": [ + "0.5" + ] +} diff --git a/benchmarks/runtime/workloads/optional-fields.json b/benchmarks/runtime/workloads/optional-fields.json new file mode 100644 index 000000000..c54d87437 --- /dev/null +++ b/benchmarks/runtime/workloads/optional-fields.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/optional-fields.ts", + "args": [ + "10" + ] +} diff --git a/benchmarks/runtime/workloads/published-graphs.json b/benchmarks/runtime/workloads/published-graphs.json new file mode 100644 index 000000000..b22f5a0db --- /dev/null +++ b/benchmarks/runtime/workloads/published-graphs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/published-graphs.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/record-boundaries.json b/benchmarks/runtime/workloads/record-boundaries.json new file mode 100644 index 000000000..a35c346eb --- /dev/null +++ b/benchmarks/runtime/workloads/record-boundaries.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/record-boundaries.ts", + "args": [ + "8" + ] +} diff --git a/benchmarks/runtime/workloads/records-sort.json b/benchmarks/runtime/workloads/records-sort.json new file mode 100644 index 000000000..fd0097fe2 --- /dev/null +++ b/benchmarks/runtime/workloads/records-sort.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/records-sort.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/regex-logs.json b/benchmarks/runtime/workloads/regex-logs.json new file mode 100644 index 000000000..636db124e --- /dev/null +++ b/benchmarks/runtime/workloads/regex-logs.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/regex-logs.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/region-builds.json b/benchmarks/runtime/workloads/region-builds.json new file mode 100644 index 000000000..dc53dbc36 --- /dev/null +++ b/benchmarks/runtime/workloads/region-builds.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/region-builds.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/template-render.json b/benchmarks/runtime/workloads/template-render.json new file mode 100644 index 000000000..9897494f6 --- /dev/null +++ b/benchmarks/runtime/workloads/template-render.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/template-render.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/thread-slots-cache.json b/benchmarks/runtime/workloads/thread-slots-cache.json new file mode 100644 index 000000000..0eb6e11ce --- /dev/null +++ b/benchmarks/runtime/workloads/thread-slots-cache.json @@ -0,0 +1,9 @@ +{ + "entry": "workloads/thread-slots.ts", + "args": [ + "cache", + "8", + "50000", + "20" + ] +} diff --git a/benchmarks/runtime/workloads/thread-slots.json b/benchmarks/runtime/workloads/thread-slots.json new file mode 100644 index 000000000..b8ed5b140 --- /dev/null +++ b/benchmarks/runtime/workloads/thread-slots.json @@ -0,0 +1,9 @@ +{ + "entry": "workloads/thread-slots.ts", + "args": [ + "slots", + "8", + "50000", + "20" + ] +} diff --git a/benchmarks/runtime/workloads/typed-array-scan-workers.json b/benchmarks/runtime/workloads/typed-array-scan-workers.json new file mode 100644 index 000000000..c01893a2c --- /dev/null +++ b/benchmarks/runtime/workloads/typed-array-scan-workers.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/typed-array-scan-workers.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/typed-array-scan.json b/benchmarks/runtime/workloads/typed-array-scan.json new file mode 100644 index 000000000..0b2ba0592 --- /dev/null +++ b/benchmarks/runtime/workloads/typed-array-scan.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/typed-array-scan.ts", + "args": [ + "3" + ] +} diff --git a/benchmarks/runtime/workloads/unicode-lexer.json b/benchmarks/runtime/workloads/unicode-lexer.json new file mode 100644 index 000000000..91ccab087 --- /dev/null +++ b/benchmarks/runtime/workloads/unicode-lexer.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/unicode-lexer.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/validate-errors.json b/benchmarks/runtime/workloads/validate-errors.json new file mode 100644 index 000000000..b8d457897 --- /dev/null +++ b/benchmarks/runtime/workloads/validate-errors.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/validate-errors.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/value-objects.json b/benchmarks/runtime/workloads/value-objects.json new file mode 100644 index 000000000..276fa4ab8 --- /dev/null +++ b/benchmarks/runtime/workloads/value-objects.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/value-objects.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/runtime/workloads/value-records.json b/benchmarks/runtime/workloads/value-records.json new file mode 100644 index 000000000..84c0e12dd --- /dev/null +++ b/benchmarks/runtime/workloads/value-records.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/value-records.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/word-graph.json b/benchmarks/runtime/workloads/word-graph.json new file mode 100644 index 000000000..95ffb27e5 --- /dev/null +++ b/benchmarks/runtime/workloads/word-graph.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/word-graph.ts", + "args": [ + "2" + ] +} diff --git a/benchmarks/scaling/strace-workloads.mjs b/benchmarks/scaling/strace-workloads.mjs index b4db5af19..f0644cf13 100644 --- a/benchmarks/scaling/strace-workloads.mjs +++ b/benchmarks/scaling/strace-workloads.mjs @@ -7,6 +7,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads } from "../runtime/workloads.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const suite = resolve(here, "../runtime"); @@ -18,7 +19,7 @@ const { values } = parseArgs({ }, }); mkdirSync(values.out, { recursive: true }); -const { workloads } = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")); +const workloads = readWorkloads(suite); const cli = join(values.scriptc, "packages/cli/dist/bootstrap.js"); // Same inputs as scripts/bench-runtime.mjs generates. function input(kind) { diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index 4e4f2c245..61134fa4c 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -1,6 +1,6 @@ /* Runtime performance A/B harness for compiled programs. * - * Builds every workload in benchmarks/runtime/workloads.json with one or two + * Builds every workload in benchmarks/runtime/workloads/ with one or two * scriptc checkouts, verifies that each executable's output is identical to * Node's (stdout, stderr, exit status), then times interleaved launches and reports * per-workload medians with bootstrap confidence intervals for the @@ -23,6 +23,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads, workloadCategory } from "../benchmarks/runtime/workloads.mjs"; const repoRoot = fileURLToPath(new URL("..", import.meta.url)); const suiteRoot = join(repoRoot, "benchmarks/runtime"); @@ -62,7 +63,7 @@ evenly over those layouts, so verdicts cover code-placement luck instead of one placement. Use it for small codegen changes (e.g. --layouts=4 --runs=16). -Workload names are listed in benchmarks/runtime/workloads.json; for example +Workload names are the file names in benchmarks/runtime/workloads/*.json; for example --workloads=cli-config,http-api runs only the startup and server workloads. --baseline-optimization/--candidate-optimization pick each contender's @@ -88,13 +89,13 @@ assert.ok( "--layouts needs macOS or Linux", ); -const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8")); +const allWorkloads = readWorkloads(suiteRoot); const selected = values.workloads ? new Set(values.workloads.split(",")) : null; -const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name)); +const workloads = allWorkloads.filter((w) => selected === null || selected.has(w.name)); if (selected) { for (const name of selected) assert.ok( - manifest.workloads.some((w) => w.name === name), + allWorkloads.some((w) => w.name === name), `unknown workload ${name}`, ); } @@ -488,8 +489,6 @@ const report = { /* Workload kinds: "process" (default) times whole launches; `launches` > 1 * makes each sample the median of that many sequential launches (startup-bound * CLIs). "server" times the measured phase of a load-client session. */ -const categoryOf = (w) => - w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu"; for (const w of workloads) { assert.ok((w.kind ?? "process") === "process" || w.kind === "server", `${w.name}: unknown kind`); if (w.kind === "server") assert.ok(w.client?.script, `${w.name}: server workloads need a client`); @@ -569,7 +568,7 @@ try { const entry = { name: w.name, args, results: {} }; if (w.kind === "server") entry.kind = "server"; if ((w.launches ?? 1) > 1) entry.launches = w.launches; - entry.category = categoryOf(w); + entry.category = workloadCategory(w); const oracleFailure = server ? serverFailure(oracle) : oracle.timedOut ? "timed out" : ""; if (oracle.status !== 0 || oracleFailure) { entry.error = `node oracle failed (status ${oracle.status}${oracleFailure ? `, ${oracleFailure}` : ""}): ${oracle.stderr.slice(0, 2000)}`; diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index de3c0d0fc..1575fc5ee 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -27,6 +27,7 @@ import { availableParallelism } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { readWorkloads } from "../benchmarks/runtime/workloads.mjs"; import { cachegrindCounts, compareArtifact, @@ -173,9 +174,8 @@ function goodCodeSet() { const programs = []; if (sets.has("bench")) { const suite = join(programsRoot, "benchmarks/runtime"); - const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")); const seen = new Set(); - for (const w of manifest.workloads) { + for (const w of readWorkloads(suite)) { const entry = join(suite, w.entry); if (seen.has(entry)) continue; seen.add(entry); diff --git a/tests/harness/bench-dynamic-tier.test.ts b/tests/harness/bench-dynamic-tier.test.ts index de19f2b21..9276221ab 100644 --- a/tests/harness/bench-dynamic-tier.test.ts +++ b/tests/harness/bench-dynamic-tier.test.ts @@ -7,7 +7,7 @@ * release binary the gate measures, so other hosts and the sanitized lane * skip the builds. */ import { execFile } from "node:child_process"; -import { mkdirSync, readFileSync } from "node:fs"; +import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { promisify } from "node:util"; import { expect, test } from "vitest"; @@ -17,6 +17,7 @@ import { definedFunctionNames, dynamicTierSymbols, } from "../../scripts/zero-cost-compare.mjs"; +import { readWorkloads } from "../../benchmarks/runtime/workloads.mjs"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -31,11 +32,8 @@ interface Workload { // One program per entry file, named after its first workload, as the gate // names them. -const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")) as { - workloads: Workload[]; -}; const programs = new Map(); -for (const w of manifest.workloads) +for (const w of readWorkloads(suite) as Workload[]) if (![...programs.values()].includes(w.entry)) programs.set(`bench/${w.name}`, w.entry); test("every workload the gate lists as dynamic is a bench program", () => {