mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-30 07:14:39 +08:00
Keep each runtime benchmark workload in its own file
workloads.json listed every workload in one array, and its README table listed them in one hand order, so changes that each added a workload appended at the same place and conflicted. Each workload is now a JSON file in benchmarks/runtime/workloads/ named after it, read through benchmarks/runtime/workloads.mjs by the bench runner, the zero-cost gate, the strace script and the dynamic-tier test, ordered by category and then name. The loader still reads workloads.json in a checkout from before this change, so the gate can compare against one. The README table follows the same order.
This commit is contained in:
Chris Tate
@@ -4,34 +4,34 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea
|
||||
|
||||
| Workload | Category | Exercises |
|
||||
| --- | --- | --- |
|
||||
| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation |
|
||||
| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` |
|
||||
| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch |
|
||||
| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` |
|
||||
| `template-render` | cpu | template literals, escaping, `map`/`join` |
|
||||
| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records |
|
||||
| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out |
|
||||
| `word-graph` | cpu | string-keyed Map/Set, BFS |
|
||||
| `alloc-trees` | cpu | allocation-heavy trees, immutable object-spread updates |
|
||||
| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner |
|
||||
| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` |
|
||||
| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage |
|
||||
| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables |
|
||||
| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread |
|
||||
| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any |
|
||||
| `ast-interp` | cpu | tokenizer, recursive-descent parser, class dispatch |
|
||||
| `async-pipeline` | cpu | `async`/`await`, `Promise.all` fan-out |
|
||||
| `atomics-contended` | cpu | four worker threads updating `SharedArrayBuffer` counters with `Atomics`: one shared counter, one counter per worker, a `compareExchange` loop, and `Atomics.wait`/`notify` handoffs |
|
||||
| `channel-messages` | cpu | 200,000 MessageChannel messages in one thread, half numbers and half small objects, all posted before the port delivers any |
|
||||
| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` |
|
||||
| `cyclic-graphs` | cpu | object graphs with back references built and dropped: trees with parent and sibling links, rings of records sharing a long-lived registry, closures capturing their owner |
|
||||
| `functional-pipeline` | cpu | `filter`/`map`/`reduce`/`flatMap` chains, closures, small records |
|
||||
| `intern-table`, `intern-table-map`, `intern-table-1`, `intern-table-map-1` | cpu | worker threads interning the same type-like objects under string keys and records under number keys, then hits and misses: one shared `InternTable` per kind of key, or a module-level `Map` per thread (the `-map` entries), with 8 threads or 1 (the `-1` entries); under Node.js every thread has its own tables |
|
||||
| `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation |
|
||||
| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe |
|
||||
| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs |
|
||||
| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes |
|
||||
| `numeric-kernels` | cpu | sieve, hashing, `number[][]` matmul, `Float64Array` |
|
||||
| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields |
|
||||
| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read |
|
||||
| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions |
|
||||
| `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` |
|
||||
| `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` |
|
||||
| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds |
|
||||
| `template-render` | cpu | template literals, escaping, `map`/`join` |
|
||||
| `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread |
|
||||
| `typed-array-scan`, `typed-array-scan-workers` | cpu | element-by-element scan loops over `Uint8Array`, `Uint16Array`, `Int32Array` and `Float64Array`; the second entry is a worker-enabled executable (it can start a worker), so each access also checks for shared storage |
|
||||
| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts |
|
||||
| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains |
|
||||
| `value-objects` | cpu | small immutable records and class instances: vector helpers, method chains, particle updates |
|
||||
| `value-records` | cpu | records that hold strings, spread state updates and destructured tuples: token helpers, an immutable reducer, tuple returns |
|
||||
| `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds |
|
||||
| `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields |
|
||||
| `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions |
|
||||
| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes |
|
||||
| `published-graphs` | cpu | syntax trees with parent links and a symbol table, published with `@scriptc/threads` and then read |
|
||||
| `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains |
|
||||
| `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` |
|
||||
| `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe |
|
||||
| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts |
|
||||
| `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs |
|
||||
| `word-graph` | cpu | string-keyed Map/Set, BFS |
|
||||
| `cli-config` | startup | a `deploy-plan` CLI: read a ~110 KB JSON manifest, validate, interpolate env values, order 80 services into dependency waves, print a summary |
|
||||
| `http-api` | server | a `node:http` JSON API: routing, query parsing, a seeded catalog, per-cart state, typed JSON bodies, `ETag`/`If-None-Match`, auth and request-id headers |
|
||||
|
||||
@@ -47,7 +47,7 @@ Node is timed in the same interleaved loop and every row reports candidate/Node;
|
||||
|
||||
## Workload kinds
|
||||
|
||||
Entries in `workloads.json` are process workloads unless they set `kind`.
|
||||
Each workload is a JSON file in `workloads/`, named after the workload, that holds its `entry`, `args` and options; adding a workload adds its file and its row in the table above, kept in category and then name order. Workloads are process workloads unless they set `kind`.
|
||||
|
||||
Process workloads time whole launches (wall time, startup included). An optional `launches` count makes each sample the median of that many sequential launches; `cli-config` uses 20 so that a startup-dominated command (a few milliseconds compiled, tens of milliseconds under Node) is measured with low noise. Its table row is labeled `(per launch)`. On a heavily loaded macOS host, whole batches of launches can run several times slower (samples become bimodal), so judge startup changes on the Linux lane or a quiet machine.
|
||||
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
{
|
||||
"workloads": [
|
||||
{
|
||||
"name": "json-records",
|
||||
"entry": "workloads/json-records.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regex-logs",
|
||||
"entry": "workloads/regex-logs.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ast-interp",
|
||||
"entry": "workloads/ast-interp.ts",
|
||||
"args": [
|
||||
"5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "records-sort",
|
||||
"entry": "workloads/records-sort.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "template-render",
|
||||
"entry": "workloads/template-render.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "functional-pipeline",
|
||||
"entry": "workloads/functional-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "async-pipeline",
|
||||
"entry": "workloads/async-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "word-graph",
|
||||
"entry": "workloads/word-graph.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "alloc-trees",
|
||||
"entry": "workloads/alloc-trees.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cyclic-graphs",
|
||||
"entry": "workloads/cyclic-graphs.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "numeric-kernels",
|
||||
"entry": "workloads/numeric-kernels.ts",
|
||||
"args": [
|
||||
"0.5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "typed-array-scan",
|
||||
"entry": "workloads/typed-array-scan.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "typed-array-scan-workers",
|
||||
"entry": "workloads/typed-array-scan-workers.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "intern-table",
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"table",
|
||||
"8",
|
||||
"100000"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "intern-table-map",
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"map",
|
||||
"8",
|
||||
"100000"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "intern-table-1",
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"table",
|
||||
"1",
|
||||
"100000"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "intern-table-map-1",
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"map",
|
||||
"1",
|
||||
"100000"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "thread-slots",
|
||||
"entry": "workloads/thread-slots.ts",
|
||||
"args": [
|
||||
"slots",
|
||||
"8",
|
||||
"50000",
|
||||
"20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "thread-slots-cache",
|
||||
"entry": "workloads/thread-slots.ts",
|
||||
"args": [
|
||||
"cache",
|
||||
"8",
|
||||
"50000",
|
||||
"20"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "channel-messages",
|
||||
"entry": "workloads/channel-messages.ts",
|
||||
"args": [
|
||||
"200000"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "atomics-contended",
|
||||
"entry": "workloads/atomics-contended.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "value-objects",
|
||||
"entry": "workloads/value-objects.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "value-records",
|
||||
"entry": "workloads/value-records.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "region-builds",
|
||||
"entry": "workloads/region-builds.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "optional-fields",
|
||||
"entry": "workloads/optional-fields.ts",
|
||||
"args": [
|
||||
"10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "record-boundaries",
|
||||
"entry": "workloads/record-boundaries.ts",
|
||||
"args": [
|
||||
"8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "number-maps",
|
||||
"entry": "workloads/number-maps.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "published-graphs",
|
||||
"entry": "workloads/published-graphs.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "validate-errors",
|
||||
"entry": "workloads/validate-errors.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "csv-numbers",
|
||||
"entry": "workloads/csv-numbers.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "log-lines",
|
||||
"entry": "workloads/log-lines.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "unicode-lexer",
|
||||
"entry": "workloads/unicode-lexer.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "log-summary",
|
||||
"entry": "../builds/log-summary/main.ts",
|
||||
"input": "log-summary",
|
||||
"args": [
|
||||
"{input}",
|
||||
"--min-status",
|
||||
"200",
|
||||
"--limit",
|
||||
"12"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "inventory-report",
|
||||
"entry": "../builds/inventory-report/main.ts",
|
||||
"input": "inventory-report",
|
||||
"args": [
|
||||
"{input}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "cli-config",
|
||||
"entry": "workloads/cli-config.ts",
|
||||
"input": "cli-config",
|
||||
"launches": 20,
|
||||
"args": [
|
||||
"{input}",
|
||||
"--env",
|
||||
"production",
|
||||
"--max-wave",
|
||||
"12"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "http-api",
|
||||
"kind": "server",
|
||||
"entry": "workloads/http-api.ts",
|
||||
"args": [],
|
||||
"client": {
|
||||
"script": "clients/http-load.mjs",
|
||||
"args": [
|
||||
"--connections=64",
|
||||
"--threads=2",
|
||||
"--warmup=4000",
|
||||
"--requests=40000"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/* The runtime benchmark workloads: one JSON file per workload in
|
||||
* workloads/, named after the workload, next to the entry programs. A file
|
||||
* holds the workload's entry, args and options; changes that add
|
||||
* different workloads add different files. */
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const CATEGORIES = ["cpu", "startup", "server"];
|
||||
|
||||
/* Server workloads run a listening program; startup-bound ones time
|
||||
* several sequential launches per sample. */
|
||||
export const workloadCategory = (w) =>
|
||||
w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu";
|
||||
|
||||
/* The workloads of the suite at `suite` (a benchmarks/runtime directory),
|
||||
* by category and then name. A checkout from before the per-workload files
|
||||
* lists them in workloads.json, in its own order. */
|
||||
export function readWorkloads(suite) {
|
||||
const legacy = join(suite, "workloads.json");
|
||||
if (existsSync(legacy)) return JSON.parse(readFileSync(legacy, "utf8")).workloads;
|
||||
const dir = join(suite, "workloads");
|
||||
return readdirSync(dir)
|
||||
.filter((file) => file.endsWith(".json"))
|
||||
.map((file) => ({
|
||||
name: file.slice(0, -".json".length),
|
||||
...JSON.parse(readFileSync(join(dir, file), "utf8")),
|
||||
}))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
CATEGORIES.indexOf(workloadCategory(a)) - CATEGORIES.indexOf(workloadCategory(b)) ||
|
||||
(a.name < b.name ? -1 : a.name > b.name ? 1 : 0),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/alloc-trees.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/ast-interp.ts",
|
||||
"args": [
|
||||
"5"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/async-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/atomics-contended.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/channel-messages.ts",
|
||||
"args": [
|
||||
"200000"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"entry": "workloads/cli-config.ts",
|
||||
"input": "cli-config",
|
||||
"launches": 20,
|
||||
"args": [
|
||||
"{input}",
|
||||
"--env",
|
||||
"production",
|
||||
"--max-wave",
|
||||
"12"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/csv-numbers.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/cyclic-graphs.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/functional-pipeline.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"kind": "server",
|
||||
"entry": "workloads/http-api.ts",
|
||||
"args": [],
|
||||
"client": {
|
||||
"script": "clients/http-load.mjs",
|
||||
"args": [
|
||||
"--connections=64",
|
||||
"--threads=2",
|
||||
"--warmup=4000",
|
||||
"--requests=40000"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"table",
|
||||
"1",
|
||||
"100000"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"map",
|
||||
"1",
|
||||
"100000"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"map",
|
||||
"8",
|
||||
"100000"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"entry": "workloads/intern-table.ts",
|
||||
"args": [
|
||||
"table",
|
||||
"8",
|
||||
"100000"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"entry": "../builds/inventory-report/main.ts",
|
||||
"input": "inventory-report",
|
||||
"args": [
|
||||
"{input}"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/json-records.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/log-lines.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"entry": "../builds/log-summary/main.ts",
|
||||
"input": "log-summary",
|
||||
"args": [
|
||||
"{input}",
|
||||
"--min-status",
|
||||
"200",
|
||||
"--limit",
|
||||
"12"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/number-maps.ts",
|
||||
"args": [
|
||||
"4"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/numeric-kernels.ts",
|
||||
"args": [
|
||||
"0.5"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/optional-fields.ts",
|
||||
"args": [
|
||||
"10"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/published-graphs.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/record-boundaries.ts",
|
||||
"args": [
|
||||
"8"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/records-sort.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/regex-logs.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/region-builds.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/template-render.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"entry": "workloads/thread-slots.ts",
|
||||
"args": [
|
||||
"cache",
|
||||
"8",
|
||||
"50000",
|
||||
"20"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"entry": "workloads/thread-slots.ts",
|
||||
"args": [
|
||||
"slots",
|
||||
"8",
|
||||
"50000",
|
||||
"20"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/typed-array-scan-workers.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/typed-array-scan.ts",
|
||||
"args": [
|
||||
"3"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/unicode-lexer.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/validate-errors.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/value-objects.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/value-records.ts",
|
||||
"args": [
|
||||
"1"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"entry": "workloads/word-graph.ts",
|
||||
"args": [
|
||||
"2"
|
||||
]
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { readWorkloads } from "../runtime/workloads.mjs";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const suite = resolve(here, "../runtime");
|
||||
@@ -18,7 +19,7 @@ const { values } = parseArgs({
|
||||
},
|
||||
});
|
||||
mkdirSync(values.out, { recursive: true });
|
||||
const { workloads } = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8"));
|
||||
const workloads = readWorkloads(suite);
|
||||
const cli = join(values.scriptc, "packages/cli/dist/bootstrap.js");
|
||||
// Same inputs as scripts/bench-runtime.mjs generates.
|
||||
function input(kind) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/* Runtime performance A/B harness for compiled programs.
|
||||
*
|
||||
* Builds every workload in benchmarks/runtime/workloads.json with one or two
|
||||
* Builds every workload in benchmarks/runtime/workloads/ with one or two
|
||||
* scriptc checkouts, verifies that each executable's output is identical to
|
||||
* Node's (stdout, stderr, exit status), then times interleaved launches and reports
|
||||
* per-workload medians with bootstrap confidence intervals for the
|
||||
@@ -23,6 +23,7 @@ import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { readWorkloads, workloadCategory } from "../benchmarks/runtime/workloads.mjs";
|
||||
|
||||
const repoRoot = fileURLToPath(new URL("..", import.meta.url));
|
||||
const suiteRoot = join(repoRoot, "benchmarks/runtime");
|
||||
@@ -62,7 +63,7 @@ evenly over those layouts, so verdicts cover code-placement luck instead of one
|
||||
placement.
|
||||
Use it for small codegen changes (e.g. --layouts=4 --runs=16).
|
||||
|
||||
Workload names are listed in benchmarks/runtime/workloads.json; for example
|
||||
Workload names are the file names in benchmarks/runtime/workloads/*.json; for example
|
||||
--workloads=cli-config,http-api runs only the startup and server workloads.
|
||||
|
||||
--baseline-optimization/--candidate-optimization pick each contender's
|
||||
@@ -88,13 +89,13 @@ assert.ok(
|
||||
"--layouts needs macOS or Linux",
|
||||
);
|
||||
|
||||
const manifest = JSON.parse(readFileSync(join(suiteRoot, "workloads.json"), "utf8"));
|
||||
const allWorkloads = readWorkloads(suiteRoot);
|
||||
const selected = values.workloads ? new Set(values.workloads.split(",")) : null;
|
||||
const workloads = manifest.workloads.filter((w) => selected === null || selected.has(w.name));
|
||||
const workloads = allWorkloads.filter((w) => selected === null || selected.has(w.name));
|
||||
if (selected) {
|
||||
for (const name of selected)
|
||||
assert.ok(
|
||||
manifest.workloads.some((w) => w.name === name),
|
||||
allWorkloads.some((w) => w.name === name),
|
||||
`unknown workload ${name}`,
|
||||
);
|
||||
}
|
||||
@@ -488,8 +489,6 @@ const report = {
|
||||
/* Workload kinds: "process" (default) times whole launches; `launches` > 1
|
||||
* makes each sample the median of that many sequential launches (startup-bound
|
||||
* CLIs). "server" times the measured phase of a load-client session. */
|
||||
const categoryOf = (w) =>
|
||||
w.kind === "server" ? "server" : (w.launches ?? 1) > 1 ? "startup" : "cpu";
|
||||
for (const w of workloads) {
|
||||
assert.ok((w.kind ?? "process") === "process" || w.kind === "server", `${w.name}: unknown kind`);
|
||||
if (w.kind === "server") assert.ok(w.client?.script, `${w.name}: server workloads need a client`);
|
||||
@@ -569,7 +568,7 @@ try {
|
||||
const entry = { name: w.name, args, results: {} };
|
||||
if (w.kind === "server") entry.kind = "server";
|
||||
if ((w.launches ?? 1) > 1) entry.launches = w.launches;
|
||||
entry.category = categoryOf(w);
|
||||
entry.category = workloadCategory(w);
|
||||
const oracleFailure = server ? serverFailure(oracle) : oracle.timedOut ? "timed out" : "";
|
||||
if (oracle.status !== 0 || oracleFailure) {
|
||||
entry.error = `node oracle failed (status ${oracle.status}${oracleFailure ? `, ${oracleFailure}` : ""}): ${oracle.stderr.slice(0, 2000)}`;
|
||||
|
||||
@@ -27,6 +27,7 @@ import { availableParallelism } from "node:os";
|
||||
import { dirname, join, relative, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { readWorkloads } from "../benchmarks/runtime/workloads.mjs";
|
||||
import {
|
||||
cachegrindCounts,
|
||||
compareArtifact,
|
||||
@@ -173,9 +174,8 @@ function goodCodeSet() {
|
||||
const programs = [];
|
||||
if (sets.has("bench")) {
|
||||
const suite = join(programsRoot, "benchmarks/runtime");
|
||||
const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8"));
|
||||
const seen = new Set();
|
||||
for (const w of manifest.workloads) {
|
||||
for (const w of readWorkloads(suite)) {
|
||||
const entry = join(suite, w.entry);
|
||||
if (seen.has(entry)) continue;
|
||||
seen.add(entry);
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* release binary the gate measures, so other hosts and the sanitized lane
|
||||
* skip the builds. */
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdirSync, readFileSync } from "node:fs";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, test } from "vitest";
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
definedFunctionNames,
|
||||
dynamicTierSymbols,
|
||||
} from "../../scripts/zero-cost-compare.mjs";
|
||||
import { readWorkloads } from "../../benchmarks/runtime/workloads.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = join(import.meta.dirname, "../..");
|
||||
@@ -31,11 +32,8 @@ interface Workload {
|
||||
|
||||
// One program per entry file, named after its first workload, as the gate
|
||||
// names them.
|
||||
const manifest = JSON.parse(readFileSync(join(suite, "workloads.json"), "utf8")) as {
|
||||
workloads: Workload[];
|
||||
};
|
||||
const programs = new Map<string, string>();
|
||||
for (const w of manifest.workloads)
|
||||
for (const w of readWorkloads(suite) as Workload[])
|
||||
if (![...programs.values()].includes(w.entry)) programs.set(`bench/${w.name}`, w.entry);
|
||||
|
||||
test("every workload the gate lists as dynamic is a bench program", () => {
|
||||
|
||||
Reference in New Issue
Block a user