mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-27 06:43:19 +08:00
Add a runner for Node's test/parallel files
tests/node-parallel/run.mjs builds files from a Node.js checkout's test/parallel with scriptc, runs each executable in that directory, and compares its exit status with Node's, writing a pass, fail or refused verdict and the first error line per file (results.json, results.md). --common minimal (the default) runs the files against a minimal test/common (common/index.cjs, common/wpt.cjs) whose helpers keep Node's semantics, so a verdict measures the module under test; --common node uses Node's own test/common.
This commit is contained in:
Chris Tate
@@ -0,0 +1,23 @@
|
||||
# Node test/parallel files
|
||||
|
||||
`run.mjs` builds files from a Node.js checkout's `test/parallel` with scriptc, runs each executable in that directory, and compares its exit status with Node's. It records a verdict per file: `pass` (same exit status), `fail` (different exit status), or `refused` (the build failed), with the first error line.
|
||||
|
||||
```bash
|
||||
git clone --depth 1 --branch v24.22.0 https://github.com/nodejs/node /path/to/node
|
||||
node tests/node-parallel/run.mjs --node /path/to/node --pattern '^test-(url-|whatwg-url-|querystring)'
|
||||
```
|
||||
|
||||
Options:
|
||||
|
||||
- `--node`: a Node.js checkout, or its `test` directory. Use the version in `.node-version`.
|
||||
- `--pattern`: a regular expression over file names in `test/parallel`.
|
||||
- `--common`: `minimal` (the default) or `node`.
|
||||
- `minimal` copies the selected files into `<out>/tree/test/parallel` and uses `common/index.cjs` and `common/wpt.cjs` from this directory as `test/common/index.js` and `test/common/wpt.js`, with the checkout's `test/fixtures` and `test/common/fixtures.js`. The minimal common has the helpers these files use (`mustCall`, `mustNotCall`, `skip`, `expectWarning`, `invalidArgTypeHelper`, `spawnPromisified`, `hasIntl`, `isWindows`, ...) with Node's semantics, so a verdict measures the module under test rather than Node's full `test/common`.
|
||||
- `node` builds the files in place against Node's own `test/common`.
|
||||
- `--out`: where executables and results go (default `node_modules/.cache/scriptc-node-parallel`).
|
||||
- `--cli`: the scriptc CLI entry (default `packages/cli/dist/main.js`).
|
||||
- `--jobs`: concurrent files (default 4).
|
||||
|
||||
Node runs each file with its `// Flags:` line. Results are written to `<out>/results.json` and `<out>/results.md`.
|
||||
|
||||
Before a lane relies on the minimal common, run its files under Node in `<out>/tree/test/parallel`: they should all pass there.
|
||||
@@ -0,0 +1,131 @@
|
||||
"use strict";
|
||||
// A minimal test/common for building Node's test/parallel files with scriptc:
|
||||
// the helpers those files use, with the semantics of Node's
|
||||
// test/common/index.js. Add a helper here, with Node's semantics, when a
|
||||
// lane's files need it.
|
||||
const assert = require("assert");
|
||||
const { inspect } = require("util");
|
||||
|
||||
const hasIntl = true;
|
||||
const isDebug = false;
|
||||
const isWindows = process.platform === "win32";
|
||||
const mustCallChecks = [];
|
||||
|
||||
function runCallChecks(exitCode) {
|
||||
if (exitCode !== 0) return;
|
||||
const failed = mustCallChecks.filter((context) => {
|
||||
if ("minimum" in context) return context.actual < context.minimum;
|
||||
return context.actual !== context.exact;
|
||||
});
|
||||
for (const context of failed) {
|
||||
console.log(
|
||||
"Mismatched %s function calls. Expected %s, actual %d.",
|
||||
context.name,
|
||||
String(context.exact ?? context.minimum),
|
||||
context.actual,
|
||||
);
|
||||
}
|
||||
if (failed.length) process.exit(1);
|
||||
}
|
||||
|
||||
function noop() {}
|
||||
|
||||
function mustCallInner(fn, criteria, field) {
|
||||
if (typeof fn === "number") {
|
||||
criteria = fn;
|
||||
fn = noop;
|
||||
} else if (fn === undefined) {
|
||||
fn = noop;
|
||||
}
|
||||
if (criteria === undefined) criteria = 1;
|
||||
const context = { actual: 0, name: fn.name || "<anonymous>" };
|
||||
context[field] = criteria;
|
||||
if (mustCallChecks.length === 0) process.on("exit", runCallChecks);
|
||||
mustCallChecks.push(context);
|
||||
return function (...args) {
|
||||
context.actual++;
|
||||
return fn.apply(this, args);
|
||||
};
|
||||
}
|
||||
|
||||
function mustCall(fn, exact) {
|
||||
return mustCallInner(fn, exact, "exact");
|
||||
}
|
||||
|
||||
function mustCallAtLeast(fn, minimum) {
|
||||
return mustCallInner(fn, minimum, "minimum");
|
||||
}
|
||||
|
||||
function mustNotCall(msg) {
|
||||
return function mustNotCall() {
|
||||
assert.fail(msg || "function should not have been called");
|
||||
};
|
||||
}
|
||||
|
||||
function skip(msg) {
|
||||
console.log(`1..0 # Skipped: ${msg}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function expectWarning(name, expected, code) {
|
||||
if (typeof expected === "string") expected = [[expected, code]];
|
||||
else if (expected.length !== 0 && !Array.isArray(expected[0]))
|
||||
expected = [[expected[0], expected[1]]];
|
||||
const check = mustCall((warning) => {
|
||||
const [message, warningCode] = expected.shift();
|
||||
assert.strictEqual(warning.name, name);
|
||||
if (typeof message === "string") assert.strictEqual(warning.message, message);
|
||||
else assert.match(warning.message, message);
|
||||
assert.strictEqual(warning.code, warningCode);
|
||||
}, expected.length);
|
||||
process.on("warning", (warning) => {
|
||||
if (warning.name !== name)
|
||||
throw new TypeError(`"${warning.name}" was triggered without being expected.`);
|
||||
check(warning);
|
||||
});
|
||||
}
|
||||
|
||||
function invalidArgTypeHelper(input) {
|
||||
if (input == null) return ` Received ${input}`;
|
||||
if (typeof input === "function") return ` Received function ${input.name}`;
|
||||
if (typeof input === "object") {
|
||||
const name = input.constructor?.name;
|
||||
if (typeof name === "string" && name !== "") return ` Received an instance of ${name}`;
|
||||
return ` Received ${inspect(input, { depth: -1 })}`;
|
||||
}
|
||||
let inspected = inspect(input, { colors: false });
|
||||
if (inspected.length > 28) inspected = `${inspected.slice(inspected, 0, 25)}...`;
|
||||
return ` Received type ${typeof input} (${inspected})`;
|
||||
}
|
||||
|
||||
function spawnPromisified(...args) {
|
||||
const { spawn } = require("child_process");
|
||||
let stderr = "";
|
||||
let stdout = "";
|
||||
const child = spawn(...args);
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (data) => {
|
||||
stderr += data;
|
||||
});
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (data) => {
|
||||
stdout += data;
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on("close", (code, signal) => resolve({ code, signal, stderr, stdout }));
|
||||
child.on("error", (code, signal) => reject({ code, signal, stderr, stdout }));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
hasIntl,
|
||||
isDebug,
|
||||
isWindows,
|
||||
mustCall,
|
||||
mustCallAtLeast,
|
||||
mustNotCall,
|
||||
skip,
|
||||
expectWarning,
|
||||
invalidArgTypeHelper,
|
||||
spawnPromisified,
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
"use strict";
|
||||
// The harness object of Node's test/common/wpt.js, without its WPT runner.
|
||||
const assert = require("assert");
|
||||
|
||||
const harness = {
|
||||
test: (fn, desc) => {
|
||||
try {
|
||||
fn();
|
||||
} catch (err) {
|
||||
console.error(`In ${desc}:`);
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
assert_equals: assert.strictEqual,
|
||||
assert_true: (value, message) => assert.strictEqual(value, true, message),
|
||||
assert_false: (value, message) => assert.strictEqual(value, false, message),
|
||||
assert_throws: (code, func, desc) => {
|
||||
assert.throws(
|
||||
func,
|
||||
function (err) {
|
||||
return typeof err === "object" && "name" in err && err.name.startsWith(code.name);
|
||||
},
|
||||
desc,
|
||||
);
|
||||
},
|
||||
assert_array_equals: assert.deepStrictEqual,
|
||||
assert_unreached(desc) {
|
||||
assert.fail(`Reached unreachable code: ${desc}`);
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = { harness };
|
||||
@@ -0,0 +1,134 @@
|
||||
// Builds Node's test/parallel files with scriptc and runs them next to Node.
|
||||
// See README.md for the options.
|
||||
import { spawn } from "node:child_process";
|
||||
import {
|
||||
cpSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const repo = resolve(here, "../..");
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
node: { type: "string" },
|
||||
pattern: { type: "string" },
|
||||
common: { type: "string", default: "minimal" },
|
||||
out: { type: "string", default: join(repo, "node_modules/.cache/scriptc-node-parallel") },
|
||||
cli: { type: "string", default: join(repo, "packages/cli/dist/main.js") },
|
||||
jobs: { type: "string", default: "4" },
|
||||
},
|
||||
});
|
||||
if (!values.node || !values.pattern || !["minimal", "node"].includes(values.common)) {
|
||||
console.error(
|
||||
"usage: node tests/node-parallel/run.mjs --node <node checkout> --pattern <regex> [--common minimal|node] [--out <dir>] [--cli <scriptc main.js>] [--jobs <n>]",
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
const nodeTest = existsSync(join(values.node, "test", "parallel"))
|
||||
? join(values.node, "test")
|
||||
: values.node;
|
||||
const pattern = new RegExp(values.pattern);
|
||||
const out = resolve(values.out);
|
||||
mkdirSync(out, { recursive: true });
|
||||
|
||||
// The minimal lane copies the selected files into a tree whose test/common is
|
||||
// this directory's index.cjs and wpt.cjs, next to the checkout's fixtures and
|
||||
// common/fixtures.js.
|
||||
let testRoot = nodeTest;
|
||||
const files = readdirSync(join(nodeTest, "parallel"))
|
||||
.filter((name) => name.endsWith(".js") && pattern.test(name))
|
||||
.sort();
|
||||
if (values.common === "minimal") {
|
||||
testRoot = join(out, "tree", "test");
|
||||
rmSync(join(out, "tree"), { recursive: true, force: true });
|
||||
mkdirSync(join(testRoot, "parallel"), { recursive: true });
|
||||
mkdirSync(join(testRoot, "common"), { recursive: true });
|
||||
for (const name of files)
|
||||
cpSync(join(nodeTest, "parallel", name), join(testRoot, "parallel", name));
|
||||
symlinkSync(join(nodeTest, "fixtures"), join(testRoot, "fixtures"));
|
||||
cpSync(join(nodeTest, "common", "fixtures.js"), join(testRoot, "common", "fixtures.js"));
|
||||
cpSync(join(here, "common", "index.cjs"), join(testRoot, "common", "index.js"));
|
||||
cpSync(join(here, "common", "wpt.cjs"), join(testRoot, "common", "wpt.js"));
|
||||
}
|
||||
|
||||
function run(cmd, args, opts) {
|
||||
return new Promise((done) => {
|
||||
const child = spawn(cmd, args, { ...opts, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (d) => (stdout += d));
|
||||
child.stderr.on("data", (d) => (stderr += d));
|
||||
const timer = setTimeout(() => child.kill("SIGKILL"), opts.timeout);
|
||||
child.on("close", (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
done({ code, signal, stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const firstCause = (text) =>
|
||||
(
|
||||
text.split("\n").find((line) => /error|Error|assert|SC\d{4}/.test(line)) ??
|
||||
text.split("\n")[0] ??
|
||||
""
|
||||
)
|
||||
.replaceAll(testRoot, "test")
|
||||
.trim()
|
||||
.slice(0, 300);
|
||||
|
||||
async function one(file) {
|
||||
const dir = join(testRoot, "parallel");
|
||||
const source = readFileSync(join(dir, file), "utf8");
|
||||
const flags = (/^\/\/ Flags: (.*)$/m.exec(source)?.[1] ?? "").split(/\s+/).filter(Boolean);
|
||||
const node = await run(process.execPath, [...flags, file], { cwd: dir, timeout: 120_000 });
|
||||
const exe = join(out, "bin", `${file}.exe`);
|
||||
mkdirSync(join(out, "bin"), { recursive: true });
|
||||
const build = await run(process.execPath, [values.cli, "build", file, "-o", exe], {
|
||||
cwd: dir,
|
||||
timeout: 900_000,
|
||||
});
|
||||
const result = { file, flags: flags.join(" "), nodeExit: node.code };
|
||||
if (build.code !== 0) {
|
||||
result.verdict = "refused";
|
||||
result.cause = firstCause(build.stdout + build.stderr);
|
||||
return result;
|
||||
}
|
||||
const native = await run(exe, [], { cwd: dir, timeout: 120_000 });
|
||||
result.exit = native.code ?? native.signal;
|
||||
result.verdict = native.code === node.code ? "pass" : "fail";
|
||||
if (result.verdict !== "pass") result.cause = firstCause(native.stderr + native.stdout);
|
||||
return result;
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const queue = [...files];
|
||||
await Promise.all(
|
||||
Array.from({ length: Number(values.jobs) }, async () => {
|
||||
for (let file = queue.shift(); file !== undefined; file = queue.shift()) {
|
||||
const result = await one(file);
|
||||
results.push(result);
|
||||
console.log(`${result.verdict.padEnd(8)} ${file}${result.cause ? ` ${result.cause}` : ""}`);
|
||||
}
|
||||
}),
|
||||
);
|
||||
results.sort((a, b) => (a.file < b.file ? -1 : 1));
|
||||
const count = (verdict) => results.filter((r) => r.verdict === verdict).length;
|
||||
const table = [
|
||||
"| file | verdict | first cause |",
|
||||
"| --- | --- | --- |",
|
||||
...results.map((r) => `| ${r.file} | ${r.verdict} | ${(r.cause ?? "").replaceAll("|", "\\|")} |`),
|
||||
"",
|
||||
`pass ${count("pass")}, fail ${count("fail")}, refused ${count("refused")} of ${results.length} (test/common: ${values.common})`,
|
||||
];
|
||||
writeFileSync(join(out, "results.json"), `${JSON.stringify(results, null, 2)}\n`);
|
||||
writeFileSync(join(out, "results.md"), `${table.join("\n")}\n`);
|
||||
console.log(table.at(-1));
|
||||
Reference in New Issue
Block a user