perf: shorten native bootstrap and cache CI toolchains

- Reuse decoded AST nodes, parent references, and recursive class type analysis during compilation.
- Split expression and library validation into smaller functions while preserving diagnostics.
- Shorten sanitizer bootstrap and reuse verified CI toolchains.
This commit is contained in:
Chris Tate
2026-09-30 15:12:04 -05:00
committed by GitHub
parent c257368c5e
commit 18c9a4233b
16 changed files with 2032 additions and 1362 deletions
+15 -4
View File
@@ -3,7 +3,7 @@ description: Reuse the pinned LLVM development distribution across CI jobs
runs:
using: composite
steps:
- uses: actions/cache@v4
- uses: actions/cache/restore@v4
id: llvm
with:
path: ${{ runner.temp }}/llvm-22.1.8
@@ -11,17 +11,28 @@ runs:
- name: Install LLVM system libraries
shell: bash
run: |
sudo apt-get update
sudo apt-get install --yes g++ zlib1g-dev libzstd-dev
if ! dpkg-query -W -f='${Status}\n' g++ zlib1g-dev libzstd-dev 2>/dev/null | grep -c '^install ok installed$' | grep -qx 3; then
sudo apt-get update
sudo apt-get install --yes g++ zlib1g-dev libzstd-dev
fi
- name: Download LLVM
if: steps.llvm.outputs.cache-hit != 'true'
shell: bash
run: |
curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz
curl -fL --connect-timeout 20 --max-time 300 --speed-limit 102400 --speed-time 30 \
--retry 2 --retry-max-time 600 -o "$RUNNER_TEMP/llvm.tar.xz" \
https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz
tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP"
mv "$RUNNER_TEMP/LLVM-22.1.8-Linux-X64" "$RUNNER_TEMP/llvm-22.1.8"
- name: Select LLVM
shell: bash
run: |
test "$("$RUNNER_TEMP/llvm-22.1.8/bin/llvm-config" --version)" = 22.1.8
echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV"
echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH"
# Publish setup work before the long tests, including when a later test fails.
- uses: actions/cache/save@v4
if: steps.llvm.outputs.cache-hit != 'true'
with:
path: ${{ runner.temp }}/llvm-22.1.8
key: ${{ steps.llvm.outputs.cache-primary-key }}
+14 -2
View File
@@ -19,13 +19,13 @@ runs:
if test "$RUNNER_OS" = Linux; then
echo "zig=$(zig version)" >> "$GITHUB_OUTPUT"
fi
- uses: actions/cache@v4
- uses: actions/cache/restore@v4
id: helper
if: inputs.helper == 'true'
with:
path: packages/llvm-${{ inputs.target }}/bin
key: helper-v1-${{ inputs.target }}-${{ steps.host.outputs.image }}-${{ steps.host.outputs.zig }}-${{ hashFiles('.node-version', '.github/actions/native-packages/action.yml', '.github/actions/llvm-linux/action.yml', 'native/llvm-codegen/**', 'packages/llvm-*/package.json', 'packages/llvm-*/scripts/**') }}
- uses: actions/cache@v4
- uses: actions/cache/restore@v4
id: runtime
with:
path: |
@@ -58,3 +58,15 @@ runs:
env:
SCRIPTC_PACKAGE_TARGET: ${{ inputs.target }}
run: node "packages/runtime-$SCRIPTC_PACKAGE_TARGET/scripts/verify.mjs"
- uses: actions/cache/save@v4
if: inputs.helper == 'true' && steps.helper.outputs.cache-hit != 'true'
with:
path: packages/llvm-${{ inputs.target }}/bin
key: ${{ steps.helper.outputs.cache-primary-key }}
- uses: actions/cache/save@v4
if: steps.runtime.outputs.cache-hit != 'true'
with:
path: |
packages/runtime-${{ inputs.target }}/artifacts
packages/runtime-${{ inputs.target }}/runtime-pack.json
key: ${{ steps.runtime.outputs.cache-primary-key }}
+19
View File
@@ -0,0 +1,19 @@
name: Pinned Zig
description: Reuse the verified Zig installation across hosted runners
runs:
using: composite
steps:
- uses: actions/cache/restore@v4
id: zig
with:
path: ${{ runner.tool_cache }}/zig/0.16.0
key: zig-0.16.0-${{ runner.os }}-${{ runner.arch }}-minisign-v1
# The installer checks the verified marker and runs `zig version` on hits.
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: actions/cache/save@v4
if: steps.zig.outputs.cache-hit != 'true'
with:
path: ${{ runner.tool_cache }}/zig/0.16.0
key: ${{ steps.zig.outputs.cache-primary-key }}
+5 -13
View File
@@ -7,7 +7,7 @@ on:
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
cancel-in-progress: true
jobs:
native_macos:
@@ -189,9 +189,7 @@ jobs:
cache: pnpm
- uses: ./.github/actions/llvm-linux
- run: pnpm install --frozen-lockfile
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: ./.github/actions/zig
- uses: ./.github/actions/native-packages
with:
target: linux-x64-gnu
@@ -265,9 +263,7 @@ jobs:
cache: pnpm
- uses: ./.github/actions/llvm-linux
- run: pnpm install --frozen-lockfile
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: ./.github/actions/zig
- uses: ./.github/actions/native-packages
with:
target: linux-x64-gnu
@@ -451,9 +447,7 @@ jobs:
SCRIPTC_TARGET: x86_64-windows-gnu
steps:
- uses: actions/checkout@v4
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: ./.github/actions/zig
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
@@ -560,9 +554,7 @@ jobs:
- run: pnpm install --frozen-lockfile
- name: Build Windows helper
run: pnpm --filter @scriptc/llvm-win32-x64-msvc build:native
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: ./.github/actions/zig
- name: Build Windows runtime pack
run: pnpm --filter @scriptc/runtime-win32-x64-msvc build:native
- run: pnpm build
@@ -1,6 +1,7 @@
import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
import { RuntimeOptionalLocals } from "./runtime-optional-locals.js";
import { sanitizeUnregisteredClassTypes } from "./sanitize-class-types.js";
import { UnregisteredClassTypes } from "./unregistered-class-types.js";
import { buildUnionNarrow } from "./union-narrow.js";
import { planUnionRetag, buildUnionRetag, planRecordUnionWrap, buildRecordUnionWrap } from "./union-retag.js";
import type { WidthLift } from "./width-lift.js";
@@ -3792,9 +3793,12 @@ export class Lowerer {
// fences), so drop it — guarded by a reference scan, with the
// validator's registration check as the backstop for anything that
// does slip through with a live reference.
const brokenGlobals = this.globalsList.filter((g) => this.typeNamesUnregisteredClass(g.type));
const unregistered = new UnregisteredClassTypes(
(id) => this.shapes.get(id), (id) => this.unions.get(id), (name) => this.classes.has(name),
);
const brokenGlobals = this.globalsList.filter((g) => unregistered.has(g.type));
const brokenLocalFns = functions.filter((fn) =>
fn.locals.some((l) => this.typeNamesUnregisteredClass(l.type)),
fn.locals.some((l) => unregistered.has(l.type)),
);
if (brokenGlobals.length > 0 || brokenLocalFns.length > 0) {
const referencedIn = (root: unknown): Set<string> => {
@@ -3837,7 +3841,7 @@ export class Lowerer {
// those out from under them.
const referenced = referencedIn([fn.body, fn.params, fn.captures ?? [], fn.classCaptures ?? []]);
fn.locals = fn.locals.filter(
(l) => referenced.has(l.id) || !this.typeNamesUnregisteredClass(l.type),
(l) => referenced.has(l.id) || !unregistered.has(l.type),
);
}
}
@@ -0,0 +1,70 @@
import { expect, test } from "vitest";
import { F64, VOID, type IrRecordShape, type IrType, type IrUnionDef } from "../../ir/ir.js";
import { UnregisteredClassTypes } from "./unregistered-class-types.js";
const record = (shapeId: string): IrType => ({ kind: "record", shapeId });
const union = (unionId: string): IrType => ({ kind: "union", unionId });
const missing: IrType = { kind: "object", className: "Missing" };
test("cyclic references wait for sibling checks before caching safe types", () => {
const records = new Map<string, IrRecordShape>([
["a", { id: "a", fields: [{ name: "cycle", type: record("b") }, { name: "missing", type: missing }] }],
["b", { id: "b", fields: [{ name: "back", type: union("u") }, { name: "safe", type: record("safe") }] }],
["safe", { id: "safe", fields: [{ name: "value", type: F64 }] }],
]);
const unions = new Map<string, IrUnionDef>([["u", { id: "u", arms: [record("a")] }]]);
const index = new UnregisteredClassTypes((id) => records.get(id), (id) => unions.get(id), () => false);
expect(index.has(record("a"))).toBe(true);
expect(index.has(record("b"))).toBe(true);
expect(index.has(union("u"))).toBe(true);
expect(index.has(record("safe"))).toBe(false);
});
test("shared recursive graphs are traversed once across repeated local types", () => {
const records = new Map<string, IrRecordShape>();
const count = 2048;
for (let i = 0; i < count; i++) records.set(String(i), {
id: String(i), fields: [{ name: "next", type: record(String((i + 1) % count)) }],
});
let lookups = 0;
const index = new UnregisteredClassTypes((id) => { lookups++; return records.get(id); }, () => undefined, () => true);
for (let i = 0; i < count; i++) {
expect(index.has({ kind: "array", elem: record(String(i)) })).toBe(false);
}
expect(lookups).toBe(count);
});
test("nested containers, signatures and index values expose missing instances", () => {
const shape: IrRecordShape = { id: "r", fields: [], indexValue: missing };
const definition: IrUnionDef = { id: "u", arms: [record("r")] };
const index = new UnregisteredClassTypes(() => shape, () => definition, () => false);
const types: IrType[] = [
{ kind: "array", elem: missing }, { kind: "set", elem: missing },
{ kind: "map", key: missing, value: F64 }, { kind: "map", key: F64, value: missing },
{ kind: "promise", inner: missing },
{ kind: "func", params: [missing], ret: VOID }, { kind: "func", params: [], ret: missing },
record("r"), union("u"),
];
for (const type of types) expect(index.has(type)).toBe(true);
expect(index.has({ kind: "classval", className: "Missing" })).toBe(false);
});
test("record and union IDs have independent identities", () => {
const index = new UnregisteredClassTypes(
(id) => ({ id, fields: [] }), (id) => ({ id, arms: [missing] }), () => false,
);
expect(index.has(record("same"))).toBe(false);
expect(index.has(union("same"))).toBe(true);
expect(index.has(record("same"))).toBe(false);
});
test("fresh indexes observe later class registration and type-table completion", () => {
let shape: IrRecordShape | undefined;
const classes = new Set<string>();
const fresh = () => new UnregisteredClassTypes(() => shape, () => undefined, (name) => classes.has(name));
expect(fresh().has(record("r"))).toBe(false);
shape = { id: "r", fields: [{ name: "value", type: missing }] };
expect(fresh().has(record("r"))).toBe(true);
classes.add("Missing");
expect(fresh().has(record("r"))).toBe(false);
});
@@ -0,0 +1,77 @@
import type { IrRecordShape, IrType, IrUnionDef } from "../../ir/ir.js";
/** Reuse class-reference checks while the type tables and class registry are
* fixed. Create a new index after lowering or retention changes those tables. */
export class UnregisteredClassTypes {
private readonly safeRecords = new Set<string>();
private readonly safeUnions = new Set<string>();
private readonly badRecords = new Set<string>();
private readonly badUnions = new Set<string>();
constructor(
private readonly record: (id: string) => IrRecordShape | undefined,
private readonly union: (id: string) => IrUnionDef | undefined,
private readonly hasClass: (name: string) => boolean,
) {}
has(root: IrType): boolean {
const pending: IrType[] = [root];
const records = new Set<string>();
const unions = new Set<string>();
let bad = false;
while (pending.length > 0 && !bad) {
const type = pending.pop()!;
switch (type.kind) {
case "object":
bad = !this.hasClass(type.className);
break;
case "array":
case "set":
pending.push(type.elem);
break;
case "map":
pending.push(type.value, type.key);
break;
case "promise":
pending.push(type.inner);
break;
case "func":
pending.push(type.ret);
for (let i = type.params.length - 1; i >= 0; i--) pending.push(type.params[i]!);
break;
case "record": {
if (this.badRecords.has(type.shapeId)) { bad = true; break; }
if (this.safeRecords.has(type.shapeId) || records.has(type.shapeId)) break;
records.add(type.shapeId);
const shape = this.record(type.shapeId);
if (shape !== undefined) {
for (let i = shape.fields.length - 1; i >= 0; i--) pending.push(shape.fields[i]!.type);
if (shape.indexValue !== undefined) pending.push(shape.indexValue);
}
break;
}
case "union": {
if (this.badUnions.has(type.unionId)) { bad = true; break; }
if (this.safeUnions.has(type.unionId) || unions.has(type.unionId)) break;
unions.add(type.unionId);
const definition = this.union(type.unionId);
if (definition !== undefined) {
for (let i = definition.arms.length - 1; i >= 0; i--) pending.push(definition.arms[i]!);
}
break;
}
}
}
if (bad) {
// Only the root is proven bad; visited sibling types may be safe.
if (root.kind === "record") this.badRecords.add(root.shapeId);
if (root.kind === "union") this.badUnions.add(root.unionId);
} else {
// A cycle back-edge is provisional. Cache negative answers only after
// the entire reachable graph has been checked, including its siblings.
for (const id of records) this.safeRecords.add(id);
for (const id of unions) this.safeUnions.add(id);
}
return bad;
}
}
@@ -187,6 +187,29 @@ test("checker handles reject cross-file, wrong-kind and nil identities", () => {
expect(() => file.resolve(`0.${AstKind.SourceFile}.${file.root.path}`)).toThrow("nil");
});
test("direct node slots preserve lazy parents, identity and invalid-index checks", () => {
let materialized = 0;
const file = new AstFile(decoded.get("main.ts")!.bytes, undefined, () => { materialized++; });
let index = file.wire.nodeCount - 1;
while (file.wire.kind(index) === KIND_NODE_LIST || file.wire.semanticParent(index) <= 1) index--;
const node = file.node(index);
expect(materialized).toBe(1);
expect(file.node(index)).toBe(node);
expect(materialized).toBe(1);
const parent = node.parent;
expect(parent?.index).toBe(file.wire.semanticParent(index));
expect(materialized).toBe(2);
expect(node.parent).toBe(parent);
expect(file.node(parent!.index)).toBe(parent);
expect(materialized).toBe(2);
expect(file.root.parent).toBeUndefined();
expect(file.root.parent).toBeUndefined();
for (const invalid of [-1, 0, 0.5, NaN, Infinity, file.wire.nodeCount]) {
expect(() => file.node(invalid)).toThrow();
expect(() => file.list(invalid)).toThrow();
}
});
test("a source view requires a source-file root", () => {
const bytes = decoded.get("main.ts")!.bytes.slice();
const words = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+19 -9
View File
@@ -11,8 +11,10 @@ import { SyntaxKind, NodeFlags } from "./enums.js";
export class AstFile {
readonly wire: AstWireFile;
readonly root: AstNode;
private readonly nodes = new Map<number, AstNode>();
private readonly lists = new Map<number, AstNode[]>();
// Wire ids are bounded dense indices. Direct slots preserve lazy object
// creation and avoid hashing and temporary optional boxes on every read.
private readonly nodes: (AstNode | undefined)[];
private readonly lists: (AstNode[] | undefined)[];
private readonly references = new Map<number, AstFileReference[]>();
private readonly structuredNodes = new Map<number, AstNode[]>();
private readonly strings = new Map<number, string[]>();
@@ -24,8 +26,10 @@ export class AstFile {
private readonly materialized?: () => void,
) {
this.wire = new AstWireFile(bytes);
this.nodes = new Array<AstNode | undefined>(this.wire.nodeCount);
this.lists = new Array<AstNode[] | undefined>(this.wire.nodeCount);
this.root = new AstNode(this, 1);
this.nodes.set(1, this.root);
this.nodes[1] = this.root;
}
/** Checker factories also return AST fragments. Only source-file
@@ -37,22 +41,22 @@ export class AstFile {
}
node(index: number): AstNode {
const existing = this.nodes.get(index);
const existing = this.nodes[index];
if (existing !== undefined) return existing;
if (index === 0 || this.wire.kind(index) === KIND_NODE_LIST) throw new AstDecodeError("expected a node index");
const node = new AstNode(this, index);
this.nodes.set(index, node);
this.nodes[index] = node;
this.materialized?.();
return node;
}
list(index: number): AstNode[] {
const existing = this.lists.get(index);
const existing = this.lists[index];
if (existing !== undefined) return existing;
const nodes: AstNode[] = [];
for (const child of this.wire.list(index)) nodes.push(this.node(child));
this.listMetadata?.(nodes, this.wire.pos(index) >>> 0, this.wire.end(index) >>> 0);
this.lists.set(index, nodes);
this.lists[index] = nodes;
this.materialized?.();
return nodes;
}
@@ -108,6 +112,8 @@ export class AstNode {
readonly end: number;
readonly flags: NodeFlags;
readonly data: number;
private parentResolved = false;
private parentCache: AstNode | undefined;
constructor(readonly file: AstFile, readonly index: number) {
// Materialize immutable scalar metadata once. Lowering repeatedly reads
@@ -122,8 +128,12 @@ export class AstNode {
get id(): string { return `${this.index}.${this.kind}.${this.file.root.path}`; }
get parent(): AstNode | undefined {
const index = this.file.wire.semanticParent(this.index);
return index === 0 || index === this.index ? undefined : this.file.node(index);
if (!this.parentResolved) {
const index = this.file.wire.semanticParent(this.index);
this.parentCache = index === 0 || index === this.index ? undefined : this.file.node(index);
this.parentResolved = true;
}
return this.parentCache;
}
get text(): string | undefined { return this.file.wire.text(this.index); }
get rawText(): string | undefined { return this.file.wire.rawText(this.index); }
+16
View File
@@ -118,6 +118,22 @@ function expressionModule(expr: IrExpr, unions: IrUnionDef[]): IrModule {
};
}
test("library callbacks retain child, specialized, and generic result diagnostics", () => {
const expr: IrExpr = {
kind: "libCall", fn: "cp.execFile", type: F64, loc,
args: [
{ kind: "strLit", value: "tool", type: STRING, loc },
{ kind: "arrayLit", elems: [], type: arrayOf(STRING), loc },
{ kind: "boolLit", value: true, type: F64, loc },
],
};
expect(validateModule(expressionModule(expr, [])).map((error) => error.message)).toEqual([
"in main: boolLit must be bool",
"in main: libCall cp.execFile callback must be a non-rest void function with at most three parameters",
"in main: libCall cp.execFile must be child, got f64",
]);
});
test("nullish chains retain child-before-parent diagnostic order", () => {
const at = (start: number) => ({ ...loc, start });
const expr: IrExpr = {
File diff suppressed because it is too large Load Diff
+11 -2
View File
@@ -7,8 +7,11 @@ const errors = child.stderr;
if (input === null || output === null || errors === null) throw new Error("missing pipe");
let echoed = "";
let outputEnded = false;
let exited = false;
let exitCode: number | null = null;
output.on("data", (chunk) => { echoed += chunk.toString(); });
output.on("end", () => { console.log("echo", JSON.stringify(echoed)); });
output.on("end", () => { outputEnded = true; });
errors.on("data", (chunk) => { process.stderr.write(chunk); });
input.on("finish", () => { console.log("finish", input.writable); });
input.on("error", (err) => { console.log("error", err.message); });
@@ -16,4 +19,10 @@ console.log("initial", input.writable);
console.log("writes", input.write("hé"), input.write(new Uint8Array([108, 108, 111, 10])));
input.end();
console.log("ended", input.writable);
child.on("exit", (code) => { console.log("exit", code); });
child.on("exit", (code) => { exited = true; exitCode = code; });
// Exit and stdout EOF can arrive in either order. Close follows both.
child.on("close", () => {
if (!exited || !outputEnded) throw new Error("close before exit or stdout end");
console.log("exit", exitCode);
console.log("echo", JSON.stringify(echoed));
});
+22 -1
View File
@@ -1,5 +1,6 @@
import { F64, typeKey, type IrExpr, type IrModule, type IrType } from "../../../packages/compiler/src/ir/ir.js";
import { F64, typeKey, type IrExpr, type IrModule, type IrRecordShape, type IrType, type IrUnionDef } from "../../../packages/compiler/src/ir/ir.js";
import { sanitizeUnregisteredClassTypes } from "../../../packages/compiler/src/frontend/lowering/sanitize-class-types.js";
import { UnregisteredClassTypes } from "../../../packages/compiler/src/frontend/lowering/unregistered-class-types.js";
const loc = { file: "class-types.ts", start: 0, end: 1 };
const missing: IrType = { kind: "object", className: "Fenced" };
@@ -21,6 +22,26 @@ for (const record of module.records ?? []) {
if (record.indexValue) console.log(typeKey(record.indexValue));
}
for (const union of module.unions ?? []) console.log(union.arms.map(typeKey).join(","));
const records = new Map<string, IrRecordShape>();
const unions = new Map<string, IrUnionDef>();
for (let i = 0; i < 1024; i++) records.set(String(i), { id: String(i), fields: [
{ name: "next", type: { kind: "record", shapeId: String((i + 1) % 1024) } },
] });
let lookups = 0;
const references = new UnregisteredClassTypes(
(id) => { lookups++; return records.get(id); }, (id) => unions.get(id), (name) => name === "Registered",
);
let bad = false;
for (let i = 0; i < 1024; i++) bad = references.has({ kind: "record", shapeId: String(i) }) || bad;
console.log(bad, lookups);
records.set("a", { id: "a", fields: [
{ name: "cycle", type: { kind: "record", shapeId: "b" } }, { name: "missing", type: missing },
] });
records.set("b", { id: "b", fields: [{ name: "back", type: { kind: "union", unionId: "u" } }] });
unions.set("u", { id: "u", arms: [{ kind: "record", shapeId: "a" }] });
const cyclic = new UnregisteredClassTypes((id) => records.get(id), (id) => unions.get(id), () => false);
console.log(cyclic.has({ kind: "record", shapeId: "a" }), cyclic.has({ kind: "record", shapeId: "b" }),
cyclic.has({ kind: "union", unionId: "u" }), cyclic.has({ kind: "record", shapeId: "0" }));
console.log(missing.kind, kept.kind, F64.kind);
sanitizeUnregisteredClassTypes(module, (name) => name === "Registered");
console.log(expr.type.kind);
@@ -25,6 +25,7 @@ test("native class type cleanup updates shared IR without dynamic snapshots", as
}
expect(native.stdout).toBe(oracle.stdout);
expect(native.stdout).toMatch(/^f64 f64\n/);
expect(native.stdout).toContain("false 1024\ntrue true true false\n");
expect(native.stdout).toMatch(/object object f64\nf64\n$/);
} finally { rmSync(directory, { recursive: true, force: true }); }
}, 300_000);
@@ -7,6 +7,7 @@ import { pathToFileURL } from "node:url";
import { WASI } from "node:wasi";
import { expect, test } from "vitest";
import type { NativeToolchainManifest } from "../../packages/compiler/src/native/toolchain.js";
import { RUNTIME_ABI_MARKER } from "../../packages/compiler/src/backend/runtime-abi.js";
import { bootstrapStep } from "./self-hosting-timing.js";
const root = join(import.meta.dirname, "../..");
@@ -39,10 +40,10 @@ test("the production CLI relocates, builds programs, and rebuilds itself with No
const distribution = join(directory, "distribution");
await bootstrapStep("build production CLI seed", () =>
exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-cli.mts"), distribution], {
// The seed builds an instrumented second generation below. Run
// the full sanitizer proof with that generation instead of paying
// for two instrumented traversals of the complete compiler graph.
...options, env: { ...process.env, SCRIPTC_SAN: "", SCRIPTC_NATIVE_EMIT_IR: "1" },
// Each lane executes its full compiler traversal in the appropriate
// seed. Instrumenting the seed avoids another ordinary traversal
// before the sanitizer can exercise the compiler.
...options, env: { ...process.env, SCRIPTC_SAN: sanitize ? "1" : "", SCRIPTC_NATIVE_EMIT_IR: "1" },
}));
// All compiler assets must survive moving the complete distribution.
const relocated = join(directory, "relocated");
@@ -88,16 +89,18 @@ test("the production CLI relocates, builds programs, and rebuilds itself with No
const entry = join(root, "packages/compiler/src/native/cli.ts");
const ffi = join(directory, ".scriptc/distribution-seed/compiler.ffi.json");
const rebuilt = executable("scriptc-rebuilt");
// Optimize the compiler that will process the full graph again. Small
// programs below still exercise development output.
// The plain lane optimizes the next generation for its fixed-point
// check. The sanitizer lane has already instrumented the seed; it
// rebuilds an instrumented development compiler and executes it below.
// --strip keeps debug metadata out of the IR/LLVM equality comparison.
const self = await bootstrapStep("production CLI rebuilds itself", () =>
invoke(seed, ["build", entry, "-o", rebuilt, "--strip", "--keep-llvm", "--emit-ir", "--ffi", ffi,
...(sanitize ? ["--sanitize"] : [])],
...(sanitize ? ["--sanitize", "--optimization=dev"] : [])],
"scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n"));
expect(self.trim()).toBe(rebuilt);
// All command, library, and dynamic probes run inside the instrumented
// compiler in the sanitizer lane, including their failure paths.
const probe = sanitize ? rebuilt : seed;
const probe = seed;
expect(await invoke(probe, ["--help"])).toContain("scriptc build");
expect((await invoke(probe, ["--version"])).trim()).toBe(manifest.compiler_version);
const checkProgram = async (compiler: string, source: string, extra: string[] = []) => {
@@ -214,8 +217,21 @@ test("the production CLI relocates, builds programs, and rebuilds itself with No
assert.ok(actual.functions.length > 1000);
assert.deepEqual(validateModule(actual), []);
assert.ok(isDeepStrictEqual(actual, expected), 'native self-lowering must match the Node seed');
assert.ok(readFileSync(process.argv[3], 'utf8') === readFileSync(process.argv[4], 'utf8'),
'native LLVM emission must match the Node seed');
const seedLlvm = readFileSync(process.argv[3], 'utf8');
let nativeLlvm = readFileSync(process.argv[4], 'utf8');
if (${sanitize}) {
// The Node sanitizer builds runtime sources directly; the native
// executable path also emits the runtime ABI check. Pin that
// difference before comparing the rest of the complete module.
const declaration = 'declare void @' + ${JSON.stringify(RUNTIME_ABI_MARKER)} + '()\\n';
const call = ' call void @' + ${JSON.stringify(RUNTIME_ABI_MARKER)} + '()\\n';
for (const line of [declaration, call]) {
assert.equal(seedLlvm.split(line).length, 1);
assert.equal(nativeLlvm.split(line).length, 2);
nativeLlvm = nativeLlvm.replace(line, '');
}
}
assert.ok(seedLlvm === nativeLlvm, 'native LLVM emission must match the Node seed');
`, join(seedDirectory, "cli.ir.json"), join(directory, "cli.ir.json"),
join(seedDirectory, "cli.ll"), join(directory, "cli.ll")], options));
expect(comparison.stdout).toBe("");
@@ -226,17 +242,20 @@ test("the production CLI relocates, builds programs, and rebuilds itself with No
await checkProgram(rebuilt, receiverSample);
await checkProgram(rebuilt, join(root, "tests/corpus/1010-json-stringify-space.ts"));
// Compare the LLVM used to build the second generation with its own
// output. Retaining the build input avoids repeating the seed's work.
const seedLlvm = join(directory, "cli.ll");
const rebuiltLlvm = join(directory, "rebuilt.ll");
await bootstrapStep("rebuilt compiler emits itself", () =>
invoke(rebuilt, ["build", entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", ffi]));
// Executable output adds a runtime ABI check; textual LLVM emission
// omits it. Compare all other output without changing either mode.
const withoutAbiCheck = (path: string): string => readFileSync(path, "utf8")
.replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "")
.replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, "");
expect(withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), "native compiler generations must emit identical LLVM").toBe(true);
// The plain lane owns the third-generation fixed point. The sanitizer
// lane already runs the full frontend and emitter under ASan in the
// self-rebuild, compares both artifacts, and executes the new compiler.
if (!sanitize) {
const seedLlvm = join(directory, "cli.ll");
const rebuiltLlvm = join(directory, "rebuilt.ll");
await bootstrapStep("rebuilt compiler emits itself", () =>
invoke(rebuilt, ["build", entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", ffi]));
// Executable output adds a runtime ABI check; textual LLVM emission
// omits it. Compare all other output without changing either mode.
const withoutAbiCheck = (path: string): string => readFileSync(path, "utf8")
.replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "")
.replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, "");
expect(withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), "native compiler generations must emit identical LLVM").toBe(true);
}
} finally { rmSync(directory, { recursive: true, force: true }); }
}, sanitize ? 10_800_000 : 5_400_000);
@@ -135,6 +135,37 @@ export function validatorCases(): ValidatorCase[] {
add("library signature", (m) => {
m.functions[0]!.body = [expression({ kind: "libCall", fn: "number.isFinite", args: [strLit("wrong", loc)], type: BOOL, loc })];
}, "expected f64");
for (const invalid of [false, true]) {
add(`library custom union result ${invalid ? "invalid" : "valid"}`, (m) => {
m.unions = [{ id: "env", arms: [STRING, { kind: "undefinedT" }] }];
m.functions[0]!.body = [expression({ kind: "libCall", fn: "process.envGet",
args: [strLit("PATH", loc)], type: invalid ? F64 : { kind: "union", unionId: "env" }, loc })];
}, invalid ? "must return the 'string | undefined' union" : undefined);
add(`library custom record result ${invalid ? "invalid" : "valid"}`, (m) => {
m.records = [{ id: "dirent", fields: [
{ name: "%dtype", type: invalid ? STRING : F64 },
{ name: "name", type: STRING }, { name: "parentPath", type: STRING },
] }];
m.functions[0]!.body = [expression({ kind: "libCall", fn: "fs.readdirTypesSync",
args: [strLit(".", loc)], type: arrayOf({ kind: "record", shapeId: "dirent" }), loc })];
}, invalid ? "must return the Dirent record array" : undefined);
}
for (const [fn, diagnostic] of [
["net.sockRead", "must return the 'Buffer | null' union"],
["spawnRes.status", "must return the 'number | null' union"],
["error.new", "must return a builtin error class"],
["stream.prop", "receiver must be a stream-hierarchy object"],
["emitter.new", "must return '%EventEmitter'"],
] as const) {
add(`library specialized validation ${fn}`, (m) => {
m.functions[0]!.body = [expression({ kind: "libCall", fn, args: [], type: F64, loc })];
}, diagnostic);
}
add("library callback validation retains generic result check", (m) => {
m.functions[0]!.body = [expression({ kind: "libCall", fn: "cp.execFile", args: [
strLit("tool", loc), { kind: "arrayLit", elems: [], type: arrayOf(STRING), loc }, numLit(0, loc),
], type: F64, loc })];
}, "must be child");
add("duplicate records", (m) => { m.records = [{ id: "r", fields: [] }, { id: "r", fields: [] }]; }, "duplicate record");
add("record ordering", (m) => {
m.records = [{ id: "r", fields: [{ name: "z", type: F64 }, { name: "a", type: STRING }] }];