mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-02 08:35:07 +08:00
feat: ship the self-hosted native scriptc CLI (#585)
* feat: ship the self-hosted native scriptc CLI - Run the shared compiler and CLI natively, including library builds and compile-time evaluation. - Install native platform packages and ship relocatable standalone distributions. - Preserve program semantics and reuse validated build artifacts for fast rebuilds. * fix: complete native CLI integration and validation - Preserve library diagnostics and run recursive compiler work on the main stack. - Restore packaged helper permissions and exercise native npm and Wasm library builds. - Recover interrupted Sandbox status probes within the original command deadline.
This commit is contained in:
Chris Tate
@@ -0,0 +1,10 @@
|
||||
const empty = new Array<null>(3).fill(null);
|
||||
console.log(JSON.stringify(empty), empty.map((value, index) => String(index) + ":" + String(value === null)).join(","));
|
||||
const mixed: (number | null)[] = [1, 2, 3, 4];
|
||||
console.log(JSON.stringify(mixed.fill(null, 1, -1)));
|
||||
console.log(JSON.stringify(mixed.fill(7, 2)));
|
||||
let calls = 0;
|
||||
function nil(): null { calls++; return null; }
|
||||
console.log(JSON.stringify(mixed.fill(nil())), calls);
|
||||
const missing = new Array<undefined>(2).fill(undefined);
|
||||
console.log(JSON.stringify(missing), missing.map((value, index) => String(index) + ":" + String(value === undefined)).join(","));
|
||||
@@ -0,0 +1,12 @@
|
||||
const order = [2, 0, 1];
|
||||
const mapped: number[] = new Array<number>(3).fill(-1);
|
||||
order.forEach((oldIndex, newIndex) => { mapped[oldIndex] = newIndex; });
|
||||
console.log(mapped.join(","));
|
||||
|
||||
const candidates = new Map<string, number[]>();
|
||||
candidates.set("entry", [1, 2]);
|
||||
const included = new Array<boolean>(3).fill(false);
|
||||
for (const candidate of candidates.get("entry") ?? new Array<number>()) {
|
||||
if (included[candidate] !== true) included[candidate] = true;
|
||||
}
|
||||
console.log(included.join(","));
|
||||
@@ -0,0 +1,65 @@
|
||||
type ValueType = { kind: "string" } | { kind: "dyn" };
|
||||
type Expr =
|
||||
| { kind: "literal"; value: string; type: ValueType }
|
||||
| { kind: "object"; fields: { key: string; value: Expr }[]; type: ValueType }
|
||||
| { kind: "call"; args: Expr[]; name: string; type: ValueType };
|
||||
const DYNAMIC: ValueType = { kind: "dyn" };
|
||||
|
||||
function assemble(groups: string[][]): Expr {
|
||||
let fields: { key: string; value: Expr }[] = [];
|
||||
let acc: Expr | null = null;
|
||||
const flush = (): void => {
|
||||
if (fields.length === 0) return;
|
||||
const chunk: Expr = { kind: "object", fields, type: DYNAMIC };
|
||||
acc = acc === null ? chunk : { kind: "call", args: [acc, chunk], name: "merge", type: DYNAMIC };
|
||||
fields = [];
|
||||
};
|
||||
for (const group of groups) {
|
||||
for (const key of group) fields.push({ key, value: { kind: "literal", value: key, type: { kind: "string" } } });
|
||||
flush();
|
||||
if (group.length > 10) {
|
||||
acc ??= { kind: "object", fields: [], type: DYNAMIC };
|
||||
acc = { kind: "call", args: [acc], name: "large", type: DYNAMIC };
|
||||
}
|
||||
}
|
||||
if (acc !== null) return acc;
|
||||
return { kind: "object", fields: [], type: DYNAMIC };
|
||||
}
|
||||
|
||||
for (const groups of [[], [["headers"]], [["a"], ["b", "c"]]]) {
|
||||
const expression = assemble(groups);
|
||||
console.log(expression.kind, expression.type.kind, JSON.stringify(expression));
|
||||
}
|
||||
|
||||
function inspect(expression: Expr): void {
|
||||
console.log(expression.kind, expression.type.kind, JSON.stringify(expression));
|
||||
}
|
||||
|
||||
async function transfer(): Promise<Expr> {
|
||||
let value: Expr = { kind: "call", args: [], name: "initial", type: DYNAMIC };
|
||||
const replacement: Expr = { kind: "object", fields: [], type: DYNAMIC };
|
||||
const update = (): void => { value = replacement; };
|
||||
update();
|
||||
inspect(value);
|
||||
let assigned: Expr = { kind: "literal", value: "initial", type: DYNAMIC };
|
||||
assigned = value;
|
||||
inspect(assigned);
|
||||
const values: Expr[] = [value];
|
||||
const holder: { value: Expr | null } = { value };
|
||||
inspect(values[0]!);
|
||||
if (holder.value !== null) inspect(holder.value);
|
||||
console.log(values[0] === replacement, holder.value === replacement);
|
||||
return (value);
|
||||
}
|
||||
|
||||
inspect(await transfer());
|
||||
|
||||
function subset(value: Expr): Expr | null {
|
||||
if (value.kind === "literal") {
|
||||
const selected: Extract<Expr, { kind: "literal" | "object" }> | null = value;
|
||||
return selected;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const selected = subset({ kind: "literal", value: "subset", type: DYNAMIC });
|
||||
if (selected !== null) inspect(selected);
|
||||
@@ -0,0 +1,17 @@
|
||||
import { createHash, createHmac } from "node:crypto";
|
||||
|
||||
const environment: Record<string, string | undefined> = { present: "hello" };
|
||||
for (const name of ["present", "missing"]) {
|
||||
const value = environment[name];
|
||||
const text: string = value ?? "";
|
||||
console.log(createHash("sha256").update(text).digest("hex"));
|
||||
console.log(createHmac("sha256", "secret").update(text).digest("hex"));
|
||||
if (value !== undefined) console.log(createHash("sha256").update(value).digest("hex"));
|
||||
}
|
||||
|
||||
const buffers: (Buffer | undefined)[] = [Buffer.from("hello"), undefined];
|
||||
for (let index = 0; index < buffers.length; index++) {
|
||||
const buffer = buffers[index] ?? Buffer.from("");
|
||||
console.log(createHash("sha256").update(buffer).digest("hex"));
|
||||
console.log(createHmac("sha256", "secret").update(buffer).digest("hex"));
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const root = mkdtempSync(join(tmpdir(), "scriptc-readonly-rm-"));
|
||||
try {
|
||||
const file = join(root, "readonly.txt");
|
||||
writeFileSync(file, "keep until removed");
|
||||
chmodSync(file, 0o400);
|
||||
rmSync(file);
|
||||
console.log(existsSync(file));
|
||||
const nested = join(root, "tree", "nested");
|
||||
mkdirSync(nested, { recursive: true });
|
||||
writeFileSync(join(nested, "payload"), "read-only payload");
|
||||
chmodSync(join(nested, "payload"), 0o400);
|
||||
rmSync(join(root, "tree"), { recursive: true, force: true });
|
||||
console.log(existsSync(join(root, "tree")));
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
@@ -0,0 +1,26 @@
|
||||
import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const directory = mkdtempSync(join(tmpdir(), "scriptc-write-union-"));
|
||||
const path = join(directory, "data");
|
||||
function write(value: string | Uint8Array): void { writeFileSync(path, value); }
|
||||
function append(value: string | Uint8Array): void { appendFileSync(path, value); }
|
||||
let order = "";
|
||||
function destination(): string { order += "path "; return path; }
|
||||
function data(bytes: boolean): string | Uint8Array {
|
||||
order += "data ";
|
||||
return bytes ? Buffer.from([0, 128, 255]) : "é";
|
||||
}
|
||||
try {
|
||||
for (const value of ["text", new Uint8Array([0, 128, 255]), Buffer.from("é"), "", new Uint8Array(0)]) {
|
||||
write(value);
|
||||
append("!");
|
||||
append(Buffer.from([0, 255]));
|
||||
console.log(readFileSync(path).toString("hex"));
|
||||
}
|
||||
writeFileSync(destination(), data(true));
|
||||
appendFileSync(destination(), data(false));
|
||||
console.log(order);
|
||||
console.log(readFileSync(path).toString("hex"));
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
@@ -0,0 +1,9 @@
|
||||
const records: { first: number; second: number }[] = [{ first: 1, second: 2 }];
|
||||
for (const record of records) console.log(Object.keys(record).join(","), Object.values(record).join(","), JSON.stringify(Object.entries(record)));
|
||||
const dictionaries: Record<string, number>[] = [{ b: 2, a: 1 }];
|
||||
for (const dict of dictionaries) console.log(Object.keys(dict).join(","), Object.values(dict).join(","), JSON.stringify(Object.entries(dict)));
|
||||
const empty: Record<string, number>[] = [];
|
||||
try { console.log(Object.values(empty[0])); }
|
||||
catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); }
|
||||
try { console.log(Object.keys(records[20])); }
|
||||
catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); }
|
||||
@@ -0,0 +1,32 @@
|
||||
interface Entry {
|
||||
name: string;
|
||||
synthesized?: true;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
function inspect(value: unknown): void {
|
||||
if (value === null || typeof value !== "object" || Array.isArray(value)) return;
|
||||
const entry = value as Record<string, unknown>;
|
||||
console.log(entry.name, "synthesized" in entry, Object.hasOwn(entry, "synthesized"), entry.synthesized === true);
|
||||
console.log(Object.keys(entry).join(","), JSON.stringify(entry));
|
||||
}
|
||||
|
||||
function make(name: string, synthesized: boolean): Entry {
|
||||
const entry: Entry = { name };
|
||||
if (synthesized) entry.synthesized = true;
|
||||
return entry;
|
||||
}
|
||||
|
||||
const entries: Entry[] = [{ name: "declared" }, { name: "inline", synthesized: true }, make("built", false), make("generated", true)];
|
||||
for (const entry of entries) inspect(JSON.parse(JSON.stringify(entry)));
|
||||
const document = { entries };
|
||||
const unknownDocument: unknown = JSON.parse(JSON.stringify(document));
|
||||
if (unknownDocument !== null && typeof unknownDocument === "object") {
|
||||
const dict = unknownDocument as Record<string, unknown>;
|
||||
const items = dict.entries;
|
||||
if (Array.isArray(items)) for (const item of items) inspect(item);
|
||||
}
|
||||
|
||||
// Boxing an explicitly present undefined value must retain its key.
|
||||
inspect({ name: "undefined", synthesized: undefined });
|
||||
inspect({ name: "undefined-count", count: undefined });
|
||||
@@ -0,0 +1,26 @@
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
const options = {
|
||||
output: { type: "string", short: "o" },
|
||||
verbose: { type: "boolean", default: false },
|
||||
keep: { type: "boolean", default: true },
|
||||
include: { type: "string", multiple: true },
|
||||
} as const;
|
||||
|
||||
type Arguments = ReturnType<typeof parseArgs<{ options: typeof options; allowPositionals: true; allowNegative: true }>>;
|
||||
function parse(args: string[]): Arguments {
|
||||
return parseArgs({ args, options, allowPositionals: true, allowNegative: true });
|
||||
}
|
||||
function print(args: string[]): void {
|
||||
const { values, positionals } = parse(args);
|
||||
console.log(values.output ?? "none", values.verbose, values.keep);
|
||||
console.log((values.include ?? []).join(":"), positionals.join(":"));
|
||||
}
|
||||
print(["build", "input.ts", "-o", "program", "--verbose", "--no-keep", "--include=one", "--include", "two"]);
|
||||
print([]);
|
||||
try { parse(["--unknown"]); } catch (error) { console.log(error instanceof Error ? error.message.split("\n")[0] : String(error)); }
|
||||
|
||||
// Similar user declarations retain their own structural field types.
|
||||
type ParsedResults = { values: number; positionals: boolean };
|
||||
function local(value: ParsedResults): number { return value.positionals ? value.values : 0; }
|
||||
console.log(local({ values: 42, positionals: true }));
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"compilerOptions": { "strict": true, "types": ["node"] }
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import { F64, typeKey, type IrExpr, type IrModule, type IrType } from "../../../packages/compiler/src/ir/ir.js";
|
||||
import { sanitizeUnregisteredClassTypes } from "../../../packages/compiler/src/frontend/lowering/sanitize-class-types.js";
|
||||
|
||||
const loc = { file: "class-types.ts", start: 0, end: 1 };
|
||||
const missing: IrType = { kind: "object", className: "Fenced" };
|
||||
const kept: IrType = { kind: "object", className: "Registered" };
|
||||
const expr: IrExpr = { kind: "varRef", localId: "x", type: missing, loc };
|
||||
const nested: IrType = { kind: "func", params: [{ kind: "array", elem: missing }],
|
||||
ret: { kind: "generator", yieldT: missing, retT: kept, nextT: missing } };
|
||||
const module: IrModule = { irVersion: 13, sourceFile: loc.file, entry: "main",
|
||||
functions: [{ name: "main", params: [{ localId: "x", name: "x", type: missing }],
|
||||
returnType: nested, locals: [], loc, body: [{ kind: "exprStmt", expr, loc }] }],
|
||||
records: [{ id: "r", fields: [{ name: "callback", type: nested }], indexValue: missing }],
|
||||
unions: [{ id: "u", arms: [missing, kept, { kind: "classval", className: "Fenced" }] }],
|
||||
};
|
||||
sanitizeUnregisteredClassTypes(module, (name) => name === "Registered");
|
||||
console.log(module.functions[0]!.params[0]!.type.kind, expr.type.kind);
|
||||
console.log(typeKey(nested));
|
||||
for (const record of module.records ?? []) {
|
||||
for (const field of record.fields) console.log(typeKey(field.type));
|
||||
if (record.indexValue) console.log(typeKey(record.indexValue));
|
||||
}
|
||||
for (const union of module.unions ?? []) console.log(union.arms.map(typeKey).join(","));
|
||||
console.log(missing.kind, kept.kind, F64.kind);
|
||||
sanitizeUnregisteredClassTypes(module, (name) => name === "Registered");
|
||||
console.log(expr.type.kind);
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import { readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { NativeCache, contentDigest } from "../../../packages/compiler/src/native/cache.js";
|
||||
import { NativeExecutableCache } from "../../../packages/compiler/src/native/executable-cache.js";
|
||||
import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "../../../packages/compiler/src/backend/toolchain-environment.js";
|
||||
|
||||
const directory = process.argv[2]!;
|
||||
const linker = process.argv[3]!;
|
||||
const script = process.argv[4]!;
|
||||
const library = process.argv[5]!;
|
||||
const output = join(directory, "program");
|
||||
const cache = new NativeCache(join(directory, "cache"));
|
||||
const key = contentDigest("native executable");
|
||||
const entry = new NativeExecutableCache(cache, key, linker, false, [library]);
|
||||
console.log(toolchainEnvironmentFingerprint({ LANG: "en_US.UTF-8" }));
|
||||
console.log(JSON.stringify(toolchainEnvironmentCachePolicy({ LIBRARY_PATH: "/fixture" })));
|
||||
console.log(entry.trace([script, library], directory));
|
||||
writeFileSync(output, "cached program");
|
||||
entry.publish(output);
|
||||
rmSync(output);
|
||||
console.log(entry.restore(output));
|
||||
console.log(readFileSync(output, "utf8"));
|
||||
writeFileSync(library, "changed library");
|
||||
console.log(entry.restore(output));
|
||||
@@ -8,7 +8,7 @@ import { compileLibrary } from "@scriptc/compiler";
|
||||
const fixture = join(import.meta.dirname, "../library-mode/wasm/dispatch.mjs");
|
||||
const modes = ["native", "sanitized", "wasm-dev"] as const;
|
||||
|
||||
test.each(modes)("synchronous checked dispatch in a %s library matches Node", async (mode, context) => {
|
||||
test.for(modes)("synchronous checked dispatch in a %s library matches Node", async (mode, context) => {
|
||||
if (mode === "wasm-dev" ? spawnSync("zig", ["version"]).status !== 0 : process.platform !== "darwin" && process.platform !== "linux") context.skip();
|
||||
const directory = await mkdtemp("/tmp/scriptc-library-dispatch-");
|
||||
const oldTarget = process.env["SCRIPTC_TARGET"];
|
||||
|
||||
@@ -2,17 +2,56 @@ import { execFile, execFileSync } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand } from "../../scripts/sandbox-command.mjs";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, waitForSandboxCommand } from "../../scripts/sandbox-command.mjs";
|
||||
|
||||
const roots: string[] = [];
|
||||
const statuses: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
vi.useRealTimers();
|
||||
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
||||
await Promise.all(statuses.splice(0).map((path) => rm(path, { force: true })));
|
||||
});
|
||||
|
||||
test("status recovery retries lost probes and pending commands until a confirmed success", async () => {
|
||||
vi.useFakeTimers();
|
||||
const probe = vi.fn()
|
||||
.mockRejectedValueOnce(new Error("transport disconnected"))
|
||||
.mockRejectedValueOnce(Object.assign(new Error("pending"), { remoteExitCode: REMOTE_COMMAND_PENDING }))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const onPending = vi.fn();
|
||||
const recovery = waitForSandboxCommand(probe, { deadline: Date.now() + 5000, label: "tests", onPending });
|
||||
await vi.runAllTimersAsync();
|
||||
await expect(recovery).resolves.toBeUndefined();
|
||||
expect(probe.mock.calls).toEqual([[5000], [4000], [3000]]);
|
||||
expect(onPending).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("status recovery preserves a confirmed remote failure after a transport failure", async () => {
|
||||
vi.useFakeTimers();
|
||||
const failure = Object.assign(new Error("tests failed"), { remoteExitCode: 7 });
|
||||
const probe = vi.fn().mockRejectedValueOnce(new Error("disconnected")).mockRejectedValue(failure);
|
||||
const recovery = expect(waitForSandboxCommand(probe, {
|
||||
deadline: Date.now() + 5000, label: "tests", onPending: vi.fn(),
|
||||
})).rejects.toBe(failure);
|
||||
await vi.runAllTimersAsync();
|
||||
await recovery;
|
||||
expect(probe).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test("unconfirmed status probes cannot extend the original command deadline", async () => {
|
||||
vi.useFakeTimers();
|
||||
const disconnected = new Error("status timeout");
|
||||
const probe = vi.fn().mockRejectedValue(disconnected);
|
||||
const recovery = expect(waitForSandboxCommand(probe, {
|
||||
deadline: Date.now() + 1500, label: "tests", onPending: vi.fn(),
|
||||
})).rejects.toMatchObject({ message: "tests did not confirm completion before its timeout", cause: disconnected });
|
||||
await vi.runAllTimersAsync();
|
||||
await recovery;
|
||||
expect(probe.mock.calls).toEqual([[1500], [500]]);
|
||||
});
|
||||
|
||||
test("short commands stay inline and retain the remote exit contract", async () => {
|
||||
const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_SHORT__`;
|
||||
const prepared = sandboxCommand("sh", ["-c", "printf 'failure detail\\n' >&2; exit 7"], marker);
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, test } from "vitest";
|
||||
import { compileInChild } from "./self-hosting-compiler-process.js";
|
||||
|
||||
test("native class type cleanup updates shared IR without dynamic snapshots", async () => {
|
||||
const root = join(import.meta.dirname, "../..");
|
||||
const entry = join(root, "tests/fixtures/self-hosting/class-types.ts");
|
||||
const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-class-types-"));
|
||||
try {
|
||||
const built = await compileInChild(entry, { outDir: directory,
|
||||
outPath: join(directory, "class-types" + (process.platform === "win32" ? ".exe" : "")),
|
||||
optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" });
|
||||
if (!built.ok) throw new Error(JSON.stringify(built.diagnostics));
|
||||
const options = { cwd: root, encoding: "utf8" as const, timeout: 30_000 };
|
||||
const oracle = spawnSync(process.execPath, ["--import", "tsx", entry], options);
|
||||
const native = spawnSync(built.binaryPath, [], options);
|
||||
for (const result of [oracle, native]) {
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.signal, result.stderr).toBeNull();
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stderr).toBe("");
|
||||
}
|
||||
expect(native.stdout).toBe(oracle.stdout);
|
||||
expect(native.stdout).toMatch(/^f64 f64\n/);
|
||||
expect(native.stdout).toMatch(/object object f64\nf64\n$/);
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
}, 300_000);
|
||||
@@ -0,0 +1,41 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, test } from "vitest";
|
||||
import { compileInChild } from "./self-hosting-compiler-process.js";
|
||||
|
||||
const root = join(import.meta.dirname, "../..");
|
||||
const entry = join(root, "tests/fixtures/self-hosting/native-cache.ts");
|
||||
|
||||
test("native executable cache discovery, restoration and invalidation match Node", async () => {
|
||||
const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-cache-"));
|
||||
try {
|
||||
const built = await compileInChild(entry, { outDir: directory,
|
||||
outPath: join(directory, "cache" + (process.platform === "win32" ? ".exe" : "")),
|
||||
optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" });
|
||||
if (!built.ok) throw new Error(JSON.stringify(built.diagnostics));
|
||||
const linker = join(directory, "linker.mjs");
|
||||
writeFileSync(linker, 'console.error(JSON.stringify(process.argv[2]));\n');
|
||||
const sdk = join(directory, "sdk");
|
||||
mkdirSync(sdk);
|
||||
const library = join(sdk, "system library.tbd");
|
||||
const results: string[] = [];
|
||||
for (const native of [false, true]) {
|
||||
const stage = join(directory, native ? "native" : "node");
|
||||
mkdirSync(stage);
|
||||
writeFileSync(library, "system library");
|
||||
const args = [stage, process.execPath, linker, library];
|
||||
const result = spawnSync(native ? built.binaryPath : process.execPath, native ? args : ["--import", "tsx", entry, ...args], {
|
||||
cwd: root, encoding: "utf8", timeout: 30_000, env: { ...process.env, ...(native ? { PATH: "" } : {}) },
|
||||
});
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.signal, result.stderr).toBeNull();
|
||||
expect(result.status, result.stderr + result.stdout).toBe(0);
|
||||
expect(result.stderr).toBe("");
|
||||
expect(result.stdout).toContain("true\ntrue\ncached program\nfalse\n");
|
||||
results.push(result.stdout);
|
||||
}
|
||||
expect(results[1]).toBe(results[0]);
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
}, 300_000);
|
||||
@@ -1,8 +1,9 @@
|
||||
import { execFile, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { WASI } from "node:wasi";
|
||||
import { expect, test } from "vitest";
|
||||
import type { NativeToolchainManifest } from "../../packages/compiler/src/native/toolchain.js";
|
||||
import { bootstrapStep } from "./self-hosting-timing.js";
|
||||
@@ -29,14 +30,18 @@ function absoluteCommand(command: string): string {
|
||||
throw new Error(`native tool is not on PATH: ${command}`);
|
||||
}
|
||||
|
||||
test("the standalone compiler builds programs and rebuilds itself with Node unavailable", async () => {
|
||||
test("the production CLI relocates, builds programs, and rebuilds itself with Node unavailable", async () => {
|
||||
const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-bootstrap-"));
|
||||
const executable = (name: string) => join(directory, name + (process.platform === "win32" ? ".exe" : ""));
|
||||
const options = { cwd: root, timeout: 1_800_000, maxBuffer: 16 * 1024 * 1024 };
|
||||
try {
|
||||
await bootstrapStep("build native compiler seed", () =>
|
||||
exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-compiler.mts"), directory], options));
|
||||
const seed = executable("scriptc-native");
|
||||
const distribution = join(directory, "distribution");
|
||||
await bootstrapStep("build production CLI seed", () =>
|
||||
exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-cli.mts"), distribution], options));
|
||||
// All compiler assets must survive moving the complete distribution.
|
||||
const relocated = join(directory, "relocated");
|
||||
renameSync(distribution, relocated);
|
||||
const seed = join(relocated, "bin", "scriptc" + (process.platform === "win32" ? ".exe" : ""));
|
||||
const manifestPath = seed + ".json";
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as NativeToolchainManifest;
|
||||
manifest.linker = absoluteCommand(manifest.linker);
|
||||
@@ -47,28 +52,27 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav
|
||||
manifest.linker_args.push("--ld-path=" + absoluteCommand(linker.stdout.trim()));
|
||||
}
|
||||
if (process.platform === "darwin") manifest.dsymutil = absoluteCommand(manifest.dsymutil);
|
||||
if (process.platform === "darwin") manifest.relocatable_linker = absoluteCommand("ld");
|
||||
manifest.archiver = absoluteCommand(process.platform === "win32" ? "zig" : "ar");
|
||||
manifest.archiver_args = process.platform === "win32" ? ["ar"] : [];
|
||||
writeFileSync(manifestPath, JSON.stringify(manifest));
|
||||
const nativeOptions = { ...options, env: { ...process.env, PATH: "" } };
|
||||
const nativeOptions = { ...options, env: { ...process.env, PATH: "", SCRIPTC_TOOLCHAIN: manifestPath,
|
||||
SCRIPTC_CACHE_DIR: join(directory, "cache") } };
|
||||
const invoke = async (compiler: string, args: string[]) => {
|
||||
const result = await exec(compiler, [...args, "--toolchain", manifestPath], nativeOptions).catch((error: unknown) => {
|
||||
const result = await exec(compiler, args, nativeOptions).catch((error: unknown) => {
|
||||
const failure = error as Error & { code?: string | number; signal?: string; stdout?: string; stderr?: string };
|
||||
throw new Error(`${failure.message}\ncode=${failure.code} signal=${failure.signal}\n${failure.stderr ?? ""}\n${failure.stdout ?? ""}`, { cause: error });
|
||||
});
|
||||
expect(comparableStderr(result.stderr)).toBe("");
|
||||
const built = JSON.parse(result.stdout) as { outputPath: string; llvmPath?: string; stats: {
|
||||
statementsTotal: number; statementsFailed: number; statementsIsland: number; functionsSkipped: number;
|
||||
} };
|
||||
expect(built.stats.statementsFailed).toBe(0);
|
||||
expect(built.stats.statementsIsland).toBe(0);
|
||||
expect(built.stats.functionsSkipped).toBe(0);
|
||||
return built;
|
||||
return result.stdout;
|
||||
};
|
||||
expect((await exec(seed, ["--help"], nativeOptions)).stdout).toContain("Usage: scriptc-native");
|
||||
const checkProgram = async (compiler: string, source: string, backend: string) => {
|
||||
expect(await invoke(seed, ["--help"])).toContain("scriptc build");
|
||||
expect((await invoke(seed, ["--version"])).trim()).toBe(manifest.compiler_version);
|
||||
const checkProgram = async (compiler: string, source: string, extra: string[] = []) => {
|
||||
// This basename formerly collided with the driver's temporary object.
|
||||
const output = executable("program.o");
|
||||
const built = await invoke(compiler, ["build", source, "-o", output, "--backend", backend, "--dev", "--strip"]);
|
||||
expect(built.outputPath).toBe(output);
|
||||
const built = await invoke(compiler, ["build", source, "-o", output, "--optimization=dev", "--strip", ...extra]);
|
||||
expect(built.trim()).toBe(output);
|
||||
const oracle = spawnSync(process.execPath, [source], options);
|
||||
const actual = spawnSync(output, [], nativeOptions);
|
||||
for (const result of [oracle, actual]) {
|
||||
@@ -82,41 +86,111 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav
|
||||
const sample = join(root, "tests/corpus/class-array-optional-return.ts");
|
||||
const unionSample = join(root, "tests/corpus/union-nested-layout-discriminant.ts");
|
||||
const receiverSample = join(root, "tests/corpus/llvm-read-receiver-lifetime.ts");
|
||||
await checkProgram(seed, sample, "llvm");
|
||||
await checkProgram(seed, unionSample, "llvm");
|
||||
await checkProgram(seed, receiverSample, "llvm");
|
||||
await checkProgram(seed, sample);
|
||||
await checkProgram(seed, unionSample);
|
||||
await checkProgram(seed, receiverSample);
|
||||
await checkProgram(seed, join(root, "tests/corpus/closure-nullable-union-return.ts"));
|
||||
await checkProgram(seed, join(root, "tests/corpus/record-optional-json-presence.ts"));
|
||||
await checkProgram(seed, join(root, "tests/corpus/1010-json-stringify-space.ts"));
|
||||
await checkProgram(seed, join(root, "tests/corpus/fs-write-string-bytes-union.ts"));
|
||||
|
||||
const fetchOptions = join(directory, "fetch-options.ts");
|
||||
writeFileSync(fetchOptions, 'async function probe() { const response = await fetch("https://example.invalid", { headers: { accept: "application/json" } }); console.log(response.status); } if (process.env["RUN_NATIVE_FETCH_PROBE"] === "1") await probe();\n');
|
||||
await checkProgram(seed, fetchOptions);
|
||||
expect(await invoke(seed, ["coverage", fetchOptions])).toContain("(100%)");
|
||||
|
||||
const dynamic = join(directory, "dynamic.ts");
|
||||
writeFileSync(dynamic, 'const value: any = { answer: 42 }; console.log(`answer:${value.answer}`);\n');
|
||||
await checkProgram(seed, dynamic, ["--dynamic"]);
|
||||
const comptime = join(directory, "comptime.ts");
|
||||
writeFileSync(comptime, 'const answer = comptime(() => [1, 2, 3].reduce((sum, value) => sum + value, 0) * 7); console.log(answer);\n');
|
||||
expect(await invoke(seed, ["run", comptime, "-o", executable("comptime"), "--strip"])).toBe("42\n");
|
||||
|
||||
const object = join(directory, "program.obj");
|
||||
const linkInfo = JSON.parse(await invoke(seed, ["build", sample, "-o", object, "--print=native-link-info"]));
|
||||
expect(linkInfo.program.object).toBe(object);
|
||||
expect(readFileSync(object).length).toBeGreaterThan(0);
|
||||
|
||||
const profile = join(root, "tests/library-mode/contract-attest/profile.json");
|
||||
const archive = join(directory, "contract.a");
|
||||
const expectedArchive = join(directory, "contract-node.a");
|
||||
await exec(process.execPath, [join(root, "packages/cli/dist/main.js"), "build", "--lib", "--profile", profile, "-o", expectedArchive], options);
|
||||
await invoke(seed, ["build", "--lib", "--profile", profile, "-o", archive]);
|
||||
expect(JSON.parse(readFileSync(archive + ".contract.json", "utf8")))
|
||||
.toEqual(JSON.parse(readFileSync(expectedArchive + ".contract.json", "utf8")));
|
||||
|
||||
// The unmodified package exercises deep validator traversals. Compile it
|
||||
// through the installed CLI, including its async command boundary.
|
||||
const threeProfile = join(root, "tests/library-mode/wasm/three.json");
|
||||
await invoke(seed, ["build", "--lib", "--profile", threeProfile, "-o", join(directory, "three.a")]);
|
||||
if (process.platform !== "win32" && spawnSync("zig", ["version"]).status === 0 && existsSync(join(root, "packages/runtime-wasm32-wasi/runtime-pack.json"))) {
|
||||
const linker = join(directory, "zigcc");
|
||||
writeFileSync(linker, `#!/bin/sh\nexec '${absoluteCommand("zig").replaceAll("'", "'\\''")}' cc "$@"\n`, { mode: 0o755 });
|
||||
const output = join(directory, "three.wasm");
|
||||
const built = await exec(seed, ["build", "--lib", "--profile", threeProfile, "-o", output], {
|
||||
...nativeOptions, env: { ...nativeOptions.env, SCRIPTC_TARGET: "wasm32-wasi", SCRIPTC_LINKER: linker,
|
||||
SCRIPTC_RUNTIME_PACK: join(root, "packages/runtime-wasm32-wasi"), SCRIPTC_NO_CACHE: "1" },
|
||||
});
|
||||
expect(built.stdout.trim()).toBe(output);
|
||||
expect(comparableStderr(built.stderr)).toBe("");
|
||||
const vertices: number[][] = [];
|
||||
const wasi = new WASI({ version: "preview1" });
|
||||
const instance = new WebAssembly.Instance(new WebAssembly.Module(readFileSync(output)), {
|
||||
...wasi.getImportObject(), scriptc: {
|
||||
vertex: (...args: number[]) => vertices.push(args.map((value) => Number(value.toFixed(9)))),
|
||||
panic: () => { throw new Error("unexpected Wasm panic"); },
|
||||
},
|
||||
});
|
||||
wasi.initialize(instance);
|
||||
const api = instance.exports as Record<string, (...args: number[]) => number>;
|
||||
api.app_init!();
|
||||
for (const time of [0, 123, 2000]) expect(api.app_frame!(time, 1.5)).toBe(24);
|
||||
const oracle = await exec(process.execPath, ["--input-type=module", "--eval", `
|
||||
const vertices = [];
|
||||
globalThis.vertex = (...args) => vertices.push(args.map(value => Number(value.toFixed(9))));
|
||||
const { frame } = await import(${JSON.stringify(join(root, "tests/library-mode/wasm/three.mjs"))});
|
||||
for (const time of [0, 123, 2000]) frame(time, 1.5);
|
||||
console.log(JSON.stringify(vertices));
|
||||
`], options);
|
||||
expect(vertices).toEqual(JSON.parse(oracle.stdout));
|
||||
expect(oracle.stderr).toBe("");
|
||||
api.app_collect!();
|
||||
}
|
||||
|
||||
const badSource = join(directory, "bad.ts");
|
||||
writeFileSync(badSource, 'const value: number = "wrong"; console.log(value);\n');
|
||||
const retained = executable("retained");
|
||||
writeFileSync(retained, "existing output");
|
||||
const failed = spawnSync(seed, [badSource, "-o", retained, "--toolchain", manifestPath], nativeOptions);
|
||||
const failed = spawnSync(seed, ["build", badSource, "-o", retained], nativeOptions);
|
||||
expect(failed.status).toBe(1);
|
||||
expect(failed.stderr.toString()).toContain("not assignable");
|
||||
expect(readFileSync(retained, "utf8")).toBe("existing output");
|
||||
|
||||
const entry = join(root, "packages/compiler/src/native/main.ts");
|
||||
const profile = join(directory, "ts7-process.ffi.json");
|
||||
const entry = join(root, "packages/compiler/src/native/cli.ts");
|
||||
const ffi = join(directory, ".scriptc/distribution-seed/compiler.ffi.json");
|
||||
const rebuilt = executable("scriptc-rebuilt");
|
||||
// Optimize the compiler that will process the full graph again. Small
|
||||
// programs above and below still exercise development output.
|
||||
const self = await bootstrapStep("native compiler rebuilds itself", () =>
|
||||
invoke(seed, [entry, "-o", rebuilt, "--backend=llvm", "--strip", "--keep-llvm", "--ffi", profile]));
|
||||
expect(self.stats.statementsTotal).toBeGreaterThan(10_000);
|
||||
expect(self.llvmPath).toBe(rebuilt + ".ll");
|
||||
console.log("native self-build", self.stats);
|
||||
await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts"), "llvm");
|
||||
await checkProgram(rebuilt, sample, "llvm");
|
||||
await checkProgram(rebuilt, unionSample, "llvm");
|
||||
await checkProgram(rebuilt, receiverSample, "llvm");
|
||||
const self = await bootstrapStep("production CLI rebuilds itself", () =>
|
||||
invoke(seed, ["build", entry, "-o", rebuilt, "--strip", "--keep-llvm", "--ffi", ffi]));
|
||||
expect(self.trim()).toBe(rebuilt);
|
||||
await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts"));
|
||||
await checkProgram(rebuilt, sample);
|
||||
await checkProgram(rebuilt, unionSample);
|
||||
await checkProgram(rebuilt, receiverSample);
|
||||
await checkProgram(rebuilt, join(root, "tests/corpus/1010-json-stringify-space.ts"));
|
||||
|
||||
// Compare the LLVM used to build the second generation with its own
|
||||
// output. Retaining the build input avoids repeating the seed's work.
|
||||
if (!self.llvmPath) throw new Error("native self-build did not retain its LLVM");
|
||||
const seedLlvm = self.llvmPath;
|
||||
const seedLlvm = join(directory, "cli.ll");
|
||||
const rebuiltLlvm = join(directory, "rebuilt.ll");
|
||||
await bootstrapStep("rebuilt compiler emits itself", () =>
|
||||
invoke(rebuilt, [entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", profile]));
|
||||
expect(readFileSync(seedLlvm).equals(readFileSync(rebuiltLlvm)), "native compiler generations must emit identical LLVM").toBe(true);
|
||||
invoke(rebuilt, ["build", entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", ffi]));
|
||||
// Executable output adds a runtime ABI check; textual LLVM emission
|
||||
// omits it. Compare all other output without changing either mode.
|
||||
const withoutAbiCheck = (path: string): string => readFileSync(path, "utf8")
|
||||
.replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "")
|
||||
.replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, "");
|
||||
expect(withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), "native compiler generations must emit identical LLVM").toBe(true);
|
||||
} finally { rmSync(directory, { recursive: true, force: true }); }
|
||||
}, 5_400_000);
|
||||
|
||||
@@ -63,8 +63,8 @@ for (const backend of ["llvm"] as const) {
|
||||
const toolchain = join(directory, "toolchain.json");
|
||||
const config: NativeToolchainManifest = {
|
||||
schema: "scriptc.native-toolchain.v1", compiler_version: manifest.version, target: target.name,
|
||||
ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack", runtime_headers: "headers",
|
||||
c_compiler: "cc", c_compiler_args: [], linker: "ld", linker_args: [], dsymutil: "dsymutil",
|
||||
ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack",
|
||||
linker: "ld", linker_args: [], dsymutil: "dsymutil",
|
||||
};
|
||||
writeFileSync(toolchain, JSON.stringify(config));
|
||||
const input = join(directory, "request.json");
|
||||
|
||||
@@ -114,6 +114,7 @@ for (const backend of ["llvm"] as const) {
|
||||
"tests/corpus/3089-array-find-narrowing.ts",
|
||||
"tests/corpus/3091-json-recursive-discriminants.ts",
|
||||
"tests/corpus/3094-json-replacer-traversal.ts",
|
||||
"tests/corpus/1010-json-stringify-space.ts",
|
||||
]) {
|
||||
const path = join(dir, "emitted.json");
|
||||
const emitted = await compile(join(root, source), { outDir: dir, outPath: path, outputKind: "ir", dynamic: false });
|
||||
|
||||
@@ -2,21 +2,49 @@ import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { WASI } from "node:wasi";
|
||||
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
||||
import { compileLibrary } from "@scriptc/compiler";
|
||||
import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
|
||||
import { compileLibrary as compileNodeLibrary, type CompileLibraryOptions } from "@scriptc/compiler";
|
||||
import { NativeCompiler } from "../../packages/compiler/src/native/compiler.js";
|
||||
import { nativeCodegenTarget, WASM32_WASI_TARGET } from "../../packages/compiler/src/backend/targets.js";
|
||||
import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js";
|
||||
|
||||
// Run both compiler drivers against the same real checker transport in this
|
||||
// harness. The production bootstrap separately exercises the native transport.
|
||||
vi.mock("../../packages/compiler/src/frontend/pipeline-native.js", async () => {
|
||||
const { runFrontend } = await import("../../packages/compiler/src/frontend/pipeline.js");
|
||||
const { loadProgram } = await import("../../packages/compiler/src/frontend/program-node.js");
|
||||
return { runNativeFrontend: (entry: string, _executable: string, npmStatic?: readonly string[] | "auto" | "lib",
|
||||
externalTypes?: Readonly<Record<string, string>>, _evaluate?: unknown, libraryNpmStatic?: readonly string[]) =>
|
||||
runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic) };
|
||||
});
|
||||
|
||||
const fixture = join(import.meta.dirname, "../library-mode/wasm");
|
||||
const hasZig = spawnSync("zig", ["version"]).status === 0;
|
||||
type Api = Record<string, (...args: number[]) => number> & { memory: WebAssembly.Memory };
|
||||
|
||||
describe.skipIf(!hasZig)("Wasm library embedding", () => {
|
||||
describe.skipIf(!hasZig).each(["node", "native"] as const)("Wasm library embedding (%s driver)", (driver) => {
|
||||
let directory: string;
|
||||
let module: WebAssembly.Module;
|
||||
let oldTarget: string | undefined;
|
||||
let profile: any;
|
||||
let native: NativeCompiler;
|
||||
const compileLibrary = (options: CompileLibraryOptions) => driver === "node"
|
||||
? compileNodeLibrary(options) : Promise.resolve(native.compileLibrary(options));
|
||||
|
||||
beforeAll(async () => {
|
||||
oldTarget = process.env["SCRIPTC_TARGET"];
|
||||
if (driver === "native") {
|
||||
const host = nativeCodegenTarget()!;
|
||||
const root = join(import.meta.dirname, "../..");
|
||||
const helperPackageRoot = join(root, "packages", host.helper.packageName.replace("@scriptc/", ""));
|
||||
const compilerPackage = JSON.parse(await readFile(join(root, "packages/compiler/package.json"), "utf8"));
|
||||
native = new NativeCompiler({
|
||||
compilerVersion: compilerPackage.version, target: WASM32_WASI_TARGET, helper: host.helper,
|
||||
helperPackageRoot, helperExecutable: join(helperPackageRoot, "bin", process.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"),
|
||||
runtimePackRoot: join(root, "packages/runtime-wasm32-wasi"), ts7Executable: ts7Executable(),
|
||||
linker: "zig", linkerArgs: ["cc"], dsymutil: "unused",
|
||||
});
|
||||
}
|
||||
process.env["SCRIPTC_TARGET"] = "wasm32-wasi";
|
||||
directory = await mkdtemp("/tmp/scriptc-wasm-library-test-");
|
||||
profile = JSON.parse(await readFile(join(fixture, "profile.json"), "utf8"));
|
||||
|
||||
Reference in New Issue
Block a user