Merge remote-tracking branch 'origin/ni/i4-deferred' into HEAD

# Conflicts:
#	packages/compiler/src/frontend/lowering/lower-modules.ts

scripts/bench-runtime.mjs

@@ -42,6 +42,8 @@ const { values } = parseArgs({
     "candidate-optimization": { type: "string", default: "release" },
     "baseline-target-cpu": { type: "string" },
     "candidate-target-cpu": { type: "string" },
+    "baseline-cpu-dispatch": { type: "boolean", default: false },
+    "candidate-cpu-dispatch": { type: "boolean", default: false },
     json: { type: "string" },
     "no-lock": { type: "boolean", default: false },
     node: { type: "boolean", default: true },
@@ -54,6 +56,7 @@ if (values.help) {
        [--workloads=a,b] [--runs=15] [--warmup=2] [--timeout=120] [--layouts=1] [--json=<file>]
        [--baseline-optimization=release] [--candidate-optimization=release]
        [--baseline-target-cpu=<level>] [--candidate-target-cpu=<level>]
+       [--baseline-cpu-dispatch] [--candidate-cpu-dispatch]
        [--no-lock] [--no-node] [--keep]
 
 Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native
@@ -74,8 +77,9 @@ Workload names are the file names in benchmarks/runtime/workloads/*.json; for ex
 contenders, e.g. --baseline=. --candidate-optimization=speed compares speed with
 release. --baseline-target-cpu/--candidate-target-cpu pass --target-cpu to each
 contender's builds, e.g. --baseline=. --candidate-target-cpu=x86-64-v3 compares
-x86-64-v3 code with the baseline. cold_build_ms records each contender's first,
-uncached build.`);
+x86-64-v3 code with the baseline. --baseline-cpu-dispatch/--candidate-cpu-dispatch
+pass --cpu-dispatch, and SCRIPTC_CPU_LEVEL in the environment reaches every timed
+run. cold_build_ms records each contender's first, uncached build.`);
   process.exit(0);
 }
 const runs = Number(values.runs);
@@ -444,7 +448,7 @@ function revision(root) {
   });
   return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : "");
 }
-function build(label, root, workload, layout, optimization, targetCpu) {
+function build(label, root, workload, layout, optimization, targetCpu, cpuDispatch) {
   const cli = cliFor(root);
   const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`);
   mkdirSync(dirname(out), { recursive: true });
@@ -466,6 +470,7 @@ function build(label, root, workload, layout, optimization, targetCpu) {
       entry,
       `--optimization=${optimization}`,
       ...(targetCpu === undefined ? [] : [`--target-cpu=${targetCpu}`]),
+      ...(cpuDispatch ? ["--cpu-dispatch"] : []),
       "-o",
       out,
     ],
@@ -492,6 +497,7 @@ const contenders = [
           root: resolve(values.baseline),
           optimization: values["baseline-optimization"],
           targetCpu: values["baseline-target-cpu"],
+          cpuDispatch: values["baseline-cpu-dispatch"],
         },
       ]
     : []),
@@ -500,6 +506,7 @@ const contenders = [
     root: resolve(values.candidate),
     optimization: values["candidate-optimization"],
     targetCpu: values["candidate-target-cpu"],
+    cpuDispatch: values["candidate-cpu-dispatch"],
   },
 ];
 const report = {
@@ -606,7 +613,7 @@ try {
       entry.results[c.label] = result;
       result.binaries = [];
       for (let layout = 0; layout < layouts && result.status === undefined; layout++) {
-        const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu);
+        const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu, c.cpuDispatch);
         result.build_ms += Math.round(built.buildMs);
         if (layout === 0) result.cold_build_ms = Math.round(built.buildMs);
         if (!built.ok) {

scripts/bootstrap-seed.mjs

@@ -25,6 +25,19 @@ export function verifyBootstrapSeed(directory, revision, sanitize) {
   }
 }
 
+/** The environment a bootstrap seed builds with, over the inherited one. The
+ * bootstrap compares the seed's LLVM with the native compiler's own emission,
+ * which has no function layout. The layout only orders the functions, so the
+ * seed is built without it. */
+export function seedBuildEnvironment(sanitize) {
+  return {
+    SCRIPTC_SAN: sanitize ? "1" : "",
+    SCRIPTC_NATIVE_EMIT_IR: "1",
+    SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release",
+    SCRIPTC_NATIVE_LAYOUT: "",
+  };
+}
+
 async function main() {
   const directory = process.env.SCRIPTC_BOOTSTRAP_SEED_DIRECTORY;
   const revision = process.env.GITHUB_SHA;
@@ -47,11 +60,7 @@ async function main() {
     {
       cwd: root,
       stdio: "inherit",
-      env: {
-        ...process.env,
-        SCRIPTC_NATIVE_EMIT_IR: "1",
-        SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release",
-      },
+      env: { ...process.env, ...seedBuildEnvironment(sanitize) },
     },
   );
   writeFileSync(

scripts/build-llvm-dist.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+/* Builds the LLVM distribution the Linux release helpers link
+ * (native/llvm-codegen/llvm-dist.mjs): LLVM_VERSION from its release source
+ * tarball, with ThinLTO and native/llvm-codegen/llvm.profdata.gz, the
+ * helper's targets, and static libraries only. clang and lld of the same
+ * version must be on PATH (LLVM's release archive for the host has both).
+ * Writes the install tree and, with --archive, a .tar.xz of it and its
+ * .sha256. The llvm-dist workflow runs it for each Linux architecture; it
+ * takes about 8 minutes on 32 cores and about an hour on 4.
+ *
+ * With --instrumented it builds the IR-instrumented libraries that
+ * scripts/refresh-llvm-profile.mjs collects the profile with instead.
+ *
+ *   node scripts/build-llvm-dist.mjs --source <llvm-project-X.src> --prefix <dir>
+ *     [--archive <file.tar.xz>] [--work <dir>] [--jobs N] [--instrumented]
+ */
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { cpus, tmpdir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { llvmCmakeArgs, writeLlvmProfile } from "../native/llvm-codegen/llvm-dist.mjs";
+
+const { values } = parseArgs({
+  options: {
+    source: { type: "string" },
+    prefix: { type: "string" },
+    archive: { type: "string" },
+    work: { type: "string" },
+    jobs: { type: "string", default: String(cpus().length) },
+    instrumented: { type: "boolean", default: false },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help || values.source === undefined || values.prefix === undefined) {
+  console.log(
+    "Usage: node scripts/build-llvm-dist.mjs --source <llvm-project source> --prefix <install dir> [--archive <file.tar.xz>] [--work <dir>] [--jobs N] [--instrumented]",
+  );
+  process.exit(values.help ? 0 : 2);
+}
+if (process.platform !== "linux") throw new Error("the LLVM distribution is built on Linux");
+const source = resolve(values.source);
+if (!existsSync(join(source, "llvm/CMakeLists.txt")))
+  throw new Error(`${source} is not an llvm-project source tree`);
+const prefix = resolve(values.prefix);
+const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-dist-")));
+const build = join(work, values.instrumented ? "llvm-instrumented" : "llvm");
+mkdirSync(build, { recursive: true });
+
+function run(command, args, log, env = process.env) {
+  console.log(`$ ${command} ${args.join(" ")}`);
+  const result = spawnSync(command, args, { encoding: "utf8", maxBuffer: 1 << 30, env });
+  if (log !== undefined) writeFileSync(log, result.stdout + result.stderr);
+  if (result.status !== 0) {
+    process.stderr.write((result.stdout + result.stderr).slice(-8000));
+    throw new Error(`${command} failed with status ${result.status}`);
+  }
+  return result.stdout + result.stderr;
+}
+
+const profile = values.instrumented ? undefined : writeLlvmProfile(work);
+run("cmake", llvmCmakeArgs({ source, build, prefix, profile, instrumented: values.instrumented }));
+const log = join(work, "build.log");
+const output = run("ninja", ["-C", build, "-j", values.jobs, "install"], log);
+
+if (profile !== undefined) {
+  // clang reports, per file, how many of its functions the profile no longer
+  // matches; on a host other than the one it was collected on, that count
+  // shows how much of the profile still applies.
+  let files = 0,
+    functions = 0,
+    mismatched = 0;
+  for (const match of output.matchAll(/of (\d+) functions?, (\d+) ha(?:ve|s) mismatched data/g)) {
+    files++;
+    functions += Number(match[1]);
+    mismatched += Number(match[2]);
+  }
+  console.log(
+    files === 0
+      ? "profile: no file reports functions with mismatched profile data"
+      : `profile: ${mismatched} of ${functions} functions in ${files} files have mismatched profile data (build log ${log})`,
+  );
+}
+
+if (values.archive !== undefined) {
+  const archive = resolve(values.archive);
+  mkdirSync(dirname(archive), { recursive: true });
+  run("tar", ["-C", dirname(prefix), "-cJf", archive, basename(prefix)], undefined, {
+    ...process.env,
+    XZ_OPT: "-T0",
+  });
+  const digest = createHash("sha256").update(readFileSync(archive)).digest("hex");
+  writeFileSync(`${archive}.sha256`, `${digest}  ${basename(archive)}\n`);
+  console.log(`wrote ${archive} (${digest})`);
+}

scripts/build-native-cli.mts

@@ -1,17 +1,26 @@
 /** Build the production CLI seed and its relocatable compiler assets. */
-import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { execFileSync, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { basename, dirname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { gunzipSync } from "node:zlib";
 import { compile } from "../packages/compiler/src/index.js";
-import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js";
+import {
+  NATIVE_TARGETS,
+  executableStripLinkerArgs,
+  nativeCodegenTarget,
+} from "../packages/compiler/src/backend/targets.js";
 import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js";
+import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js";
 import { stageNativeCliAssets } from "./native-cli-assets.mjs";
 import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js";
 
 const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const output = resolve(process.argv[2] ?? join(root, ".scriptc/native-cli"));
+// --platform-packages builds an npm platform package's distribution, which
+// depends on the helper and runtime pack packages instead of bundling them.
+const platformPackages = process.argv.includes("--platform-packages");
 const optimization = process.env["SCRIPTC_NATIVE_OPTIMIZATION"] === "dev" ? "dev" : "release";
 const sanitize = process.env["SCRIPTC_SAN"] === "1";
 // SCRIPTC_NATIVE_PROFILE_GENERATE=<pattern> builds a compiler that writes a
@@ -35,6 +44,15 @@ const useProfile =
   !sanitize &&
   requestedProfile !== "" &&
   (requestedProfile !== undefined || existsSync(committedProfile));
+// Builds lay out the compiler's functions with the committed layout
+// (scripts/refresh-cli-layout.mjs), or the one SCRIPTC_NATIVE_LAYOUT names;
+// an empty SCRIPTC_NATIVE_LAYOUT, and instrumented builds, use none.
+const committedLayout = join(root, "packages/compiler/native/cli.layout.gz");
+const requestedLayout = process.env["SCRIPTC_NATIVE_LAYOUT"];
+const functionLayout =
+  profileGenerate !== undefined || requestedLayout === ""
+    ? undefined
+    : (requestedLayout ?? (existsSync(committedLayout) ? committedLayout : undefined));
 let target = nativeCodegenTarget();
 if (target === null || target.platform === "wasi")
   throw new Error("a supported native host is required to build the compiler");
@@ -92,12 +110,14 @@ const ffi = join(seed, "compiler.ffi.json");
 writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n");
 const executable = join(bin, "scriptc" + target.outputSuffixes.exe);
 const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe);
+const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1";
 execFileSync(
   compiler,
   [
     ...compilerArgs,
     ...cFlags,
     "-O2",
+    ...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []),
     "-I",
     join(root, "packages/runtime/vendor/quickjs-ng"),
     join(nativeSources, "comptime.c"),
@@ -114,16 +134,20 @@ execFileSync(
   ],
   { stdio: "inherit" },
 );
+// The Windows link writes the debug database beside the executable.
+if (strip && target.platform === "win32")
+  rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true });
 const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), {
   outDir: seed,
   outPath: executable,
   backend: "llvm",
   optimization,
-  strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1",
+  strip,
   ffiProfilePath: ffi,
   sanitize,
   ...(profileGenerate === undefined ? {} : { profileGenerate }),
   ...(profileUse === undefined ? {} : { profileUse }),
+  ...(functionLayout === undefined ? {} : { functionLayout }),
   emitIr: process.env["SCRIPTC_NATIVE_EMIT_IR"] === "1",
 });
 if (!result.ok)
@@ -165,6 +189,8 @@ const assets = stageNativeCliAssets({
   target,
   ts7: tsgoExecutable(),
   compilerVersion,
+  strip,
+  platformPackages,
 });
 // A cross-libc seed may use Zig on the build host. The installed compiler
 // selects the destination platform's ordinary linker and SDK.
@@ -181,4 +207,60 @@ const manifest: NativeToolchainManifest = {
   comptime: relative(bin, comptime),
 };
 writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n");
+// The prebuilt lib diagnostics (frontend/ts7/lib-diagnostics.ts): the
+// declaration file sets the distribution's tsc-ts reports no diagnostics in
+// for programs without a tsconfig, recorded by the built compiler itself. A
+// compiler this host cannot run ships without them.
+const prebuiltKeys = recordPrebuiltLibDiagnostics(executable);
+if (prebuiltKeys === null) {
+  console.warn(
+    "warning: the built compiler did not run here; it ships without prebuilt lib diagnostics",
+  );
+} else {
+  const prebuilt = join(lib, "declarations", "lib-diagnostics.json");
+  const checker = (
+    JSON.parse(readFileSync(join(lib, "tsc-ts", "package.json"), "utf8")) as { version: string }
+  ).version;
+  writeFileSync(prebuilt, prebuiltLibDiagnosticsText(checker, prebuiltKeys));
+  manifest.lib_diagnostics = relative(bin, prebuilt);
+  writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n");
+}
 console.log(executable);
+
+/** The content keys the built compiler records building, each in an empty
+ * cache, a module without imports (the program it lowers) and a module with
+ * a type error (which also checks the program without scriptc's overrides);
+ * null when it cannot run. */
+function recordPrebuiltLibDiagnostics(compiler: string): string[] | null {
+  const work = mkdtempSync(join(tmpdir(), "scriptc-lib-diagnostics-"));
+  try {
+    const record = join(work, "keys.txt");
+    const probes = [
+      { name: "clean", source: "export {};\n", status: 0 },
+      { name: "error", source: 'export const n: number = "x";\n', status: 1 },
+    ];
+    for (const probe of probes) {
+      const dir = join(work, probe.name);
+      mkdirSync(dir);
+      writeFileSync(join(dir, "main.ts"), probe.source);
+      const env: NodeJS.ProcessEnv = {
+        ...process.env,
+        SCRIPTC_CACHE_DIR: join(dir, "cache"),
+        SCRIPTC_LIB_DIAGNOSTICS_RECORD: record,
+      };
+      delete env["SCRIPTC_TIMING"];
+      const run = spawnSync(
+        compiler,
+        ["build", join(dir, "main.ts"), "--optimization", "dev", "-o", join(dir, "main")],
+        { cwd: dir, env, encoding: "utf8" },
+      );
+      if (run.status !== probe.status) return null;
+    }
+    if (!existsSync(record)) return null;
+    return readFileSync(record, "utf8")
+      .split("\n")
+      .filter((key) => key !== "");
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+}

scripts/check-llvm-dist.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+/* Compares two LLVM helpers on the helper inputs of this checkout
+ * (scripts/llvm-helper-inputs.mjs): every input must give identical objects
+ * with both, and the script prints each input group's median wall time per
+ * helper over the rounds, with the candidate/baseline ratio. The llvm-dist
+ * workflow runs it with the helper linked against LLVM's release archive as
+ * the baseline and the one linked against the profiled ThinLTO distribution
+ * as the candidate, on each architecture it builds. Exits with status 1 when
+ * an object differs.
+ *
+ *   node scripts/check-llvm-dist.mjs --baseline <helper> --candidate <helper>
+ *     [--rounds N] [--work <dir>] [--tsc-ts <checkout>] [--extra <dir of .ts programs>]
+ */
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { performance } from "node:perf_hooks";
+import { parseArgs } from "node:util";
+import { captureHelperInputs, helperInputPrograms } from "./llvm-helper-inputs.mjs";
+
+const { values } = parseArgs({
+  options: {
+    baseline: { type: "string" },
+    candidate: { type: "string" },
+    rounds: { type: "string", default: "3" },
+    work: { type: "string" },
+    "tsc-ts": { type: "string" },
+    extra: { type: "string", multiple: true, default: [] },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help || values.baseline === undefined || values.candidate === undefined) {
+  console.log(
+    "Usage: node scripts/check-llvm-dist.mjs --baseline <helper> --candidate <helper> [--rounds N] [--work <dir>] [--tsc-ts <checkout>] [--extra <dir>]",
+  );
+  process.exit(values.help ? 0 : 2);
+}
+const helpers = { baseline: resolve(values.baseline), candidate: resolve(values.candidate) };
+const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-check-")));
+const inputs = await captureHelperInputs(
+  helperInputPrograms({
+    tscTs: values["tsc-ts"] === undefined ? undefined : resolve(values["tsc-ts"]),
+    extra: values.extra.map((dir) => resolve(dir)),
+  }),
+  join(work, "inputs"),
+);
+
+function emit(helper, input, out) {
+  rmSync(out, { recursive: true, force: true });
+  mkdirSync(out, { recursive: true });
+  const start = performance.now();
+  const result = spawnSync(helper, input.args(out), { encoding: "utf8" });
+  const ms = performance.now() - start;
+  if (result.status !== 0)
+    throw new Error(`${helper} failed on ${input.name}: ${result.stdout}${result.stderr}`);
+  return ms;
+}
+
+const times = new Map(); // `${group}|${helper}` -> per-round totals
+const group = (name) => (name === "tsc-ts" ? name : name.split("-")[0]);
+const rounds = Number(values.rounds);
+let differing = 0;
+for (let round = 0; round < rounds; round++) {
+  const order = round % 2 === 0 ? ["baseline", "candidate"] : ["candidate", "baseline"];
+  for (const input of inputs) {
+    for (const which of order) {
+      const ms = emit(helpers[which], input, join(work, which));
+      const key = `${group(input.name)}|${which}`;
+      const totals = times.get(key) ?? new Array(rounds).fill(0);
+      totals[round] += ms;
+      times.set(key, totals);
+    }
+    if (round === 0) {
+      for (const file of readdirSync(join(work, "baseline"))) {
+        const a = readFileSync(join(work, "baseline", file));
+        const b = readFileSync(join(work, "candidate", file));
+        if (!a.equals(b)) {
+          differing++;
+          console.log(`${input.name}: ${file} differs between the helpers`);
+        }
+      }
+    }
+  }
+}
+
+const median = (xs) => {
+  const s = [...xs].sort((x, y) => x - y);
+  return s.length % 2 ? s[s.length >> 1] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2;
+};
+console.log(`${inputs.length} inputs, ${rounds} rounds; median wall time per group`);
+console.log("| group | baseline | candidate | candidate/baseline |");
+console.log("| --- | ---: | ---: | ---: |");
+for (const name of [...new Set(inputs.map((input) => group(input.name)))]) {
+  const b = median(times.get(`${name}|baseline`)),
+    c = median(times.get(`${name}|candidate`));
+  console.log(
+    `| ${name} | ${(b / 1000).toFixed(2)} s | ${(c / 1000).toFixed(2)} s | ${(c / b).toFixed(3)} |`,
+  );
+}
+if (differing > 0) {
+  console.log(`${differing} objects differ between the helpers`);
+  process.exit(1);
+}
+console.log("every object is identical between the helpers");

scripts/generate-ir-fields.mjs

@@ -0,0 +1,215 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+// Generates packages/compiler/src/ir/fields.generated.ts from the node types
+// in ir.ts: for each IrExpr and IrStmt kind, the fields that can hold a
+// string, read by name. Code that would enumerate a node's keys reads them
+// through this switch instead: the native compiler enumerates and reads an
+// untyped object's keys through its dynamic representation, one conversion
+// per node. `--check` fails when the file is stale.
+const root = fileURLToPath(new URL("..", import.meta.url));
+const require = createRequire(join(root, "package.json"));
+const ts = require("typescript");
+const irPath = join(root, "packages/compiler/src/ir/ir.ts");
+const target = join(root, "packages/compiler/src/ir/fields.generated.ts");
+
+const program = ts.createProgram([irPath], { strict: true, target: ts.ScriptTarget.ES2022 });
+const checker = program.getTypeChecker();
+const source = program.getSourceFile(irPath);
+const aliases = new Map();
+ts.forEachChild(source, (node) => {
+  if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node);
+});
+
+/** Whether a value of `type` can be a string. */
+function holdsString(type) {
+  if (type.isUnion()) return type.types.some(holdsString);
+  return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0;
+}
+
+/** Whether every value of `type` is a string. */
+function onlyString(type) {
+  if (type.isUnion()) return type.types.every(onlyString);
+  return (type.flags & ts.TypeFlags.StringLike) !== 0;
+}
+
+/** Each kind's string fields, in declaration order, merged over the unions:
+ * a kind both unions declare lists the fields of both. */
+const kinds = new Map();
+for (const name of ["IrExpr", "IrStmt"]) {
+  const alias = aliases.get(name);
+  const union = checker.getTypeAtLocation(alias.name);
+  for (const member of union.isUnion() ? union.types : [union]) {
+    const kindSymbol = member.getProperty("kind");
+    const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias);
+    if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`);
+    const fields = kinds.get(kindType.value) ?? [];
+    for (const property of member.getProperties()) {
+      const type = checker.getTypeOfSymbolAtLocation(property, alias);
+      if (!holdsString(type)) continue;
+      const known = fields.find((field) => field.name === property.name);
+      const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0;
+      if (known === undefined) fields.push({ name: property.name, always });
+      else known.always &&= always;
+    }
+    kinds.set(kindType.value, fields);
+  }
+}
+
+const lines = [
+  "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.",
+  "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.",
+  'import type { IrExpr, IrStmt, IrType } from "./ir.js";',
+  "",
+  "/** Calls `visit` with each string field of `node` (including `kind`) and its",
+  " * name: the string values `for (const key in node)` finds on a node, read",
+  " * by name in the order ir.ts declares them. */",
+  "export function eachStringField(",
+  "  node: IrExpr | IrStmt,",
+  "  visit: (value: string, field: string) => void,",
+  "): void {",
+  "  switch (node.kind) {",
+];
+for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+  lines.push(`    case ${JSON.stringify(kind)}: {`);
+  for (const { name, always } of fields) {
+    const read = `node.${name}`;
+    if (always) lines.push(`      visit(${read}, ${JSON.stringify(name)});`);
+    else
+      lines.push(`      if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`);
+  }
+  lines.push("      return;", "    }");
+}
+lines.push("  }", "}", "");
+
+// The deep walkers: every field of every object under an IR node, as a
+// recursive key walk finds them, read by name.
+const members = new Map();
+for (const name of ["IrExpr", "IrStmt", "IrType"]) {
+  const union = checker.getTypeAtLocation(aliases.get(name).name);
+  members.set(name, new Set(union.isUnion() ? union.types : [union]));
+}
+const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void;
+const scalar =
+  ts.TypeFlags.NumberLike |
+  ts.TypeFlags.BooleanLike |
+  ts.TypeFlags.BigIntLike |
+  ts.TypeFlags.ESSymbolLike;
+
+/** The walker an IR union member belongs to, or null. */
+function irKind(type) {
+  for (const [name, set] of members) if (set.has(type)) return name;
+  return null;
+}
+
+let temp = 0;
+/** Lines that walk `expr` of `type`: strings to `walker.string`, IR nodes to
+ * their walkers, arrays by element and other objects by field. */
+function walkLines(expr, type, indent, depth) {
+  if (depth > 12) throw new Error(`field type nests too deep at ${expr}`);
+  const parts = type.isUnion() ? type.types : [type];
+  const present = parts.filter((part) => (part.flags & nullish) === 0);
+  const optional = present.length !== parts.length;
+  const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0);
+  const objects = present.filter((part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0);
+  for (const part of present)
+    if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0)
+      throw new Error(`untyped field ${expr}`);
+  const body = [];
+  if (objects.length > 0) {
+    const kinds = new Set(objects.map(irKind));
+    if (kinds.size === 1 && !kinds.has(null)) {
+      const [kind] = kinds;
+      body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`);
+    } else if (objects.every((part) => checker.isArrayType(part))) {
+      if (objects.length !== 1) throw new Error(`several array types at ${expr}`);
+      const item = `item${temp++}`;
+      body.push(`for (const ${item} of ${expr}) {`);
+      body.push(...walkLines(item, checker.getTypeArguments(objects[0])[0], "  ", depth + 1));
+      body.push("}");
+    } else if (kinds.has(null) && objects.length === 1) {
+      for (const property of checker.getPropertiesOfType(objects[0])) {
+        if (property.name === "loc") continue;
+        const fieldType = checker.getTypeOfSymbol(property);
+        const field = `${expr}.${property.name}`;
+        const lines = walkLines(field, fieldType, "", depth + 1);
+        if (lines.length === 0) continue;
+        if (property.name === "type")
+          body.push("if (!skipType) {", ...lines.map((l) => `  ${l}`), "}");
+        else body.push(...lines);
+      }
+    } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`);
+  }
+  let lines = body;
+  if (strings.length > 0 && objects.length > 0)
+    lines = [
+      `if (typeof ${expr} === "string") walker.string(${expr});`,
+      ...(body.length > 0 ? [`else {`, ...body.map((l) => `  ${l}`), "}"] : []),
+    ];
+  else if (strings.length > 0) lines = [`walker.string(${expr});`];
+  if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0))
+    lines = [
+      `if (${expr} !== undefined && ${expr} !== null) {`,
+      ...lines.map((l) => `  ${l}`),
+      "}",
+    ];
+  else if (lines.length > 0 && optional)
+    lines = [
+      `if (${expr} !== undefined && ${expr} !== null) {`,
+      ...lines.map((l) => `  ${l}`),
+      "}",
+    ];
+  return lines.map((line) => indent + line);
+}
+
+lines.push(
+  "/** What a field walk reports (walkExprFields): each expression and",
+  " * statement node before its fields, and each string a field holds. */",
+  "export interface IrFieldWalker {",
+  "  node(node: IrExpr | IrStmt): void;",
+  "  string(value: string): void;",
+  "}",
+  "",
+);
+for (const name of ["IrExpr", "IrStmt", "IrType"]) {
+  const short = name.slice(2);
+  const nodeCall = name === "IrType" ? [] : ["  walker.node(node);"];
+  lines.push(
+    `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`,
+    " * every field of each object under it: the strings and nodes a recursive",
+    " * `for (const key in value)` walk with the same skips finds, read by name in",
+    " * the order ir.ts declares the fields. */",
+    `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`,
+    ...nodeCall,
+    "  switch (node.kind) {",
+  );
+  const byKind = new Map();
+  for (const member of members.get(name)) {
+    const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value;
+    if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`);
+    byKind.set(kind, member);
+  }
+  for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+    lines.push(`    case ${JSON.stringify(kind)}: {`);
+    for (const property of member.getProperties()) {
+      if (property.name === "loc") continue;
+      const fieldType = checker.getTypeOfSymbol(property);
+      const field = `node.${property.name}`;
+      const walk = walkLines(field, fieldType, "", 0);
+      if (walk.length === 0) continue;
+      if (property.name === "type")
+        lines.push("      if (!skipType) {", ...walk.map((l) => `        ${l}`), "      }");
+      else lines.push(...walk.map((l) => `      ${l}`));
+    }
+    lines.push("      return;", "    }");
+  }
+  lines.push("  }", "}", "");
+}
+const text = lines.join("\n");
+
+if (process.argv.includes("--check")) {
+  if (readFileSync(target, "utf8") !== text)
+    throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs");
+} else writeFileSync(target, text);

scripts/llvm-helper-inputs.mjs

@@ -0,0 +1,114 @@
+/* Inputs of the LLVM helper, captured once and replayed with any helper:
+ * scripts/refresh-llvm-profile.mjs trains the helper's profile on them and
+ * scripts/check-llvm-dist.mjs compares helpers on them. Each input is a
+ * program's LLVM text from `scriptc build --emit llvm` and the `emit`
+ * arguments the compiler passes for an executable build of it
+ * (native-codegen.ts), partitions included. The partition cache is left out,
+ * so every replay compiles every partition. */
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const EFFECT_FIXTURES = [
+  "async-sleep",
+  "async-promise-typed",
+  "async-timeout",
+  "async-resume-timer",
+  "async-yield",
+];
+
+/** The programs: tsc-ts (when a checkout is given), five Effect fixtures,
+ * the runtime benchmark workloads, every 50th numbered corpus program, and
+ * `extra` directories of standalone programs. */
+export function helperInputPrograms({ tscTs, extra = [] } = {}) {
+  const programs = [];
+  if (tscTs !== undefined)
+    programs.push({ name: "tsc-ts", cwd: join(tscTs, "packages/tsc-ts"), entry: "src/main.ts" });
+  for (const fixture of EFFECT_FIXTURES)
+    programs.push({
+      name: `effect-${fixture}`,
+      cwd: root,
+      entry: `tests/fixtures/effect/${fixture}.ts`,
+      args: ["--npm-static", "effect,fast-check,pure-rand"],
+    });
+  const suite = join(root, "benchmarks/runtime");
+  for (const file of readdirSync(join(suite, "workloads"))
+    .filter((f) => f.endsWith(".json"))
+    .sort()) {
+    const { entry } = JSON.parse(readFileSync(join(suite, "workloads", file), "utf8"));
+    // Variants of a workload share its entry and differ only in arguments.
+    if (!programs.some((program) => program.cwd === suite && program.entry === entry))
+      programs.push({ name: `wl-${file.slice(0, -5)}`, cwd: suite, entry });
+  }
+  const corpus = join(root, "tests/corpus");
+  for (const file of readdirSync(corpus).sort()) {
+    const number = /^(\d+)-.*\.ts$/.exec(file)?.[1];
+    if (number !== undefined && Number(number) % 50 === 0)
+      programs.push({ name: `corpus-${file.slice(0, -3)}`, cwd: corpus, entry: file });
+  }
+  for (const dir of extra)
+    for (const file of readdirSync(dir)
+      .filter((f) => f.endsWith(".ts"))
+      .sort())
+      programs.push({ name: `extra-${file.slice(0, -3)}`, cwd: dir, entry: file });
+  return programs;
+}
+
+/** Writes `<dir>/<name>.ll` for each program with this checkout's compiler
+ * and returns the inputs that compiled: `{ name, input, outputs, args(out) }`,
+ * where `args(out)` gives the helper's arguments writing into `out`. */
+export async function captureHelperInputs(programs, dir) {
+  const { nativeProgramPartitions } = await import(
+    join(root, "packages/compiler/dist/backend/native-codegen-core.js")
+  );
+  const cli = join(root, "packages/cli/dist/main.js");
+  const target =
+    process.arch === "arm64" ? "aarch64-unknown-linux-gnu" : "x86_64-unknown-linux-gnu";
+  mkdirSync(dir, { recursive: true });
+  const inputs = [];
+  for (const program of programs) {
+    const input = join(dir, `${program.name}.ll`);
+    if (!existsSync(input)) {
+      const built = spawnSync(
+        process.execPath,
+        [cli, "build", program.entry, "--emit", "llvm", "-o", input, ...(program.args ?? [])],
+        { cwd: program.cwd, encoding: "utf8" },
+      );
+      if (built.status !== 0) {
+        console.log(`skipped ${program.name}: the compiler refused it`);
+        continue;
+      }
+    }
+    const partitions = nativeProgramPartitions(statSync(input).size);
+    const source = join(program.cwd, program.entry);
+    inputs.push({
+      name: program.name,
+      input,
+      partitions,
+      args: (out) => [
+        "emit",
+        "--input",
+        input,
+        ...Array.from({ length: partitions }, (_, i) => [
+          "--output",
+          join(out, `${program.name}.part${i}.o`),
+        ]).flat(),
+        "--filetype",
+        "obj",
+        "--target",
+        target,
+        "--opt-level",
+        "2",
+        "--relocation-model",
+        "pic",
+        "--diagnostic-format",
+        "json",
+        "--source-path",
+        source,
+      ],
+    });
+  }
+  return inputs;
+}

scripts/native-cli-assets.mts

@@ -1,25 +1,46 @@
+import { spawnSync } from "node:child_process";
 import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
 import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js";
+import { copyPackage, llvmNotice } from "./native-cli-bundle.mjs";
 import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js";
 import {
   tscTsExecutable,
   tscTsPlatformPackage,
 } from "../packages/compiler/src/frontend/ts7/checker-backend.js";
 
-/** Copy release payloads explicitly; development caches and build trees never ship. */
-function copyPackage(source: string, destination: string, members: readonly string[]): void {
-  mkdirSync(destination, { recursive: true });
-  for (const member of ["package.json", ...members]) {
-    cpSync(join(source, member), join(destination, member), {
-      recursive: true,
-      filter: (path) => ![".cache", "node_modules", ".DS_Store"].includes(basename(path)),
-    });
+/** Removes the symbol table and debug sections of a copied executable.
+ * llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O
+ * executable, which arm64 macOS requires to run it. Windows executables keep
+ * their debug information in separate .pdb files, which are not copied.
+ * Without llvm-strip on PATH the executable keeps its symbols; the release
+ * hosts install it with LLVM. */
+function stripExecutable(path: string, platform: string): void {
+  if (platform === "win32") return;
+  const result = spawnSync("llvm-strip", [path], { stdio: "inherit" });
+  if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") {
+    console.warn(`llvm-strip is not on PATH; ${basename(path)} keeps its symbols`);
+    return;
   }
-  for (const name of readdirSync(source)) {
-    if (/^(?:LICENSE|COPYING|NOTICE|SCRIPTC_LICENSE|THIRD_PARTY_NOTICES)(?:\..*)?$/.test(name)) {
-      cpSync(join(source, name), join(destination, name));
+  if (result.error !== undefined) throw result.error;
+  if (result.status !== 0) throw new Error(`llvm-strip ${path} exited with ${result.status}`);
+}
+
+/** Rewrites CRLF line endings to LF in the text files under `directory`. A
+ * Windows checkout converts the repository's LF text files to CRLF; this
+ * restores the files the other hosts ship. Text is what Git's conversion
+ * treats as text: no NUL byte in the first 8000 bytes. */
+function normalizeLineEndings(directory: string): void {
+  for (const entry of readdirSync(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      normalizeLineEndings(path);
+      continue;
     }
+    if (!entry.isFile()) continue;
+    const bytes = readFileSync(path);
+    if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue;
+    writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1");
   }
 }
 
@@ -29,6 +50,12 @@ export function stageNativeCliAssets(options: {
   target: NativeTargetSpec;
   ts7: string;
   compilerVersion: string;
+  /** Strip the copied tsc-ts executable. */
+  strip: boolean;
+  /** Leave the LLVM helper and host runtime pack out: an npm platform
+   * package depends on their packages, and the compiler resolves them
+   * beside itself. */
+  platformPackages: boolean;
 }): Pick<
   NativeToolchainManifest,
   | "ts7"
@@ -47,16 +74,18 @@ export function stageNativeCliAssets(options: {
   cpSync(join(repository, "LICENSE"), join(output, "LICENSE"));
   const helper = join(lib, "llvm");
   const runtime = join(lib, target.runtimePackPackage.replace("@scriptc/", ""));
-  copyPackage(
-    join(repository, "packages", target.helper.packageName.replace("@scriptc/", "")),
-    helper,
-    ["bin"],
-  );
-  copyPackage(
-    join(repository, "packages", target.runtimePackPackage.replace("@scriptc/", "")),
-    runtime,
-    ["artifacts", "runtime-pack.json"],
-  );
+  if (!options.platformPackages) {
+    copyPackage(
+      join(repository, "packages", target.helper.packageName.replace("@scriptc/", "")),
+      helper,
+      ["bin"],
+    );
+    copyPackage(
+      join(repository, "packages", target.runtimePackPackage.replace("@scriptc/", "")),
+      runtime,
+      ["artifacts", "runtime-pack.json"],
+    );
+  }
   const typescript = join(lib, "typescript");
   copyPackage(dirname(dirname(options.ts7)), typescript, ["lib"]);
   // tsc-ts, the default checker, from the tsc-ts package the compiler depends on.
@@ -67,17 +96,25 @@ export function stageNativeCliAssets(options: {
   );
   const tscTs = join(lib, "tsc-ts");
   copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]);
+  if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform);
   const sources = join(lib, "runtime-sources");
   copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]);
+  normalizeLineEndings(sources);
   const declarations = join(lib, "declarations");
   mkdirSync(declarations, { recursive: true });
   for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) {
     cpSync(join(repository, "packages/compiler/ambient", name), join(declarations, name));
   }
+  cpSync(
+    join(repository, "packages/compiler/ambient/node-fallback"),
+    join(declarations, "node-fallback"),
+    { recursive: true },
+  );
   // Package shims (packageShimRoot): the built-in npm package implementations.
   cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), {
     recursive: true,
   });
+  normalizeLineEndings(declarations);
   // This optional Node host runs the built WASI module, never the compiler.
   const wasi = join(lib, "wasi");
   mkdirSync(join(wasi, "cli"), { recursive: true });
@@ -100,7 +137,7 @@ export function stageNativeCliAssets(options: {
       `scriptc ${options.compilerVersion} includes the existing TypeScript ${identity.version}, tsc-ts ${tscTsIdentity.version}, LLVM, and C runtime dependencies.`,
       "TypeScript licensing: lib/typescript/LICENSE and lib/typescript/NOTICE.txt.",
       "tsc-ts licensing and notices: lib/tsc-ts/LICENSE and lib/tsc-ts/THIRD_PARTY_NOTICES.md.",
-      "LLVM licensing and notices: lib/llvm/LICENSE and lib/llvm/THIRD_PARTY_NOTICES.",
+      llvmNotice(options.platformPackages ? target.helper.packageName : undefined),
       "Runtime dependency licenses: lib/runtime-sources/vendor/*/LICENSE*.",
       "",
     ].join("\n"),
@@ -108,9 +145,13 @@ export function stageNativeCliAssets(options: {
   return {
     ts7: relative(bin, join(typescript, "lib", basename(options.ts7))),
     tsc_ts: relative(bin, join(tscTs, "lib", basename(tscTsSource))),
-    llvm_package: relative(bin, helper),
-    runtime_pack: relative(bin, runtime),
-    runtime_packs: [{ target: target.name, path: relative(bin, runtime) }],
+    ...(options.platformPackages
+      ? {}
+      : {
+          llvm_package: relative(bin, helper),
+          runtime_pack: relative(bin, runtime),
+          runtime_packs: [{ target: target.name, path: relative(bin, runtime) }],
+        }),
     runtime_sources: relative(bin, sources),
     declarations: relative(bin, declarations),
     wasi_node_runner: relative(bin, join(wasi, "cli/wasi-runner.js")),

scripts/native-cli-bundle.mjs

@@ -0,0 +1,72 @@
+import { cpSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+
+/** Copy release payloads explicitly; development caches and build trees never ship. */
+export function copyPackage(source, destination, members) {
+  mkdirSync(destination, { recursive: true });
+  for (const member of ["package.json", ...members]) {
+    cpSync(join(source, member), join(destination, member), {
+      recursive: true,
+      filter: (path) => ![".cache", "node_modules", ".DS_Store"].includes(basename(path)),
+    });
+  }
+  for (const name of readdirSync(source)) {
+    if (/^(?:LICENSE|COPYING|NOTICE|SCRIPTC_LICENSE|THIRD_PARTY_NOTICES)(?:\..*)?$/.test(name)) {
+      cpSync(join(source, name), join(destination, name));
+    }
+  }
+}
+
+/** The LLVM helper and host runtime pack packages a native CLI package
+ * depends on. */
+export function platformPackagesOf(identity) {
+  const names = Object.keys(identity.dependencies ?? {});
+  const helper = names.find((name) => name.startsWith("@scriptc/llvm-"));
+  const runtime = names.find((name) => name.startsWith("@scriptc/runtime-"));
+  if (helper === undefined || runtime === undefined)
+    throw new Error(`${identity.name} must depend on its LLVM helper and runtime pack packages`);
+  return { helper, runtime };
+}
+
+/** Copies the helper and runtime pack packages from `packages` (the
+ * workspace's packages directory) into a distribution's lib directory, as a
+ * standalone archive needs them, and returns the manifest paths, relative to
+ * the distribution's bin directory in the target's path style. */
+export function bundlePlatformPackages({ packages, distribution, helper, runtime, windows }) {
+  const short = (name) => name.replace("@scriptc/", "");
+  copyPackage(join(packages, short(helper)), join(distribution, "lib", "llvm"), ["bin"]);
+  copyPackage(join(packages, short(runtime)), join(distribution, "lib", short(runtime)), [
+    "artifacts",
+    "runtime-pack.json",
+  ]);
+  const path = (name) => ["..", "lib", name].join(windows ? "\\" : "/");
+  return { llvm_package: path("llvm"), runtime_pack: path(short(runtime)) };
+}
+
+/** The THIRD_PARTY_NOTICES line for the LLVM helper, bundled or as a dependency. */
+export function llvmNotice(helper) {
+  return helper === undefined
+    ? "LLVM licensing and notices: lib/llvm/LICENSE and lib/llvm/THIRD_PARTY_NOTICES."
+    : `LLVM licensing and notices: LICENSE and THIRD_PARTY_NOTICES in the ${helper} package.`;
+}
+
+/** Turns a distribution that bundles its LLVM helper and runtime pack into
+ * an npm platform package's: removes both from lib and from the manifest at
+ * `manifestPath`, as `build-native-cli.mts --platform-packages` builds it. */
+export function unbundlePlatformPackages(distribution, manifestPath, helper) {
+  const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+  for (const path of [manifest.llvm_package, manifest.runtime_pack])
+    rmSync(resolve(dirname(manifestPath), path.replaceAll("\\", "/")), {
+      recursive: true,
+      force: true,
+    });
+  delete manifest.llvm_package;
+  delete manifest.runtime_pack;
+  delete manifest.runtime_packs;
+  writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
+  const notices = join(distribution, "THIRD_PARTY_NOTICES");
+  writeFileSync(
+    notices,
+    readFileSync(notices, "utf8").replace(llvmNotice(undefined), llvmNotice(helper)),
+  );
+}

scripts/package-native-cli.mjs

@@ -13,6 +13,7 @@ import {
 import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { bundlePlatformPackages, llvmNotice, platformPackagesOf } from "./native-cli-bundle.mjs";
 
 const repository = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const packages = resolve(process.argv[2] ?? join(repository, "packages"));
@@ -39,6 +40,27 @@ for (const name of readdirSync(packages)
     const binary = join(bin, name.includes("win32-") ? "scriptc.exe" : "scriptc");
     const manifest = JSON.parse(readFileSync(binary + ".json", "utf8"));
     const hostPath = (value) => (name.includes("win32-") ? value.replaceAll("\\", "/") : value);
+    // The standalone archive bundles the LLVM helper and runtime pack that
+    // the npm platform package depends on.
+    if (manifest.llvm_package === undefined) {
+      const { helper, runtime } = platformPackagesOf(identity);
+      const paths = bundlePlatformPackages({
+        packages,
+        distribution,
+        helper,
+        runtime,
+        windows: name.includes("win32-"),
+      });
+      manifest.llvm_package = paths.llvm_package;
+      manifest.runtime_pack = paths.runtime_pack;
+      manifest.runtime_packs = [{ target: manifest.target, path: paths.runtime_pack }];
+      writeFileSync(binary + ".json", JSON.stringify(manifest, null, 2) + "\n");
+      const notices = join(distribution, "THIRD_PARTY_NOTICES");
+      writeFileSync(
+        notices,
+        readFileSync(notices, "utf8").replace(llvmNotice(helper), llvmNotice(undefined)),
+      );
+    }
     // GitHub artifact transport drops executable modes. Restore the native
     // tools before validating and creating the standalone archive.
     for (const path of [

scripts/refresh-cli-layout.mjs

@@ -0,0 +1,275 @@
+#!/usr/bin/env node
+/* Refreshes packages/compiler/native/cli.layout.gz, the function layout that
+ * builds of the native compiler use (scripts/build-native-cli.mts). The
+ * layout lists the functions the compiler runs, in tiers: tier 1 for
+ * `--version` and rebuilds of unchanged programs, then the functions of the
+ * other builds by the share of training builds that run them (COLD_TIERS).
+ * The emitter writes the listed functions first, by tier, so the code a build
+ * runs sits together in the compiler's executable and a run maps fewer of its
+ * pages: a function nearly every build runs sits with the others that do, not
+ * among functions only a few programs need.
+ *
+ * Builds an instrumented native compiler for this Linux host (or reuses one
+ * with --instrumented), builds each training program cold and then again
+ * unchanged, and reads the functions each kind of run executed from the raw
+ * profiles. The training programs are the runtime benchmark workloads at
+ * `speed` and every seventh corpus program at the default optimization. A
+ * refresh takes about 15 minutes on 16 cores, most of it for the instrumented
+ * build. A stale layout lists some functions that no longer exist and misses
+ * new ones, which then keep their usual order.
+ *
+ *   node scripts/refresh-cli-layout.mjs [--jobs 8] [--instrumented <compiler dir>]
+ */
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+  appendFileSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { cpus, tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { constants, gzipSync, inflateSync } from "node:zlib";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const { values } = parseArgs({
+  options: {
+    jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) },
+    output: { type: "string", default: join(root, "packages/compiler/native/cli.layout.gz") },
+    work: { type: "string" },
+    instrumented: { type: "string" },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help) {
+  console.log(
+    "Usage: node scripts/refresh-cli-layout.mjs [--jobs N] [--output <layout.gz>] [--work <dir>] [--instrumented <compiler dir>]",
+  );
+  process.exit(0);
+}
+if (process.platform !== "linux" || process.arch !== "x64")
+  throw new Error("the compiler layout is collected on Linux x64");
+const jobs = Number(values.jobs);
+const output = resolve(values.output);
+mkdirSync(dirname(output), { recursive: true });
+const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-cli-layout-")));
+const raw = join(work, "raw");
+const builds = join(work, "builds");
+rmSync(raw, { recursive: true, force: true });
+for (const dir of [raw, builds]) mkdirSync(dir, { recursive: true });
+const log = join(work, "training.log");
+
+const cli = resolve(values.instrumented ?? join(work, "cli"));
+if (values.instrumented === undefined) {
+  console.log(`building the instrumented compiler in ${cli}`);
+  const built = spawnSync(
+    process.execPath,
+    [
+      "--max-old-space-size=8192",
+      "--import",
+      "tsx",
+      join(root, "scripts/build-native-cli.mts"),
+      cli,
+    ],
+    {
+      cwd: root,
+      stdio: "inherit",
+      env: { ...process.env, SCRIPTC_NATIVE_PROFILE_GENERATE: join(raw, "build-%4m.profraw") },
+    },
+  );
+  if (built.status !== 0) throw new Error("the instrumented compiler did not build");
+}
+const scriptc = join(cli, "bin/scriptc");
+// Phase timing shows whether a rebuild hit both caches. It is on for every
+// run, so the rebuild's environment matches the first build's.
+const baseEnv = { ...process.env, SCRIPTC_TIMING: "1" };
+delete baseEnv["SCRIPTC_LLVM_HELPER"];
+
+/** Runs the instrumented compiler, writing its profile to `options.profile`. */
+function run(args, options) {
+  return new Promise((done) => {
+    const child = spawn(scriptc, args, {
+      cwd: options.cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: {
+        ...baseEnv,
+        SCRIPTC_CACHE_DIR: options.cache,
+        LLVM_PROFILE_FILE: options.profile,
+      },
+    });
+    let text = "";
+    child.stdout.on("data", (chunk) => (text += chunk));
+    child.stderr.on("data", (chunk) => (text += chunk));
+    child.on("close", (code) => {
+      appendFileSync(log, `$ scriptc ${args.join(" ")}\n${text}exit ${code}\n`);
+      done({ ok: code === 0, text });
+    });
+  });
+}
+
+async function pool(tasks) {
+  let next = 0;
+  let passed = 0;
+  await Promise.all(
+    Array.from({ length: jobs }, async () => {
+      while (next < tasks.length) if (await tasks[next++]()) passed++;
+    }),
+  );
+  return passed;
+}
+
+const programNumber = (name) => Number.parseInt(name, 10);
+const training = [
+  ...readdirSync(join(root, "benchmarks/runtime/workloads"))
+    .filter((name) => name.endsWith(".ts"))
+    .map((name) => ({
+      file: join(root, "benchmarks/runtime/workloads", name),
+      optimization: "speed",
+    })),
+  ...readdirSync(join(root, "tests/corpus"))
+    .filter((name) => /^\d+-.*\.ts$/.test(name) && programNumber(name) % 7 === 0)
+    .map((name) => ({ file: join(root, "tests/corpus", name), optimization: "release" })),
+];
+
+// Tier 1 collects each run that took the warm path from its own profile; a
+// rebuild that missed a cache (it shows no `frontend-cache-hit` or
+// `executable-cache-hit` phase) is left out. The first builds each write
+// their own profile (cold-<program>-*), merged across the build's processes.
+const warm = new Set();
+let skipped = 0;
+async function warmRun(args, cache, name, cacheHits) {
+  const profile = join(raw, `warm-${name}.profraw`);
+  const { ok, text } = await run(args, { cwd: root, cache, profile });
+  if (ok && cacheHits.every((phase) => text.includes(`"phase":"${phase}"`)))
+    for (const fn of executedFunctions(profile)) warm.add(fn);
+  else skipped++;
+  rmSync(profile, { force: true });
+}
+
+const started = Date.now();
+await warmRun(["--version"], join(work, "cache-version"), "version", []);
+console.log(`training: ${training.length} programs, cold then unchanged, ${jobs} at a time`);
+const passed = await pool(
+  training.map((program, index) => async () => {
+    const cache = join(work, `cache-${index}`);
+    const out = join(builds, `program-${index}`);
+    const args = ["build", program.file, `--optimization=${program.optimization}`, "-o", out];
+    const { ok } = await run(args, {
+      cwd: root,
+      cache,
+      profile: join(raw, `cold-${index}-%1m.profraw`),
+    });
+    if (ok)
+      await warmRun(args, cache, String(index), ["frontend-cache-hit", "executable-cache-hit"]);
+    rmSync(cache, { recursive: true, force: true });
+    rmSync(out, { force: true });
+    return ok;
+  }),
+);
+console.log(
+  `trained in ${Math.round((Date.now() - started) / 1000)} s; ${passed} of ${training.length} programs built, ${skipped} rebuilds left out of tier 1 (see ${log})`,
+);
+
+/** Functions a raw profile (LLVM's format version 10) records as run: the
+ * names whose counters are not all zero, without the file prefix of
+ * internal functions. */
+function executedFunctions(path) {
+  const buf = readFileSync(path);
+  const u64 = (offset) => Number(buf.readBigUInt64LE(offset));
+  if (buf.readBigUInt64LE(0) !== 0xff6c70726f667281n)
+    throw new Error(`${path} is not a 64-bit raw profile`);
+  const version = Number(buf.readBigUInt64LE(8) & 0xffffffffn);
+  if (version !== 10) throw new Error(`${path} has raw profile version ${version}, expected 10`);
+  const [binaryIdsBytes, records, beforeCounters, counters, afterCounters] = [2, 3, 4, 5, 6].map(
+    (field) => u64(field * 8),
+  );
+  const [bitmapBytes, afterBitmap, namesBytes] = [7, 8, 9].map((field) => u64(field * 8));
+  const valueKinds = u64(15 * 8) + 1;
+  const recordBytes = Math.ceil((6 * 8 + 4 + 2 * valueKinds + 4) / 8) * 8;
+  const dataStart = 16 * 8 + binaryIdsBytes;
+  const countersStart = dataStart + records * recordBytes + beforeCounters;
+  const namesStart = countersStart + counters * 8 + afterCounters + bitmapBytes + afterBitmap;
+  const countersDelta = buf.readBigInt64LE(10 * 8);
+  const names = new Map();
+  let at = namesStart;
+  const uleb = () => {
+    let value = 0;
+    let scale = 1;
+    for (;;) {
+      const byte = buf[at++];
+      value += (byte & 0x7f) * scale;
+      if ((byte & 0x80) === 0) return value;
+      scale *= 128;
+    }
+  };
+  while (at < namesStart + namesBytes) {
+    const plain = uleb();
+    const packed = uleb();
+    if (plain === 0) break;
+    const bytes =
+      packed === 0 ? buf.subarray(at, at + plain) : inflateSync(buf.subarray(at, at + packed));
+    at += packed === 0 ? plain : packed;
+    for (const name of bytes.toString("latin1").split("\x01"))
+      names.set(createHash("md5").update(name, "latin1").digest().readBigUInt64LE(0), name);
+  }
+  const ran = [];
+  for (let index = 0; index < records; index++) {
+    const record = dataStart + index * recordBytes;
+    const counterOffset = Number(
+      buf.readBigInt64LE(record + 16) - (countersDelta - BigInt(index * recordBytes)),
+    );
+    const count = buf.readUInt32LE(record + 48);
+    for (let c = 0; c < count; c++) {
+      if (buf.readBigUInt64LE(countersStart + counterOffset + c * 8) !== 0n) {
+        const name = names.get(buf.readBigUInt64LE(record));
+        if (name !== undefined) ran.push(name.slice(name.indexOf(";") + 1));
+        break;
+      }
+    }
+  }
+  return ran;
+}
+
+// The cold tiers, by the least share of training builds that run a function:
+// tier 2 for functions at least 90% of the builds run, tier 3 for 60%, and so
+// on; tier 7 holds the functions fewer than 3% of the builds run.
+const COLD_TIERS = [0.9, 0.6, 0.3, 0.1, 0.03];
+const coldRuns = new Map();
+for (const name of readdirSync(raw)) {
+  const program = /^cold-(\d+)-/.exec(name)?.[1];
+  if (program === undefined) continue;
+  const ran = coldRuns.get(program) ?? new Set();
+  for (const fn of executedFunctions(join(raw, name))) ran.add(fn);
+  coldRuns.set(program, ran);
+}
+const runs = new Map();
+for (const ran of coldRuns.values()) for (const fn of ran) runs.set(fn, (runs.get(fn) ?? 0) + 1);
+const coldTier = (count) => {
+  const share = count / coldRuns.size;
+  const index = COLD_TIERS.findIndex((least) => share >= least);
+  return 2 + (index < 0 ? COLD_TIERS.length : index);
+};
+const lines = [
+  ...[...warm].sort().map((name) => `1 ${name}`),
+  ...[...runs]
+    .filter(([name]) => !warm.has(name))
+    .map(([name, count]) => [coldTier(count), name])
+    .sort((a, b) => a[0] - b[0] || (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0))
+    .map(([tier, name]) => `${tier} ${name}`),
+];
+writeFileSync(output, gzipSync(`${lines.join("\n")}\n`, { level: constants.Z_BEST_COMPRESSION }));
+const perTier = new Map();
+for (const line of lines)
+  perTier.set(line.split(" ")[0], (perTier.get(line.split(" ")[0]) ?? 0) + 1);
+console.log(
+  `wrote ${output}: ${[...perTier].map(([tier, count]) => `tier ${tier}: ${count}`).join(", ")} functions, from ${coldRuns.size} builds (${statSync(output).size} bytes)`,
+);
+rmSync(builds, { recursive: true, force: true });
+if (values.work === undefined) rmSync(work, { recursive: true, force: true });

scripts/refresh-llvm-profile.mjs

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+/* Refreshes native/llvm-codegen/llvm.profdata.gz, the profile the release
+ * LLVM distribution and the Linux helpers' own sources are optimized with
+ * (native/llvm-codegen/llvm-dist.mjs). The profile follows LLVM's sources
+ * and the kind of code scriptc emits rather than scriptc's commits, so it
+ * needs a refresh when LLVM is bumped, or when the emitted code changes
+ * enough that the distribution check (scripts/check-llvm-dist.mjs) loses
+ * much of its gain.
+ *
+ * Builds LLVM with IR instrumentation (scripts/build-llvm-dist.mjs
+ * --instrumented) and an instrumented helper against it, then runs that
+ * helper on the training inputs (scripts/llvm-helper-inputs.mjs): tsc-ts,
+ * five Effect fixtures, the runtime benchmark workloads and every 50th
+ * corpus program. The merged profile is written gzip-compressed. Needs a
+ * built checkout on Linux x64, clang, lld and llvm-profdata of LLVM_VERSION
+ * on PATH, and the LLVM release sources. About 20 minutes on 32 cores.
+ *
+ *   node scripts/refresh-llvm-profile.mjs --source <llvm-project-X.src>
+ *     --tsc-ts <tsc-ts checkout> [--work <dir>] [--jobs N] [--output <profdata.gz>]
+ */
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { cpus, tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { constants, gzipSync } from "node:zlib";
+import { llvmProfile } from "../native/llvm-codegen/llvm-dist.mjs";
+import { buildLinuxHelper } from "../native/llvm-codegen/linux-helper.mjs";
+import { captureHelperInputs, helperInputPrograms } from "./llvm-helper-inputs.mjs";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const { values } = parseArgs({
+  options: {
+    source: { type: "string" },
+    "tsc-ts": { type: "string" },
+    work: { type: "string" },
+    jobs: { type: "string", default: String(cpus().length) },
+    output: { type: "string", default: llvmProfile },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help || values.source === undefined || values["tsc-ts"] === undefined) {
+  console.log(
+    "Usage: node scripts/refresh-llvm-profile.mjs --source <llvm-project source> --tsc-ts <tsc-ts checkout> [--work <dir>] [--jobs N] [--output <profdata.gz>]",
+  );
+  process.exit(values.help ? 0 : 2);
+}
+if (process.platform !== "linux" || process.arch !== "x64")
+  throw new Error("the LLVM profile is collected on Linux x64");
+const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-llvm-profile-")));
+const prefix = join(work, "llvm-instrumented-install");
+const helper = join(work, "helper", "scriptc-llvm-codegen");
+const raw = join(work, "raw");
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, { stdio: "inherit", ...options });
+  if (result.status !== 0)
+    throw new Error(`${command} ${args[0]} failed with status ${result.status}`);
+}
+
+console.log(`building the instrumented LLVM in ${work}`);
+run(process.execPath, [
+  join(root, "scripts/build-llvm-dist.mjs"),
+  "--instrumented",
+  "--source",
+  resolve(values.source),
+  "--prefix",
+  prefix,
+  "--work",
+  work,
+  "--jobs",
+  values.jobs,
+]);
+const manifest = JSON.parse(
+  readFileSync(join(root, "packages/llvm-linux-x64-gnu/package.json"), "utf8"),
+);
+process.env.LLVM_DIR = join(prefix, "lib/cmake/llvm");
+process.env.SCRIPTC_LLVM_PROFILE_GENERATE = "1";
+delete process.env.SCRIPTC_LLVM_THINLTO;
+buildLinuxHelper({
+  arch: "x64",
+  libc: "gnu",
+  version: manifest.version,
+  build: join(work, "helper-build"),
+  output: helper,
+});
+
+const inputs = await captureHelperInputs(
+  helperInputPrograms({ tscTs: resolve(values["tsc-ts"]) }),
+  join(work, "inputs"),
+);
+rmSync(raw, { recursive: true, force: true });
+mkdirSync(raw, { recursive: true });
+const out = join(work, "objects");
+for (const input of inputs) {
+  rmSync(out, { recursive: true, force: true });
+  mkdirSync(out, { recursive: true });
+  const result = spawnSync(helper, input.args(out), {
+    env: { ...process.env, LLVM_PROFILE_FILE: join(raw, "helper-%p.profraw") },
+    encoding: "utf8",
+  });
+  console.log(`${input.name}: ${input.partitions} partitions, status ${result.status}`);
+  if (result.status !== 0) throw new Error(`${input.name}: ${result.stdout}${result.stderr}`);
+}
+
+const merged = join(work, "llvm.profdata");
+run("llvm-profdata", ["merge", "-o", merged, ...readdirSync(raw).map((file) => join(raw, file))]);
+const output = resolve(values.output);
+mkdirSync(dirname(output), { recursive: true });
+writeFileSync(output, gzipSync(readFileSync(merged), { level: constants.Z_BEST_COMPRESSION }));
+console.log(`wrote ${output} from ${inputs.length} inputs`);

scripts/smoke-native-install.mjs

@@ -54,7 +54,27 @@ function bytes(directory) {
   return size;
 }
 try {
-  const platformTarball = pack(platform);
+  // A platform package depends on its LLVM helper and runtime pack packages.
+  // Pack them from this checkout, and pack the platform package with the
+  // exact versions pnpm would publish for its workspace dependencies.
+  const dependencies = Object.keys(identity.dependencies ?? {});
+  let packed = platform;
+  if (dependencies.length > 0) {
+    packed = join(root, "platform");
+    mkdirSync(packed);
+    cpSync(join(platform, "dist"), join(packed, "dist"), { recursive: true });
+    writeFileSync(
+      join(packed, "package.json"),
+      JSON.stringify({
+        ...identity,
+        dependencies: Object.fromEntries(dependencies.map((name) => [name, identity.version])),
+      }),
+    );
+  }
+  const platformTarball = pack(packed);
+  const dependencyTarballs = dependencies.map((name) =>
+    pack(join(repository, "packages", name.replace("@scriptc/", ""))),
+  );
   const wrapper = join(root, "wrapper");
   mkdirSync(wrapper);
   cpSync(join(repository, "packages/cli/scripts"), join(wrapper, "scripts"), { recursive: true });
@@ -77,7 +97,11 @@ try {
   const tarball = pack(wrapper);
   const project = join(root, "project");
   mkdirSync(project);
-  command(npm, ["install", "--offline", "--no-audit", "--no-fund", tarball], project);
+  command(
+    npm,
+    ["install", "--offline", "--no-audit", "--no-fund", tarball, ...dependencyTarballs],
+    project,
+  );
   const executable = join(project, "node_modules/scriptc/bin/scriptc.exe");
   assert.equal(
     command(executable, ["--version"], project, { ...env, PATH: "" }).trim(),
@@ -99,8 +123,11 @@ try {
   );
   command(executable, ["build", entry, "-o", binary], project, compileEnv);
   assert.equal(command(binary, [], project, { PATH: "" }), "true true true\n");
-  const installed = readdirSync(join(project, "node_modules/@scriptc"));
-  assert.deepEqual(installed, [identity.name.slice("@scriptc/".length)]);
+  const installed = readdirSync(join(project, "node_modules/@scriptc")).sort();
+  assert.deepEqual(
+    installed,
+    [identity.name, ...dependencies].map((name) => name.slice("@scriptc/".length)).sort(),
+  );
   console.log(
     JSON.stringify({
       package: identity.name,

scripts/verify-native-cli.mjs

@@ -1,5 +1,6 @@
 import { execFileSync } from "node:child_process";
 import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs";
+import { createRequire } from "node:module";
 import { dirname, isAbsolute, join, relative, resolve, sep, win32 } from "node:path";
 import { verifyLinuxAbi } from "./verify-linux-abi.mjs";
 
@@ -26,11 +27,30 @@ const within = (path) => {
   if (child === ".." || child.startsWith(".." + sep) || isAbsolute(child))
     throw new Error(`distribution references external asset: ${path}`);
 };
+// An npm platform package depends on its LLVM helper and runtime pack
+// packages at its own version instead of bundling them.
+const platformPackages = toolchain.llvm_package === undefined;
+const dependencies = identity.dependencies ?? {};
+const dependency = (prefix) => {
+  const name = Object.keys(dependencies).find((key) => key.startsWith(prefix));
+  if (name === undefined || ![identity.version, "workspace:*"].includes(dependencies[name]))
+    throw new Error(
+      `${identity.name} must depend on its ${prefix}* package at ${identity.version}`,
+    );
+  return dirname(createRequire(join(root, "package.json")).resolve(`${name}/package.json`));
+};
+if (platformPackages && (toolchain.runtime_pack !== undefined || toolchain.runtime_packs))
+  throw new Error("a platform package's runtime pack is its dependency");
+const helperRoot = platformPackages
+  ? dependency("@scriptc/llvm-")
+  : resolve(dirname(binary), hostPath(toolchain.llvm_package));
+const runtimeRoot = platformPackages
+  ? dependency("@scriptc/runtime-")
+  : resolve(dirname(binary), hostPath(toolchain.runtime_pack));
 for (const name of [
   "ts7",
   "tsc_ts",
-  "llvm_package",
-  "runtime_pack",
+  ...(platformPackages ? [] : ["llvm_package", "runtime_pack"]),
   "runtime_sources",
   "declarations",
   "comptime",
@@ -41,14 +61,18 @@ for (const name of [
     throw new Error(`native toolchain needs a relative ${name}`);
   within(resolve(dirname(binary), hostPath(value)));
 }
-const pack = JSON.parse(
-  readFileSync(
-    resolve(dirname(binary), hostPath(toolchain.runtime_pack), "runtime-pack.json"),
-    "utf8",
-  ),
-);
+// Optional: a compiler this host could not run while building ships without it.
+if (toolchain.lib_diagnostics !== undefined) {
+  const value = toolchain.lib_diagnostics;
+  if (typeof value !== "string" || absolutePath(value))
+    throw new Error("native toolchain needs a relative lib_diagnostics");
+  within(resolve(dirname(binary), hostPath(value)));
+}
+const pack = JSON.parse(readFileSync(join(runtimeRoot, "runtime-pack.json"), "utf8"));
 if (pack.version !== identity.version || pack.target.name !== toolchain.target)
   throw new Error("native compiler runtime identity mismatch");
+if (JSON.parse(readFileSync(join(helperRoot, "package.json"), "utf8")).version !== identity.version)
+  throw new Error("native compiler LLVM helper version mismatch");
 const seen = new Set();
 for (const runtime of toolchain.runtime_packs ?? []) {
   if (seen.has(runtime.target)) throw new Error(`duplicate runtime target: ${runtime.target}`);
@@ -84,9 +108,7 @@ if (process.argv.includes("--run")) {
     ]) {
       verifyLinuxAbi(executable, { staticOnly: target.endsWith("-musl") });
     }
-    verifyLinuxAbi(resolve(dirname(binary), toolchain.llvm_package, "bin/scriptc-llvm-codegen"), {
-      staticOnly: true,
-    });
+    verifyLinuxAbi(join(helperRoot, "bin/scriptc-llvm-codegen"), { staticOnly: true });
   }
   const version = execFileSync(binary, ["--version"], {
     encoding: "utf8",
@@ -104,5 +126,12 @@ if (process.argv.includes("--run")) {
   }).trim();
   if (!tscTsVersion.startsWith("Version "))
     throw new Error(`the distribution's tsc-ts reported ${JSON.stringify(tscTsVersion)}`);
+  // The release strips tsc-ts and the comptime evaluator; arm64 macOS runs
+  // only executables whose linker-generated code signature matches their
+  // contents.
+  if (process.platform === "darwin" && process.arch === "arm64") {
+    for (const executable of [binary, tscTs, resolve(dirname(binary), toolchain.comptime)])
+      execFileSync("codesign", ["--verify", "--strict", executable], { stdio: "inherit" });
+  }
 }
 console.log(`${identity.name}@${identity.version}: native distribution verified`);

scripts/wait-npm-available.mjs

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+// Waits until every package of a release resolves on the npm registry.
+//
+// usage: wait-npm-available.mjs <uploads file> [--interval <seconds>] [--timeout <seconds>]
+//
+// Each line of the uploads file is `<name>@<version>`, optionally followed by
+// the Unix time (seconds) at which npm accepted the upload. Every interval
+// (120 seconds by default), each package that has not resolved yet is asked
+// for with `npm view <name>@<version> version --prefer-online`. A package
+// resolves when npm prints its version. Each package is reported once it
+// resolves, with the time from upload to the check that found it.
+//
+// Exits with 0 once every package resolves. When packages are still missing
+// at the timeout (2 hours by default), prints their names and versions and
+// exits with 1. The script only reads from the registry.
+import { execFile } from "node:child_process";
+import { appendFileSync, readFileSync } from "node:fs";
+import { setTimeout as delay } from "node:timers/promises";
+
+function usage() {
+  throw new Error(
+    "usage: wait-npm-available.mjs <uploads file> [--interval <seconds>] [--timeout <seconds>]",
+  );
+}
+
+const args = process.argv.slice(2);
+let file;
+let intervalSeconds = 120;
+let timeoutSeconds = 2 * 60 * 60;
+for (let i = 0; i < args.length; i++) {
+  const arg = args[i];
+  if (arg === "--interval" || arg === "--timeout") {
+    const value = Number(args[++i]);
+    if (!Number.isFinite(value) || value <= 0) usage();
+    if (arg === "--interval") intervalSeconds = value;
+    else timeoutSeconds = value;
+  } else if (file === undefined) {
+    file = arg;
+  } else {
+    usage();
+  }
+}
+if (file === undefined) usage();
+
+const packages = [];
+for (const line of readFileSync(file, "utf8").split("\n")) {
+  const fields = line.trim().split(/\s+/);
+  if (fields[0] === "") continue;
+  const spec = fields[0];
+  const at = spec.lastIndexOf("@");
+  if (at <= 0 || at === spec.length - 1 || fields.length > 2) {
+    throw new Error(`malformed upload line: ${line}`);
+  }
+  const acceptedAt = fields[1] === undefined ? null : Number(fields[1]) * 1000;
+  if (acceptedAt !== null && !Number.isFinite(acceptedAt)) {
+    throw new Error(`malformed upload time: ${line}`);
+  }
+  packages.push({ spec, version: spec.slice(at + 1), acceptedAt, availableAt: null });
+}
+if (packages.length === 0) throw new Error(`${file} names no packages`);
+
+function resolves(pkg) {
+  return new Promise((resolve) => {
+    execFile(
+      "npm",
+      ["view", pkg.spec, "version", "--prefer-online"],
+      { encoding: "utf8", timeout: 60_000 },
+      (error, stdout, stderr) => {
+        if (error === null && stdout.trim() === pkg.version) {
+          resolve(true);
+          return;
+        }
+        const code = /npm error code (\S+)/.exec(stderr)?.[1];
+        if (error !== null && code !== "E404") {
+          const reason = code ?? error.code ?? error.message.split("\n")[0];
+          console.log(`${pkg.spec}: npm view failed (${reason}); checking again`);
+        }
+        resolve(false);
+      },
+    );
+  });
+}
+
+function duration(ms) {
+  const seconds = Math.max(0, Math.round(ms / 1000));
+  const minutes = Math.floor(seconds / 60);
+  return minutes === 0 ? `${seconds}s` : `${minutes}m ${String(seconds % 60).padStart(2, "0")}s`;
+}
+
+function uploadToAvailable(pkg) {
+  return pkg.acceptedAt === null
+    ? "upload time not recorded"
+    : duration(pkg.availableAt - pkg.acceptedAt);
+}
+
+function summary(missing) {
+  const summaryFile = process.env.GITHUB_STEP_SUMMARY;
+  if (!summaryFile) return;
+  const rows = packages.map(
+    (pkg) =>
+      `| \`${pkg.spec}\` | ${pkg.availableAt === null ? "missing" : uploadToAvailable(pkg)} |`,
+  );
+  const heading =
+    missing.length === 0
+      ? `Every package resolves on npm (${packages.length}).`
+      : `${missing.length} of ${packages.length} packages do not resolve on npm after ${duration(timeoutSeconds * 1000)}.`;
+  appendFileSync(
+    summaryFile,
+    [heading, "", "| Package | Upload to available |", "| --- | --- |", ...rows, ""].join("\n"),
+  );
+}
+
+const started = Date.now();
+const deadline = started + timeoutSeconds * 1000;
+for (;;) {
+  const pending = packages.filter((pkg) => pkg.availableAt === null);
+  const checked = [];
+  // A few checks at a time keep each round short without flooding the registry.
+  for (let i = 0; i < pending.length; i += 8) {
+    const batch = pending.slice(i, i + 8);
+    const results = await Promise.all(batch.map(resolves));
+    const now = Date.now();
+    batch.forEach((pkg, j) => {
+      if (results[j]) {
+        pkg.availableAt = now;
+        checked.push(pkg);
+      }
+    });
+  }
+  for (const pkg of checked) console.log(`${pkg.spec} is available (${uploadToAvailable(pkg)})`);
+  const missing = packages.filter((pkg) => pkg.availableAt === null);
+  if (missing.length === 0) {
+    console.log(`Every package resolves on npm (${packages.length}).`);
+    summary(missing);
+    process.exit(0);
+  }
+  const now = Date.now();
+  if (now >= deadline) {
+    console.log(
+      `${missing.length} of ${packages.length} packages do not resolve on npm after ${duration(now - started)}:`,
+    );
+    for (const pkg of missing) console.log(`::error::${pkg.spec} does not resolve on npm`);
+    summary(missing);
+    process.exit(1);
+  }
+  console.log(
+    `${missing.length} of ${packages.length} packages do not resolve yet; checking again in ${duration(Math.min(intervalSeconds * 1000, deadline - now))}`,
+  );
+  await delay(Math.min(intervalSeconds * 1000, deadline - now));
+}

scripts/zero-cost-gate.mjs

@@ -80,7 +80,8 @@ runtime pack). Both compile the same program files, taken from --programs
 
 --set picks the good-code set: bench (benchmarks/runtime workloads and the
 benchmarks/builds applications), self (packages/compiler/src/native/cli.ts,
-LLVM only), corpus (tests/corpus TypeScript programs; --corpus=all adds
+LLVM only; the candidate's own compiler source must also build with the
+candidate), corpus (tests/corpus TypeScript programs; --corpus=all adds
 JavaScript entries). --quick is --set=bench --timing-runs=0, for CI.
 --shard=i/n keeps every n-th corpus program; --only filters program ids.
 
@@ -634,6 +635,30 @@ if (timingRuns > 0) {
   }
 }
 
+// The self program is the baseline's compiler source (the programs come
+// from one checkout), so a change that adds compiler code the self-hosted
+// frontend cannot lower would pass unnoticed. The candidate's own compiler
+// source must still build with the candidate (LLVM only, not compared).
+if (sets.has("self") && programsRoot !== trees.candidate) {
+  const entry = join(trees.candidate, "packages/compiler/src/native/cli.ts");
+  const result = await buildProgram(
+    { id: "self/candidate-source", entry, kind: "source" },
+    "candidate",
+  );
+  report.candidateSelf = result.ok
+    ? { ok: true, buildMs: result.ms }
+    : { ok: false, error: result.error };
+  console.log(
+    result.ok
+      ? `self       candidate compiler source builds with the candidate (${fmtMs(result.ms)})`
+      : `self       candidate compiler source FAILS to build with the candidate`,
+  );
+  if (!result.ok) {
+    failed = true;
+    report.failures.push(`self/candidate-source: ${result.error.split("\n")[0]}`);
+  }
+}
+
 const counts = {};
 for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1;
 if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) {