Store identical runtime artifacts once and strip the copied CLI executables

Runtime packs: variants of a unit that do not read their defines, and
flavors that compile a unit the same way, produce identical objects and
bitcode. The pack build now points those manifest entries at one file
(shareIdenticalArtifacts) and deletes the copies. Each link selects one
flavor and one variant per unit, so no link names a shared file twice. The
linux-x64-gnu pack keeps 1,191 of its 1,448 artifact files.

Native CLI packages:
- The comptime evaluator builds with -g0 and the executable strip flags,
  and the Windows build no longer ships its .pdb.
- The copied tsc-ts executable is stripped with llvm-strip, which rewrites
  the linker-signed code signature of Mach-O executables.
- On arm64 macOS, verify-native-cli.mjs --run checks the code signatures
  of the compiler, tsc-ts and the comptime evaluator.
- SCRIPTC_NATIVE_KEEP_SYMBOLS=1 keeps the symbols of all three.
- The copied runtime sources and declarations get LF line endings. A
  Windows checkout converts them to CRLF, so the Windows CLI shipped files
  that differed from @scriptc/runtime's.

packages/runtime-darwin-arm64/scripts/build.mjs

@@ -15,6 +15,7 @@ import {
 } from "../../runtime-pack-common/scripts/runtime-unit.mjs";
 import { createDeterministicArchive } from "./archive.mjs";
 import { installRuntimePack, withBuildLock } from "./build-state.mjs";
+import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs";
 
 const run = promisify(execFile);
 const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
@@ -365,6 +366,7 @@ async function build() {
         await copyFile(source, output);
       }),
     );
+    await shareIdenticalArtifacts(flavors, buildRoot);
     const manifest = {
       schema: "scriptc.runtime-pack.v1",
       format: 1,

packages/runtime-pack-common/scripts/build.mjs

@@ -15,6 +15,7 @@ import {
 import { createDeterministicArchive } from "./archive.mjs";
 import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
 import { installRuntimePack, withBuildLock } from "./build-state.mjs";
+import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
 import { parallelMap } from "./parallel-map.mjs";
 
 const run = promisify(execFile);
@@ -468,6 +469,7 @@ async function build() {
         await copyFile(source, output);
       }),
     );
+    await shareIdenticalArtifacts(flavors, buildRoot);
     const manifest = {
       schema: "scriptc.runtime-pack.v1",
       format: 1,

packages/runtime-pack-common/scripts/share-artifacts.mjs

@@ -0,0 +1,48 @@
+import { readdir, rm, rmdir } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Points the artifacts of one runtime unit that hold the same bytes at one
+ * file and deletes the copies. Variants whose defines a unit does not read,
+ * and flavors that compile a unit the same way, produce identical objects
+ * and bitcode. Each link selects one flavor and one variant of each unit, so
+ * no link names a shared file twice. `root` is the directory artifact paths
+ * are relative to; flavors keep the order they were built in, so the first
+ * flavor's path holds the shared file. */
+export async function shareIdenticalArtifacts(flavors, root) {
+  const firstPaths = new Map();
+  for (const flavor of Object.values(flavors)) {
+    for (const unit of flavor.runtime_units) {
+      let paths = firstPaths.get(unit.source);
+      if (paths === undefined) {
+        paths = new Map();
+        firstPaths.set(unit.source, paths);
+      }
+      for (const variant of unit.variants) {
+        for (const artifact of variant.bitcode === undefined
+          ? [variant]
+          : [variant, variant.bitcode]) {
+          const first = paths.get(artifact.sha256);
+          if (first === undefined) {
+            paths.set(artifact.sha256, artifact.path);
+          } else if (first !== artifact.path) {
+            await rm(join(root, artifact.path));
+            artifact.path = first;
+          }
+        }
+      }
+    }
+  }
+  await removeEmptyDirectories(join(root, "artifacts"));
+}
+
+async function removeEmptyDirectories(directory) {
+  const entries = await readdir(directory, { withFileTypes: true });
+  let remaining = entries.length;
+  for (const entry of entries) {
+    if (entry.isDirectory() && (await removeEmptyDirectories(join(directory, entry.name))))
+      remaining--;
+  }
+  if (remaining > 0) return false;
+  await rmdir(directory);
+  return true;
+}

packages/runtime-pack-common/test/share-artifacts.test.ts

@@ -0,0 +1,79 @@
+import { existsSync } from "node:fs";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { expect, test } from "vitest";
+import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs";
+
+type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact };
+
+async function artifact(root: string, path: string, sha256: string): Promise<Artifact> {
+  await mkdir(dirname(join(root, path)), { recursive: true });
+  await writeFile(join(root, path), sha256);
+  return { path, sha256, size: sha256.length };
+}
+
+test("identical artifacts of one unit share the first flavor's file", async () => {
+  const root = await mkdtemp(join(tmpdir(), "scriptc-share-artifacts-"));
+  try {
+    const unit = async (flavor: string, source: string, ids: Record<string, string>) => ({
+      source,
+      variants: await Promise.all(
+        Object.entries(ids).map(async ([id, sha]) => ({
+          id,
+          ...(await artifact(root, `artifacts/${flavor}/runtime/${id}/${source}.o`, sha)),
+        })),
+      ),
+    });
+    const release = {
+      runtime_units: [
+        await unit("release", "scr_a", { default: "a1", "default-workers": "a1" }),
+        await unit("release", "scr_b", { default: "b1" }),
+      ],
+    };
+    release.runtime_units[1]!.variants[0]!.bitcode = await artifact(
+      root,
+      "artifacts/release/runtime/default/scr_b.bc",
+      "b-bitcode",
+    );
+    const speed = {
+      runtime_units: [
+        await unit("speed", "scr_a", { default: "a2" }),
+        // The same bytes as another unit's object stay a separate file.
+        await unit("speed", "scr_b", { default: "b1", other: "a1" }),
+      ],
+    };
+    speed.runtime_units[1]!.variants[0]!.bitcode = await artifact(
+      root,
+      "artifacts/speed/runtime/default/scr_b.bc",
+      "b-bitcode",
+    );
+    const flavors = { release, speed };
+    await shareIdenticalArtifacts(flavors, root);
+    const paths = (flavor: typeof release) =>
+      flavor.runtime_units.map((unit) =>
+        unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]),
+      );
+    expect(paths(release)).toEqual([
+      [
+        ["artifacts/release/runtime/default/scr_a.o", undefined],
+        ["artifacts/release/runtime/default/scr_a.o", undefined],
+      ],
+      [["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]],
+    ]);
+    expect(paths(speed)).toEqual([
+      [["artifacts/speed/runtime/default/scr_a.o", undefined]],
+      [
+        ["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"],
+        ["artifacts/speed/runtime/other/scr_b.o", undefined],
+      ],
+    ]);
+    expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false);
+    expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false);
+    expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false);
+    expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true);
+    expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});

scripts/build-native-cli.mts

@@ -6,7 +6,11 @@ import { basename, dirname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { gunzipSync } from "node:zlib";
 import { compile } from "../packages/compiler/src/index.js";
-import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js";
+import {
+  NATIVE_TARGETS,
+  executableStripLinkerArgs,
+  nativeCodegenTarget,
+} from "../packages/compiler/src/backend/targets.js";
 import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js";
 import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js";
 import { stageNativeCliAssets } from "./native-cli-assets.mjs";
@@ -94,12 +98,14 @@ const ffi = join(seed, "compiler.ffi.json");
 writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n");
 const executable = join(bin, "scriptc" + target.outputSuffixes.exe);
 const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe);
+const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1";
 execFileSync(
   compiler,
   [
     ...compilerArgs,
     ...cFlags,
     "-O2",
+    ...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []),
     "-I",
     join(root, "packages/runtime/vendor/quickjs-ng"),
     join(nativeSources, "comptime.c"),
@@ -116,12 +122,15 @@ execFileSync(
   ],
   { stdio: "inherit" },
 );
+// The Windows link writes the debug database beside the executable.
+if (strip && target.platform === "win32")
+  rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true });
 const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), {
   outDir: seed,
   outPath: executable,
   backend: "llvm",
   optimization,
-  strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1",
+  strip,
   ffiProfilePath: ffi,
   sanitize,
   ...(profileGenerate === undefined ? {} : { profileGenerate }),
@@ -167,6 +176,7 @@ const assets = stageNativeCliAssets({
   target,
   ts7: tsgoExecutable(),
   compilerVersion,
+  strip,
 });
 // A cross-libc seed may use Zig on the build host. The installed compiler
 // selects the destination platform's ordinary linker and SDK.

scripts/native-cli-assets.mts

@@ -1,3 +1,4 @@
+import { execFileSync } from "node:child_process";
 import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, relative } from "node:path";
 import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js";
@@ -23,12 +24,41 @@ function copyPackage(source: string, destination: string, members: readonly stri
   }
 }
 
+/** Removes the symbol table and debug sections of a copied executable.
+ * llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O
+ * executable, which arm64 macOS requires to run it. Windows executables keep
+ * their debug information in separate .pdb files, which are not copied. */
+function stripExecutable(path: string, platform: string): void {
+  if (platform === "win32") return;
+  execFileSync("llvm-strip", [path], { stdio: "inherit" });
+}
+
+/** Rewrites CRLF line endings to LF in the text files under `directory`. A
+ * Windows checkout converts the repository's LF text files to CRLF; this
+ * restores the files the other hosts ship. Text is what Git's conversion
+ * treats as text: no NUL byte in the first 8000 bytes. */
+function normalizeLineEndings(directory: string): void {
+  for (const entry of readdirSync(directory, { withFileTypes: true })) {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) {
+      normalizeLineEndings(path);
+      continue;
+    }
+    if (!entry.isFile()) continue;
+    const bytes = readFileSync(path);
+    if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue;
+    writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1");
+  }
+}
+
 export function stageNativeCliAssets(options: {
   repository: string;
   output: string;
   target: NativeTargetSpec;
   ts7: string;
   compilerVersion: string;
+  /** Strip the copied tsc-ts executable. */
+  strip: boolean;
 }): Pick<
   NativeToolchainManifest,
   | "ts7"
@@ -67,8 +97,10 @@ export function stageNativeCliAssets(options: {
   );
   const tscTs = join(lib, "tsc-ts");
   copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]);
+  if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform);
   const sources = join(lib, "runtime-sources");
   copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]);
+  normalizeLineEndings(sources);
   const declarations = join(lib, "declarations");
   mkdirSync(declarations, { recursive: true });
   for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) {
@@ -78,6 +110,7 @@ export function stageNativeCliAssets(options: {
   cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), {
     recursive: true,
   });
+  normalizeLineEndings(declarations);
   // This optional Node host runs the built WASI module, never the compiler.
   const wasi = join(lib, "wasi");
   mkdirSync(join(wasi, "cli"), { recursive: true });

scripts/verify-native-cli.mjs

@@ -111,5 +111,12 @@ if (process.argv.includes("--run")) {
   }).trim();
   if (!tscTsVersion.startsWith("Version "))
     throw new Error(`the distribution's tsc-ts reported ${JSON.stringify(tscTsVersion)}`);
+  // The release strips tsc-ts and the comptime evaluator; arm64 macOS runs
+  // only executables whose linker-generated code signature matches their
+  // contents.
+  if (process.platform === "darwin" && process.arch === "arm64") {
+    for (const executable of [binary, tscTs, resolve(dirname(binary), toolchain.comptime)])
+      execFileSync("codesign", ["--verify", "--strict", executable], { stdio: "inherit" });
+  }
 }
 console.log(`${identity.name}@${identity.version}: native distribution verified`);