diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 6006e73ac..92f6a432a 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -15,6 +15,7 @@ import { } from "../../runtime-pack-common/scripts/runtime-unit.mjs"; import { createDeterministicArchive } from "./archive.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs"; +import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs"; const run = promisify(execFile); const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); @@ -365,6 +366,7 @@ async function build() { await copyFile(source, output); }), ); + await shareIdenticalArtifacts(flavors, buildRoot); const manifest = { schema: "scriptc.runtime-pack.v1", format: 1, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 357ba7337..8f2eb3bde 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -15,6 +15,7 @@ import { import { createDeterministicArchive } from "./archive.mjs"; import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs"; +import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { parallelMap } from "./parallel-map.mjs"; const run = promisify(execFile); @@ -468,6 +469,7 @@ async function build() { await copyFile(source, output); }), ); + await shareIdenticalArtifacts(flavors, buildRoot); const manifest = { schema: "scriptc.runtime-pack.v1", format: 1, diff --git a/packages/runtime-pack-common/scripts/share-artifacts.mjs b/packages/runtime-pack-common/scripts/share-artifacts.mjs new file mode 100644 index 000000000..8eaf0707c --- /dev/null +++ b/packages/runtime-pack-common/scripts/share-artifacts.mjs @@ -0,0 +1,48 @@ +import { readdir, rm, rmdir } from "node:fs/promises"; +import { join } from "node:path"; + +/** Points the artifacts of one runtime unit that hold the same bytes at one + * file and deletes the copies. Variants whose defines a unit does not read, + * and flavors that compile a unit the same way, produce identical objects + * and bitcode. Each link selects one flavor and one variant of each unit, so + * no link names a shared file twice. `root` is the directory artifact paths + * are relative to; flavors keep the order they were built in, so the first + * flavor's path holds the shared file. */ +export async function shareIdenticalArtifacts(flavors, root) { + const firstPaths = new Map(); + for (const flavor of Object.values(flavors)) { + for (const unit of flavor.runtime_units) { + let paths = firstPaths.get(unit.source); + if (paths === undefined) { + paths = new Map(); + firstPaths.set(unit.source, paths); + } + for (const variant of unit.variants) { + for (const artifact of variant.bitcode === undefined + ? [variant] + : [variant, variant.bitcode]) { + const first = paths.get(artifact.sha256); + if (first === undefined) { + paths.set(artifact.sha256, artifact.path); + } else if (first !== artifact.path) { + await rm(join(root, artifact.path)); + artifact.path = first; + } + } + } + } + } + await removeEmptyDirectories(join(root, "artifacts")); +} + +async function removeEmptyDirectories(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + let remaining = entries.length; + for (const entry of entries) { + if (entry.isDirectory() && (await removeEmptyDirectories(join(directory, entry.name)))) + remaining--; + } + if (remaining > 0) return false; + await rmdir(directory); + return true; +} diff --git a/packages/runtime-pack-common/test/share-artifacts.test.ts b/packages/runtime-pack-common/test/share-artifacts.test.ts new file mode 100644 index 000000000..a2cb68128 --- /dev/null +++ b/packages/runtime-pack-common/test/share-artifacts.test.ts @@ -0,0 +1,79 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { expect, test } from "vitest"; +import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs"; + +type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact }; + +async function artifact(root: string, path: string, sha256: string): Promise { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), sha256); + return { path, sha256, size: sha256.length }; +} + +test("identical artifacts of one unit share the first flavor's file", async () => { + const root = await mkdtemp(join(tmpdir(), "scriptc-share-artifacts-")); + try { + const unit = async (flavor: string, source: string, ids: Record) => ({ + source, + variants: await Promise.all( + Object.entries(ids).map(async ([id, sha]) => ({ + id, + ...(await artifact(root, `artifacts/${flavor}/runtime/${id}/${source}.o`, sha)), + })), + ), + }); + const release = { + runtime_units: [ + await unit("release", "scr_a", { default: "a1", "default-workers": "a1" }), + await unit("release", "scr_b", { default: "b1" }), + ], + }; + release.runtime_units[1]!.variants[0]!.bitcode = await artifact( + root, + "artifacts/release/runtime/default/scr_b.bc", + "b-bitcode", + ); + const speed = { + runtime_units: [ + await unit("speed", "scr_a", { default: "a2" }), + // The same bytes as another unit's object stay a separate file. + await unit("speed", "scr_b", { default: "b1", other: "a1" }), + ], + }; + speed.runtime_units[1]!.variants[0]!.bitcode = await artifact( + root, + "artifacts/speed/runtime/default/scr_b.bc", + "b-bitcode", + ); + const flavors = { release, speed }; + await shareIdenticalArtifacts(flavors, root); + const paths = (flavor: typeof release) => + flavor.runtime_units.map((unit) => + unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]), + ); + expect(paths(release)).toEqual([ + [ + ["artifacts/release/runtime/default/scr_a.o", undefined], + ["artifacts/release/runtime/default/scr_a.o", undefined], + ], + [["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]], + ]); + expect(paths(speed)).toEqual([ + [["artifacts/speed/runtime/default/scr_a.o", undefined]], + [ + ["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"], + ["artifacts/speed/runtime/other/scr_b.o", undefined], + ], + ]); + expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true); + expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 05f36cf68..c16e72777 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -6,7 +6,11 @@ import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { gunzipSync } from "node:zlib"; import { compile } from "../packages/compiler/src/index.js"; -import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; +import { + NATIVE_TARGETS, + executableStripLinkerArgs, + nativeCodegenTarget, +} from "../packages/compiler/src/backend/targets.js"; import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; @@ -94,12 +98,14 @@ const ffi = join(seed, "compiler.ffi.json"); writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n"); const executable = join(bin, "scriptc" + target.outputSuffixes.exe); const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe); +const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1"; execFileSync( compiler, [ ...compilerArgs, ...cFlags, "-O2", + ...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []), "-I", join(root, "packages/runtime/vendor/quickjs-ng"), join(nativeSources, "comptime.c"), @@ -116,12 +122,15 @@ execFileSync( ], { stdio: "inherit" }, ); +// The Windows link writes the debug database beside the executable. +if (strip && target.platform === "win32") + rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true }); const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), { outDir: seed, outPath: executable, backend: "llvm", optimization, - strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1", + strip, ffiProfilePath: ffi, sanitize, ...(profileGenerate === undefined ? {} : { profileGenerate }), @@ -167,6 +176,7 @@ const assets = stageNativeCliAssets({ target, ts7: tsgoExecutable(), compilerVersion, + strip, }); // A cross-libc seed may use Zig on the build host. The installed compiler // selects the destination platform's ordinary linker and SDK. diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts index bdecc1da2..8162c9091 100644 --- a/scripts/native-cli-assets.mts +++ b/scripts/native-cli-assets.mts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; @@ -23,12 +24,41 @@ function copyPackage(source: string, destination: string, members: readonly stri } } +/** Removes the symbol table and debug sections of a copied executable. + * llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O + * executable, which arm64 macOS requires to run it. Windows executables keep + * their debug information in separate .pdb files, which are not copied. */ +function stripExecutable(path: string, platform: string): void { + if (platform === "win32") return; + execFileSync("llvm-strip", [path], { stdio: "inherit" }); +} + +/** Rewrites CRLF line endings to LF in the text files under `directory`. A + * Windows checkout converts the repository's LF text files to CRLF; this + * restores the files the other hosts ship. Text is what Git's conversion + * treats as text: no NUL byte in the first 8000 bytes. */ +function normalizeLineEndings(directory: string): void { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + normalizeLineEndings(path); + continue; + } + if (!entry.isFile()) continue; + const bytes = readFileSync(path); + if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue; + writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1"); + } +} + export function stageNativeCliAssets(options: { repository: string; output: string; target: NativeTargetSpec; ts7: string; compilerVersion: string; + /** Strip the copied tsc-ts executable. */ + strip: boolean; }): Pick< NativeToolchainManifest, | "ts7" @@ -67,8 +97,10 @@ export function stageNativeCliAssets(options: { ); const tscTs = join(lib, "tsc-ts"); copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]); + if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform); const sources = join(lib, "runtime-sources"); copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]); + normalizeLineEndings(sources); const declarations = join(lib, "declarations"); mkdirSync(declarations, { recursive: true }); for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) { @@ -78,6 +110,7 @@ export function stageNativeCliAssets(options: { cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), { recursive: true, }); + normalizeLineEndings(declarations); // This optional Node host runs the built WASI module, never the compiler. const wasi = join(lib, "wasi"); mkdirSync(join(wasi, "cli"), { recursive: true }); diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index 310cc04a9..6a18f9e70 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -111,5 +111,12 @@ if (process.argv.includes("--run")) { }).trim(); if (!tscTsVersion.startsWith("Version ")) throw new Error(`the distribution's tsc-ts reported ${JSON.stringify(tscTsVersion)}`); + // The release strips tsc-ts and the comptime evaluator; arm64 macOS runs + // only executables whose linker-generated code signature matches their + // contents. + if (process.platform === "darwin" && process.arch === "arm64") { + for (const executable of [binary, tscTs, resolve(dirname(binary), toolchain.comptime)]) + execFileSync("codesign", ["--verify", "--strict", executable], { stdio: "inherit" }); + } } console.log(`${identity.name}@${identity.version}: native distribution verified`);