Store identical runtime artifacts once and strip the copied CLI executables (#2301)

This commit is contained in:
Chris Tate
2026-10-26 07:06:31 -05:00
committed by GitHub
10 changed files with 377 additions and 89 deletions
@@ -399,6 +399,19 @@ export function runtimePackFlavorKey(
return optimized;
}
/** The units a flavor links for `features`, each with its selected
* variant. Variants of one unit can share an artifact file (the pack stores
* identical objects once), so a variant is identified by its unit and id,
* not by its path. */
export function selectedRuntimeUnits(
flavor: RuntimePackFlavorManifest,
features: RuntimeFeatureSet,
): { source: string; variant: RuntimePackVariant }[] {
return flavor.runtime_units
.filter((unit) => evaluateRuntimePredicate(unit.predicate, features))
.map((unit) => ({ source: unit.source, variant: selectVariant(unit, features) }));
}
/** Selection is deterministic and independent of installation paths. Both
* hosts verify and privately stage exactly these artifacts before linking. */
export function selectRuntimePackArtifacts(
@@ -420,9 +433,7 @@ export function selectRuntimePackArtifacts(
);
return {
features,
runtime: selectedFlavor.runtime_units
.filter((unit) => evaluateRuntimePredicate(unit.predicate, features))
.map((unit) => selectVariant(unit, features)),
runtime: selectedRuntimeUnits(selectedFlavor, features).map(({ variant }) => variant),
archives: manifest.archives.filter((archive) =>
evaluateRuntimePredicate(archive.predicate, features),
),
+159 -70
View File
@@ -8,6 +8,78 @@ import type { RuntimePackManifest } from "../backend/runtime-pack-core.js";
import { NATIVE_TARGETS } from "../backend/targets.js";
import { buildSanitizedRuntime } from "./sanitizer.js";
/** Every runtime feature off except regex. */
const features = {
workers: false,
youngNursery: false,
eventTarget: false,
wasm: false,
dynTier: false,
streamDyn: false,
handleProps: false,
declAdopt: false,
typeOrigins: false,
dynForAwait: false,
eventsOnceSignal: false,
mime: false,
utilTiming: false,
dynBuffer: false,
builtinValues: false,
computedRequire: false,
warningReplace: false,
stdioExtended: false,
brotli: false,
moduleNamespace: false,
regex: true,
copying: false,
textDecoderLegacy: false,
fileHandle: false,
fetch: false,
zlib: false,
assert: false,
inspect: false,
dynInvoke: false,
dc: false,
dynAsync: false,
events: false,
emitter: false,
symbol: false,
bigint: false,
searchParams: false,
qs: false,
utilPromisify: false,
dynBuiltinClass: false,
dynBuiltinStream: false,
parseArgs: false,
stream: false,
net: false,
http: false,
http2: false,
dgram: false,
dnsPromises: false,
dgramDyn: false,
textCodecDyn: false,
watch: false,
foreignFfi: false,
nodeTest: false,
tls: false,
tlsCa: false,
cipher: false,
ecdh: false,
dh: false,
pki: false,
ed25519: false,
x509: false,
webcrypto: false,
perfLoop: false,
nativeFetch: false,
netEffective: false,
httpEffective: false,
tlsEffective: false,
tlsCaEffective: false,
zlibEffective: false,
};
test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
"sanitized vendor archives compile with host libc declarations",
() => {
@@ -72,76 +144,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
runtime: [],
archives: [artifact],
systemLibraries: [],
features: {
workers: false,
youngNursery: false,
eventTarget: false,
wasm: false,
dynTier: false,
streamDyn: false,
handleProps: false,
declAdopt: false,
typeOrigins: false,
dynForAwait: false,
eventsOnceSignal: false,
mime: false,
utilTiming: false,
dynBuffer: false,
builtinValues: false,
computedRequire: false,
warningReplace: false,
stdioExtended: false,
brotli: false,
moduleNamespace: false,
regex: true,
copying: false,
textDecoderLegacy: false,
fileHandle: false,
fetch: false,
zlib: false,
assert: false,
inspect: false,
dynInvoke: false,
dc: false,
dynAsync: false,
events: false,
emitter: false,
symbol: false,
bigint: false,
searchParams: false,
qs: false,
utilPromisify: false,
dynBuiltinClass: false,
dynBuiltinStream: false,
parseArgs: false,
stream: false,
net: false,
http: false,
http2: false,
dgram: false,
dnsPromises: false,
dgramDyn: false,
textCodecDyn: false,
watch: false,
foreignFfi: false,
nodeTest: false,
tls: false,
tlsCa: false,
cipher: false,
ecdh: false,
dh: false,
pki: false,
ed25519: false,
x509: false,
webcrypto: false,
perfLoop: false,
nativeFetch: false,
netEffective: false,
httpEffective: false,
tlsEffective: false,
tlsCaEffective: false,
zlibEffective: false,
},
features,
},
},
directory,
@@ -160,3 +163,89 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
}
},
);
test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
"variants that share an artifact file compile once, as the selected variant",
() => {
const target = NATIVE_TARGETS.find(
(item) =>
item.platform === process.platform &&
item.architecture === process.arch &&
!item.name.endsWith("-musl"),
);
if (target === undefined) throw new Error("unsupported sanitizer test host");
const runtimeSourceRoot = fileURLToPath(new URL("../../../runtime", import.meta.url));
const { version } = JSON.parse(
readFileSync(join(runtimeSourceRoot, "package.json"), "utf8"),
) as { version: string };
const directory = mkdtempSync("/tmp/scriptc-sanitized-shared-");
// The pack stores the identical objects of both variants once.
const shared = {
path: "artifacts/dev/runtime/default/scr_hash.o",
sha256: "0".repeat(64),
size: 0,
};
const variants = [
{ ...shared, id: "default", when: { workers: false }, defines: [] },
{ ...shared, id: "default-workers", when: { workers: true }, defines: ["SCR_WORKERS"] },
];
const manifest: RuntimePackManifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
package: target.runtimePackPackage,
version,
target: {
name: target.name,
llvm_triple: target.llvmTriple,
architecture: target.architecture,
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: RUNTIME_ABI_VERSION, marker: RUNTIME_ABI_MARKER },
compiler: { command: "clang", identity: "test", target: target.llvmTriple },
macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" },
flavors: {
dev: {
optimization: "-O0",
runtime_units: [{ source: "scr_hash.c", predicate: true, variants }],
},
},
archives: [],
system_libraries: [],
licenses: [],
};
try {
const built = buildSanitizedRuntime(
{
compilerVersion: version,
target,
helper: target.helper,
runtimeSourceRoot,
ts7Executable: "unused",
helperExecutable: "unused",
helperPackageRoot: "unused",
runtimePackRoot: "unused",
linker: "unused",
linkerArgs: [],
dsymutil: "unused",
},
{
root: directory,
manifest,
licenses: manifest.licenses,
packageText: "",
manifestText: "",
flavor: "dev",
selected: { runtime: [variants[0]!], archives: [], systemLibraries: [], features },
},
directory,
"executable",
null,
);
expect(built.runtimeObjects).toEqual([join(directory, "scr_hash.o")]);
expect(statSync(join(directory, "scr_hash.o")).size).toBeGreaterThan(0);
} finally {
rmSync(directory, { recursive: true, force: true });
}
},
);
+13 -14
View File
@@ -9,7 +9,11 @@ import {
type NativeRuntimePack,
type NativeRuntimeSelection,
} from "../backend/runtime-pack-native.js";
import { runtimePackFlavorKey, type RuntimePackMode } from "../backend/runtime-pack-core.js";
import {
runtimePackFlavorKey,
selectedRuntimeUnits,
type RuntimePackMode,
} from "../backend/runtime-pack-core.js";
import { LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js";
import { contentDigest, type NativeCache } from "./cache.js";
import type { NativeToolchain } from "./toolchain.js";
@@ -127,7 +131,6 @@ export function buildSanitizedRuntime(
const key = runtimePackFlavorKey(manifest, selection.flavor, mode);
const flavor = manifest.flavors[key];
if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`);
const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path));
const common = [
"-std=c11",
"-pthread",
@@ -147,18 +150,14 @@ export function buildSanitizedRuntime(
join(brotli, "include"),
...(toolchain.target.platform === "linux" ? ["-ffunction-sections", "-fdata-sections"] : []),
];
const runtimeObjects: string[] = [];
for (const unit of flavor.runtime_units) {
for (const variant of unit.variants) {
if (wanted.has(variant.path))
runtimeObjects.push(
compile(join(src, unit.source), unit.source.replace(/\.c$/, ".o"), [
...common,
...variant.defines.map((define) => "-D" + define),
]),
);
}
}
// The units and variants the pack selection chose, compiled from source.
const runtimeObjects = selectedRuntimeUnits(flavor, selection.selected.features).map(
({ source, variant }) =>
compile(join(src, source), source.replace(/\.c$/, ".o"), [
...common,
...variant.defines.map((define) => "-D" + define),
]),
);
const archives: string[] = [];
for (const archive of selection.selected.archives) {
const name = basename(archive.path)
@@ -15,6 +15,7 @@ import {
} from "../../runtime-pack-common/scripts/runtime-unit.mjs";
import { createDeterministicArchive } from "./archive.mjs";
import { installRuntimePack, withBuildLock } from "./build-state.mjs";
import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs";
const run = promisify(execFile);
const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
@@ -365,6 +366,7 @@ async function build() {
await copyFile(source, output);
}),
);
await shareIdenticalArtifacts(flavors, buildRoot);
const manifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
@@ -15,6 +15,7 @@ import {
import { createDeterministicArchive } from "./archive.mjs";
import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
import { installRuntimePack, withBuildLock } from "./build-state.mjs";
import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
import { parallelMap } from "./parallel-map.mjs";
const run = promisify(execFile);
@@ -468,6 +469,7 @@ async function build() {
await copyFile(source, output);
}),
);
await shareIdenticalArtifacts(flavors, buildRoot);
const manifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
@@ -0,0 +1,48 @@
import { readdir, rm, rmdir } from "node:fs/promises";
import { join } from "node:path";
/** Points the artifacts of one runtime unit that hold the same bytes at one
* file and deletes the copies. Variants whose defines a unit does not read,
* and flavors that compile a unit the same way, produce identical objects
* and bitcode. Each link selects one flavor and one variant of each unit, so
* no link names a shared file twice. `root` is the directory artifact paths
* are relative to; flavors keep the order they were built in, so the first
* flavor's path holds the shared file. */
export async function shareIdenticalArtifacts(flavors, root) {
const firstPaths = new Map();
for (const flavor of Object.values(flavors)) {
for (const unit of flavor.runtime_units) {
let paths = firstPaths.get(unit.source);
if (paths === undefined) {
paths = new Map();
firstPaths.set(unit.source, paths);
}
for (const variant of unit.variants) {
for (const artifact of variant.bitcode === undefined
? [variant]
: [variant, variant.bitcode]) {
const first = paths.get(artifact.sha256);
if (first === undefined) {
paths.set(artifact.sha256, artifact.path);
} else if (first !== artifact.path) {
await rm(join(root, artifact.path));
artifact.path = first;
}
}
}
}
}
await removeEmptyDirectories(join(root, "artifacts"));
}
async function removeEmptyDirectories(directory) {
const entries = await readdir(directory, { withFileTypes: true });
let remaining = entries.length;
for (const entry of entries) {
if (entry.isDirectory() && (await removeEmptyDirectories(join(directory, entry.name))))
remaining--;
}
if (remaining > 0) return false;
await rmdir(directory);
return true;
}
@@ -0,0 +1,79 @@
import { existsSync } from "node:fs";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { expect, test } from "vitest";
import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs";
type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact };
async function artifact(root: string, path: string, sha256: string): Promise<Artifact> {
await mkdir(dirname(join(root, path)), { recursive: true });
await writeFile(join(root, path), sha256);
return { path, sha256, size: sha256.length };
}
test("identical artifacts of one unit share the first flavor's file", async () => {
const root = await mkdtemp(join(tmpdir(), "scriptc-share-artifacts-"));
try {
const unit = async (flavor: string, source: string, ids: Record<string, string>) => ({
source,
variants: await Promise.all(
Object.entries(ids).map(async ([id, sha]) => ({
id,
...(await artifact(root, `artifacts/${flavor}/runtime/${id}/${source}.o`, sha)),
})),
),
});
const release = {
runtime_units: [
await unit("release", "scr_a", { default: "a1", "default-workers": "a1" }),
await unit("release", "scr_b", { default: "b1" }),
],
};
release.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/release/runtime/default/scr_b.bc",
"b-bitcode",
);
const speed = {
runtime_units: [
await unit("speed", "scr_a", { default: "a2" }),
// The same bytes as another unit's object stay a separate file.
await unit("speed", "scr_b", { default: "b1", other: "a1" }),
],
};
speed.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/speed/runtime/default/scr_b.bc",
"b-bitcode",
);
const flavors = { release, speed };
await shareIdenticalArtifacts(flavors, root);
const paths = (flavor: typeof release) =>
flavor.runtime_units.map((unit) =>
unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]),
);
expect(paths(release)).toEqual([
[
["artifacts/release/runtime/default/scr_a.o", undefined],
["artifacts/release/runtime/default/scr_a.o", undefined],
],
[["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]],
]);
expect(paths(speed)).toEqual([
[["artifacts/speed/runtime/default/scr_a.o", undefined]],
[
["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"],
["artifacts/speed/runtime/other/scr_b.o", undefined],
],
]);
expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true);
expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true);
} finally {
await rm(root, { recursive: true, force: true });
}
});
+12 -2
View File
@@ -6,7 +6,11 @@ import { basename, dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { gunzipSync } from "node:zlib";
import { compile } from "../packages/compiler/src/index.js";
import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js";
import {
NATIVE_TARGETS,
executableStripLinkerArgs,
nativeCodegenTarget,
} from "../packages/compiler/src/backend/targets.js";
import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js";
import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js";
import { stageNativeCliAssets } from "./native-cli-assets.mjs";
@@ -103,12 +107,14 @@ const ffi = join(seed, "compiler.ffi.json");
writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n");
const executable = join(bin, "scriptc" + target.outputSuffixes.exe);
const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe);
const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1";
execFileSync(
compiler,
[
...compilerArgs,
...cFlags,
"-O2",
...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []),
"-I",
join(root, "packages/runtime/vendor/quickjs-ng"),
join(nativeSources, "comptime.c"),
@@ -125,12 +131,15 @@ execFileSync(
],
{ stdio: "inherit" },
);
// The Windows link writes the debug database beside the executable.
if (strip && target.platform === "win32")
rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true });
const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), {
outDir: seed,
outPath: executable,
backend: "llvm",
optimization,
strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1",
strip,
ffiProfilePath: ffi,
sanitize,
...(profileGenerate === undefined ? {} : { profileGenerate }),
@@ -177,6 +186,7 @@ const assets = stageNativeCliAssets({
target,
ts7: tsgoExecutable(),
compilerVersion,
strip,
});
// A cross-libc seed may use Zig on the build host. The installed compiler
// selects the destination platform's ordinary linker and SDK.
+41
View File
@@ -1,3 +1,4 @@
import { spawnSync } from "node:child_process";
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { basename, dirname, join, relative } from "node:path";
import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js";
@@ -23,12 +24,49 @@ function copyPackage(source: string, destination: string, members: readonly stri
}
}
/** Removes the symbol table and debug sections of a copied executable.
* llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O
* executable, which arm64 macOS requires to run it. Windows executables keep
* their debug information in separate .pdb files, which are not copied.
* Without llvm-strip on PATH the executable keeps its symbols; the release
* hosts install it with LLVM. */
function stripExecutable(path: string, platform: string): void {
if (platform === "win32") return;
const result = spawnSync("llvm-strip", [path], { stdio: "inherit" });
if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") {
console.warn(`llvm-strip is not on PATH; ${basename(path)} keeps its symbols`);
return;
}
if (result.error !== undefined) throw result.error;
if (result.status !== 0) throw new Error(`llvm-strip ${path} exited with ${result.status}`);
}
/** Rewrites CRLF line endings to LF in the text files under `directory`. A
* Windows checkout converts the repository's LF text files to CRLF; this
* restores the files the other hosts ship. Text is what Git's conversion
* treats as text: no NUL byte in the first 8000 bytes. */
function normalizeLineEndings(directory: string): void {
for (const entry of readdirSync(directory, { withFileTypes: true })) {
const path = join(directory, entry.name);
if (entry.isDirectory()) {
normalizeLineEndings(path);
continue;
}
if (!entry.isFile()) continue;
const bytes = readFileSync(path);
if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue;
writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1");
}
}
export function stageNativeCliAssets(options: {
repository: string;
output: string;
target: NativeTargetSpec;
ts7: string;
compilerVersion: string;
/** Strip the copied tsc-ts executable. */
strip: boolean;
}): Pick<
NativeToolchainManifest,
| "ts7"
@@ -67,8 +105,10 @@ export function stageNativeCliAssets(options: {
);
const tscTs = join(lib, "tsc-ts");
copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]);
if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform);
const sources = join(lib, "runtime-sources");
copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]);
normalizeLineEndings(sources);
const declarations = join(lib, "declarations");
mkdirSync(declarations, { recursive: true });
for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) {
@@ -83,6 +123,7 @@ export function stageNativeCliAssets(options: {
cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), {
recursive: true,
});
normalizeLineEndings(declarations);
// This optional Node host runs the built WASI module, never the compiler.
const wasi = join(lib, "wasi");
mkdirSync(join(wasi, "cli"), { recursive: true });
+7
View File
@@ -111,5 +111,12 @@ if (process.argv.includes("--run")) {
}).trim();
if (!tscTsVersion.startsWith("Version "))
throw new Error(`the distribution's tsc-ts reported ${JSON.stringify(tscTsVersion)}`);
// The release strips tsc-ts and the comptime evaluator; arm64 macOS runs
// only executables whose linker-generated code signature matches their
// contents.
if (process.platform === "darwin" && process.arch === "arm64") {
for (const executable of [binary, tscTs, resolve(dirname(binary), toolchain.comptime)])
execFileSync("codesign", ["--verify", "--strict", executable], { stdio: "inherit" });
}
}
console.log(`${identity.name}@${identity.version}: native distribution verified`);