feat: tree-shake executable runtime sections

Co-authored-by: mmamedel <23098414+mmamedel@users.noreply.github.com>
This commit is contained in:
Chris Tate
2026-08-31 08:29:39 -05:00
co-authored by mmamedel
parent 2b6e6524a3
commit 8fd115f556
9 changed files with 335 additions and 13 deletions
@@ -84,6 +84,20 @@ describe("native link info recipes", () => {
});
expect(info.runtime_pack.source_sets[0]?.c_flags).toContain("-O0");
expect(info.runtime_pack.source_sets[0]?.c_flags).not.toContain("-O2");
// External objects are compiled separately and the driver recipe is an
// executable link. The two halves must not leak into each other's recipe.
expect(info.runtime_pack.source_sets[0]?.c_flags).not.toContain("-Wl,-dead_strip");
expect(info.link.driver_flags).toContain("-Wl,-dead_strip");
});
test("static external-object links dead-strip just like dynamic links", async () => {
const info = await createNativeLinkInfo({
programObject: "/out/app.o",
target: MACOS_ARM64_TARGET,
features: BASE,
ffi: null,
});
expect(info.link.driver_flags).toContain("-Wl,-dead_strip");
});
test("FFI symbols and resolved inputs remain ordered before the runtime", async () => {
@@ -2,7 +2,11 @@ import { readFile, readdir } from "node:fs/promises";
import { dirname, join } from "node:path";
import type { FfiProfile } from "../ffi/ffi-manifest.js";
import { compilerReleaseVersion } from "../library/sidecar.js";
import { EXECUTABLE_RUNTIME_SOURCES, runtimeSrcDir } from "./native-toolchain.js";
import {
EXECUTABLE_RUNTIME_SOURCES,
executableSectionEliminationFlags,
runtimeSrcDir,
} from "./native-toolchain.js";
import {
EXTERNAL_OBJECT_ABI_STABILITY,
RUNTIME_ABI_MARKER,
@@ -113,6 +117,10 @@ function runtimeSourceRecipe(
env: NodeJS.ProcessEnv,
optimization: "release" | "dev",
): { sets: NativeSourceSet[]; systemLibraries: string[]; driverFlags: string[] } {
// Native link info currently describes the supported Mach-O external-object
// recipe. Keep this in the same target-aware helper as compileC so a
// consumer's link has the identical executable reachability semantics.
const executableSectionFlags = executableSectionEliminationFlags("darwin");
const dynamic = features.dynamic;
const curlFetch = dynamic && features.fetch && env["SCRIPTC_FETCH_CURL"] === "1";
const nativeFetch = features.fetch && !curlFetch;
@@ -185,6 +193,7 @@ function runtimeSourceRecipe(
c_flags: [
"-std=c11", ...commonTargetFlags, "-pthread",
optimization === "dev" ? "-O0" : "-O2",
...executableSectionFlags.compile,
"-fno-math-errno", "-fno-strict-aliasing", "-Wno-deprecated-declarations",
],
}];
@@ -239,7 +248,7 @@ function runtimeSourceRecipe(
driverFlags: [
...commonTargetFlags,
"-pthread",
...(dynamic ? ["-Wl,-dead_strip"] : []),
...executableSectionFlags.link,
],
};
}
@@ -10,6 +10,7 @@ import {
ccVersion,
compileC,
compileLibArchive,
executableSectionEliminationFlags,
executableNativeEnvironmentFingerprint,
implicitDependencyProbeIncludes,
parseLinkTraceFiles,
@@ -132,6 +133,22 @@ test("the production cache root follows overrides, platform defaults, and the ha
);
});
test("executable section elimination flags are target-aware and never enter library recipes", () => {
expect(executableSectionEliminationFlags("darwin")).toEqual({
compile: [],
link: ["-Wl,-dead_strip"],
});
expect(executableSectionEliminationFlags("linux")).toEqual({
compile: ["-ffunction-sections", "-fdata-sections"],
link: ["-Wl,--gc-sections"],
});
expect(executableSectionEliminationFlags("win32")).toEqual({
compile: ["-ffunction-sections", "-fdata-sections"],
link: ["-Wl,--gc-sections"],
});
expect(executableSectionEliminationFlags("wasi")).toEqual({ compile: [], link: [] });
});
test.skipIf(process.platform === "win32")(
"the early executable identity follows a compiler selected behind a stable driver",
async () => {
@@ -2741,7 +2758,14 @@ exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
`,
);
await chmod(wrapper, 0o755);
await writeFile(cPath, "int main(void) { return 0; }\n");
// Keep the injected datum reachable. With executable section GC an
// unreferenced test-only global is rightly removed before the binary
// assertion below can observe it.
await writeFile(
cPath,
"extern int scriptc_runtime_race_marker(void);\n" +
"int main(void) { return scriptc_runtime_race_marker(); }\n",
);
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = fakeRuntime;
process.env["SCRIPTC_TEST_REAL_CLANG"] = realClang!;
@@ -2768,7 +2792,11 @@ exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
}
await writeFile(
raceSource,
`${originalRuntimeSource}\nconst char scriptc_runtime_race_marker[] = "${marker}";\n`,
`${originalRuntimeSource}\n` +
`volatile const char scriptc_runtime_race_marker_data[] = "${marker}";\n` +
"int scriptc_runtime_race_marker(void) {\n" +
" return scriptc_runtime_race_marker_data[0] == '\\0';\n" +
"}\n",
);
await writeFile(release, "go");
await firstBuild;
@@ -81,6 +81,46 @@ function stableTestMemo<T>(
export const EXECUTABLE_RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"] as const;
/**
* Per-executable section-elimination recipe. This belongs beside the native
* driver rather than a particular build path: one-shot compilation, cached
* runtime objects, external object recipes, and runtime packs must all agree
* on what their final executable linker is allowed to discard.
*
* Archives and compile-only object/library recipes deliberately do not use
* the link half. In particular, `--lib` preserves its established object and
* archive contract; section GC is an executable-link optimization only.
*/
export function executableSectionEliminationFlags(platform: string): {
compile: string[];
link: string[];
} {
switch (platform) {
case "darwin":
// ld64's symbol subsections make this sufficient for ordinary C/LLVM
// objects. Do not attach it only to --dynamic: static programs have the
// same unreachable runtime sections.
return { compile: [], link: ["-Wl,-dead_strip"] };
case "linux":
return {
compile: ["-ffunction-sections", "-fdata-sections"],
link: ["-Wl,--gc-sections"],
};
case "win32":
// clang's MinGW driver forwards this to GNU-flavor ld/lld. A direct
// local lld-link invocation instead uses /OPT:REF, but scriptc only
// drives the compiler's GNU-flavor route here.
return {
compile: ["-ffunction-sections", "-fdata-sections"],
link: ["-Wl,--gc-sections"],
};
// WASI has a distinct linker/runtime contract. Keep its existing object
// layout until its linker invocation is validated separately.
default:
return { compile: [], link: [] };
}
}
/** Environment variables consumed by clang, its linker/subtools, or the
* platform SDK selection. They are implicit command-line inputs: changing one
* must never reuse an artifact produced under the old toolchain posture. */
@@ -3649,15 +3689,21 @@ function localArtifactIdentity(
)
.sort(([a], [b]) => a.localeCompare(b)),
);
const executableSectionFlags = executableSectionEliminationFlags(targetPlatform(driver));
const hash = createHash("sha256")
.update("local-artifact-v1\0")
// v2 adds the executable section-GC recipe. A v1 output-local stamp can
// name a same-source binary from before unused runtime sections were
// eliminated, so it must never short-circuit the new linker invocation.
.update("local-artifact-v2\0")
.update(cacheTargetIdentity(driver)).update("\0")
.update(environmentFingerprint).update("\0")
.update(compilerIdentity).update("\0")
.update(runtimeHash).update("\0")
.update(driver.argv.join("\x1f")).update("\0")
.update(driver.targetArgs.join("\x1f")).update("\0")
.update(driver.linkArgs.join("\x1f")).update("\0");
.update(driver.linkArgs.join("\x1f")).update("\0")
.update(executableSectionFlags.compile.join("\x1f")).update("\0")
.update(executableSectionFlags.link.join("\x1f")).update("\0");
if (programShardMerge !== null) {
updateProgramShardCacheIdentity(
hash,
@@ -3981,6 +4027,7 @@ async function compileCInternal(
const runtimeSources = targetPlatform(driver) === "wasi"
? EXECUTABLE_RUNTIME_SOURCES.filter((source) => source !== "scr_child.c")
: EXECUTABLE_RUNTIME_SOURCES;
const executableSectionFlags = executableSectionEliminationFlags(targetPlatform(driver));
// scr_async.c submits callback-style filesystem work to a native worker.
// POSIX drivers need the thread compile/link mode; win32 uses CreateThread.
const threadArgs = targetPlatform(driver) === "win32" || targetPlatform(driver) === "wasi"
@@ -4343,6 +4390,7 @@ async function compileCInternal(
...(sanitize
? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
: [optimization === "dev" ? "-O0" : "-O2"]),
...executableSectionFlags.compile,
...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
"-fno-math-errno",
// The emitted object model is deliberately type-punned C: a hierarchy
@@ -4483,11 +4531,6 @@ async function compileCInternal(
// left-to-right archive resolution. Other dynamic targets keep
// the historical engine-adjacent spelling.
...(driver.linkArgs.includes("-lm") ? [] : ["-lm"]),
// ld64 dead-stripping claws back a chunk of the engine archive;
// harmless elsewhere but only spelled this way on macOS. Keyed on
// the TARGET platform (= the host on the default path, where this
// expression is byte-identical to the historical one).
...(targetPlatform(driver) === "darwin" ? ["-Wl,-dead_strip"] : []),
// The PE stack reserve, pinned to the 8MB POSIX main-stack
// geometry ISL_MAIN_STACK_BUDGET is sized against (4MB engine
// budget + 4MB excursion margin) — quickjs-ng's own CMake makes
@@ -4525,6 +4568,7 @@ async function compileCInternal(
// program and every native FFI input because GNU ld resolves archives
// from left to right.
...driver.linkArgs,
...executableSectionFlags.link,
"-o", build.outPath ?? opts.outPath,
];
// Compile-only flags shared by runtime-object population and the caller-TU
@@ -4537,6 +4581,7 @@ async function compileCInternal(
...(sanitize
? ["-O1", "-fsanitize=address", "-DSCR_RC_AUDIT"]
: [optimization === "dev" ? "-O0" : "-O2"]),
...executableSectionFlags.compile,
...(opts.textDecoderLegacy ? ["-DSCR_TEXT_DECODER_LEGACY"] : []),
"-fno-math-errno",
"-fno-strict-aliasing", // the emitted object model type-puns — see buildArgs
@@ -4721,6 +4766,7 @@ async function compileCInternal(
...(dynamic && !driver.linkArgs.includes("-lm") ? ["-lm"] : []),
...(((opts.zlib ?? false) || nativeFetch) && driver.target === null ? ["-lz"] : []),
...driver.linkArgs,
...executableSectionFlags.link,
];
// Both the wrapper dry run and dependency trace need the real build's
// compile/link flag shape: wrappers commonly inject flags or native inputs
@@ -4735,12 +4781,12 @@ async function compileCInternal(
...(curlStubDir !== null ? [`-L${curlStubDir}`] : []),
...(curlFetch && driver.target === null ? ["-lcurl"] : []),
...(dynamic && !driver.linkArgs.includes("-lm") ? ["-lm"] : []),
...(dynamic && targetPlatform(driver) === "darwin" ? ["-Wl,-dead_strip"] : []),
...(dynamic && targetPlatform(driver) === "win32"
? ["-Wl,--stack,8388608"]
: []),
...(((opts.zlib ?? false) || nativeFetch) && driver.target === null ? ["-lz"] : []),
...driver.linkArgs,
...executableSectionFlags.link,
];
// A complete hit is checked before cross-target curl's generated import stub
// is materialized. Its -L spelling still joins the dry-run identity, while
@@ -137,6 +137,20 @@ describe("runtime pack manifests", () => {
expect(evaluateRuntimePredicate({ any: ["regex", "dynamic"] }, features)).toBe(true);
});
test("static runtime-pack executable links dead-strip too", async () => {
const { packagePath, root } = await fixture();
const plan = await createRuntimeLinkPlan({
target: MACOS_ARM64_TARGET,
programObject: join(root, "program.o"),
outPath: join(root, "program"),
features: BASE,
ffi: null,
optimization: "release",
resolver: () => packagePath,
});
expect(plan.driverFlags).toContain("-Wl,-dead_strip");
});
test("selection chooses the most-specific variant and feature archive", async () => {
const { packagePath } = await fixture();
const resolver = () => packagePath;
@@ -10,6 +10,7 @@ import { compilerReleaseVersion } from "../library/sidecar.js";
import type { NativeLinkFeatures } from "./native-link-info.js";
import {
CcCompileError,
executableSectionEliminationFlags,
nativeArtifactDependenciesStillMatch,
nativeLinkerDependencyPaths,
subprocessFailureDetail,
@@ -434,7 +435,7 @@ export async function createRuntimeLinkPlan(options: {
])],
driverFlags: [
"-target", options.target.llvmTriple, "-pthread",
...(runtimePack.features.dynamic ? ["-Wl,-dead_strip"] : []),
...executableSectionEliminationFlags("darwin").link,
],
dependencyPaths: [
...runtimePack.dependencyPaths,
@@ -16,6 +16,14 @@ const BASE_RUNTIME_SOURCES = [
"scr_async.c", "scr_child.c", "scr_cycle.c",
];
// ld64 dead-strips Mach-O symbol subsections without an ELF-style compile
// flag. Keep this checked-in metadata alongside the matrix so pack build and
// source/external linking share one documented executable recipe.
export const EXECUTABLE_SECTION_ELIMINATION = {
compile_flags: [],
link_flags: ["-Wl,-dead_strip"],
};
const optional = [
["scr_copying.c", "copying"],
["scr_file_handle.c", "fileHandle"],
@@ -95,6 +103,7 @@ export const RUNTIME_PACK_MATRIX = {
release: { optimization: "-O2" },
dev: { optimization: "-O0" },
},
executable_section_elimination: EXECUTABLE_SECTION_ELIMINATION,
runtime_units: [
...BASE_RUNTIME_SOURCES.map((source) => ({ source, predicate: true })),
...optional.map(([source, predicate]) => ({ source, predicate })),
@@ -47,6 +47,7 @@ async function build() {
const commonFlags = [
"-target", RUNTIME_PACK_MATRIX.target.llvm_triple,
"-std=c11", "-pthread", "-fno-math-errno", "-fno-strict-aliasing",
...RUNTIME_PACK_MATRIX.executable_section_elimination.compile_flags,
"-Wno-deprecated-declarations", "-I", runtimeSrc,
];
const quickjs = join(vendorRoot, "quickjs-ng");
+200
View File
@@ -0,0 +1,200 @@
/* Executable runtime reachability. The source-runtime recipe intentionally
* still includes the complete historical base for direct compileC callers;
* executable section GC is what makes its unused functions/data disappear.
* These fixtures therefore pin both halves of that contract: hello has no
* reachable members from the formerly-unavoidable families, while every
* feature fixture retains an anchor and behaves exactly like Node. */
import { execFile } from "node:child_process";
import { createServer } from "node:http";
import { mkdirSync, statSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { promisify } from "node:util";
import { expect, test } from "vitest";
import { compile } from "@scriptc/compiler";
const execFileAsync = promisify(execFile);
const repoRoot = join(import.meta.dirname, "../..");
const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests/runtime-tree-shaking");
const portableTest = process.env["SCRIPTC_PORTABLE_ONLY"] === "1" ? test.skip : test;
interface Fixture {
name: string;
source: string;
anchor: string;
}
const FIXTURES: Fixture[] = [
{
name: "child",
source: `import { spawnSync } from "node:child_process";
const r = spawnSync("/bin/echo", ["child"], { encoding: "utf8" });
console.log(r.stdout.trim(), r.status);
`,
anchor: "scr_spawn_sync",
},
{
name: "path-posix",
source: `import { join } from "node:path/posix";
console.log(join("a", "..", "b"));
`,
anchor: "scr_path_join",
},
{
name: "path-win32",
source: `import { join } from "node:path/win32";
console.log(join("C:\\\\a", "..", "b"));
`,
anchor: "scr_path_win32_join",
},
{
name: "url",
source: `import { fileURLToPath, pathToFileURL } from "node:url";
console.log(fileURLToPath(pathToFileURL("/tmp/a b")));
`,
// This composition is optimized through the URL bridge, so its direct
// anchors are the file-path conversion helpers rather than URL parsing.
anchor: "scr_url_from_path",
},
{
name: "json-parse",
source: `console.log(JSON.parse('{"ok":true}').ok);
`,
anchor: "scr_json_parse",
},
{
name: "date",
source: `console.log(new Date(0).toISOString());
`,
anchor: "scr_date_to_iso",
},
{
name: "stream-consumers-json",
source: `import { Readable } from "node:stream";
import { json } from "node:stream/consumers";
const stream = new Readable({ read() {} });
stream.push('{"value":7}');
stream.push(null);
console.log((await json(stream)).value);
`,
anchor: "scr_json_parse",
},
];
async function build(name: string, source: string) {
const outDir = join(cacheDir, name);
const sourcePath = join(outDir, "main.mjs");
mkdirSync(outDir, { recursive: true });
writeFileSync(sourcePath, source);
const result = await compile(sourcePath, {
outPath: join(outDir, "program"),
outDir,
// The C lane proves source-toolchain linking. macOS additionally runs
// the default lane below, which selects the helper/runtime-pack path.
backend: "c",
});
if (!result.ok) {
throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"));
}
return { sourcePath, binaryPath: result.binaryPath };
}
async function output(command: string, args: string[]): Promise<string> {
return (await execFileAsync(command, args, { encoding: "utf8" })).stdout;
}
async function symbols(binaryPath: string): Promise<string> {
// `nm` is supplied by Xcode/binutils on the supported native lanes. Global
// executable symbols are the meaningful ABI/reachability contract: local
// compiler bookkeeping may legitimately retain a similarly named datum.
return output("nm", ["-g", binaryPath]);
}
async function expectNodeParity(sourcePath: string, binaryPath: string, args: string[] = []): Promise<void> {
const [node, native] = await Promise.all([
output(process.execPath, [sourcePath, ...args]),
output(binaryPath, args),
]);
expect(native).toBe(node);
}
portableTest("static hello strips unreachable runtime families while feature programs retain them", async () => {
const helloSource = `console.log("hello", "world");\n`;
const hello = await build("hello", helloSource);
await expectNodeParity(hello.sourcePath, hello.binaryPath);
const helloSymbols = await symbols(hello.binaryPath);
for (const family of [
"scr_path_win32_",
"scr_exec_",
"scr_url_",
"scr_json_parse",
"scr_date_",
]) {
expect(helloSymbols, `hello retains ${family}`).not.toContain(family);
}
const built = [] as { fixture: Fixture; binaryPath: string }[];
for (const fixture of FIXTURES) {
// /bin/echo is the portable POSIX child fixture. The Windows child
// surface remains covered by its cross-target corpus contracts.
if (fixture.name === "child" && process.platform === "win32") continue;
const result = await build(fixture.name, fixture.source);
await expectNodeParity(result.sourcePath, result.binaryPath);
const nativeSymbols = await symbols(result.binaryPath);
expect(nativeSymbols, `${fixture.name} lost ${fixture.anchor}`).toContain(fixture.anchor);
built.push({ fixture, binaryPath: result.binaryPath });
}
// Page-sized slack makes this a directional regression check rather than a
// linker-version byte-count pin. Child is a representative formerly-base
// payload and must remain materially larger than a no-feature executable.
const child = built.find(({ fixture }) => fixture.name === "child");
if (child !== undefined) {
expect(statSync(hello.binaryPath).size + 16 * 1024).toBeLessThan(statSync(child.binaryPath).size);
}
});
portableTest("fetch response JSON retains the URL and parser runtime", async () => {
const server = createServer((_request, response) => {
response.setHeader("content-type", "application/json");
response.end('{"ok":true}');
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
try {
const address = server.address();
if (address === null || typeof address === "string") throw new Error("test server has no TCP address");
const fixture: Fixture = {
name: "fetch-response-json",
source: `const response = await fetch(process.argv[2]);
console.log((await response.json()).ok);
`,
anchor: "scr_json_parse",
};
const result = await build(fixture.name, fixture.source);
await expectNodeParity(result.sourcePath, result.binaryPath, [`http://127.0.0.1:${address.port}`]);
const nativeSymbols = await symbols(result.binaryPath);
expect(nativeSymbols).toContain("scr_json_parse");
expect(nativeSymbols).toContain("scr_url_release");
} finally {
await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
}
});
// The helper emits a native program object only on supported macOS arm64.
// Its normal backend path links the precompiled runtime pack; run the same
// reachability assertion there so the source-only C lane cannot regress it.
test.skipIf(process.platform !== "darwin" || process.arch !== "arm64")(
"macOS helper/runtime-pack links dead-strip static hello too",
async () => {
const outDir = join(cacheDir, "hello-runtime-pack");
const sourcePath = join(outDir, "main.mjs");
mkdirSync(outDir, { recursive: true });
writeFileSync(sourcePath, `console.log("hello", "world");\n`);
const result = await compile(sourcePath, { outPath: join(outDir, "program"), outDir });
if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n"));
await expectNodeParity(sourcePath, result.binaryPath);
const nativeSymbols = await symbols(result.binaryPath);
for (const family of ["scr_path_win32_", "scr_exec_", "scr_url_", "scr_json_parse", "scr_date_"]) {
expect(nativeSymbols, `runtime pack retained ${family}`).not.toContain(family);
}
},
);