Stop building and shipping runtime import bitcode (#2583)

This commit is contained in:
Chris Tate
2026-10-30 13:57:51 -05:00
committed by GitHub
15 changed files with 158 additions and 249 deletions
+4 -4
View File
@@ -49,19 +49,19 @@ Runtime-pack builds emit each unit of the executable `speed` flavor through the
```console ```console
scriptc-llvm-codegen runtime-unit --input unit.opt.bc --object unit.o \ scriptc-llvm-codegen runtime-unit --input unit.opt.bc --object unit.o \
--bitcode unit.bc --target x86_64-unknown-linux-gnu --tag scr_array \ --target x86_64-unknown-linux-gnu --tag scr_array \
[--function-sections --data-sections] [--function-sections --data-sections]
``` ```
The input is the runtime compiler's optimized bitcode. The command promotes unit-local symbols to hidden `<name>.scrunit.<tag>` globals and writes the object and the import bitcode from that one module, so imported bodies can reference the unit's private state and helpers. The input is the runtime compiler's bitcode for the unit. The command promotes unit-local symbols to hidden `<name>.scrunit.<tag>` globals and writes the object. With `--bitcode unit.bc` it also writes import bitcode from the same module, so imported bodies can reference the unit's private state and helpers. Runtime packs ship no bitcode, since program builds don't import it.
Release units keep the runtime compiler's own objects. Their import bitcode is the same compilation with `-emit-llvm`, reduced by the helper: `prune-import` reduces a unit's `-emit-llvm` bitcode to import bitcode without promotion:
```console ```console
scriptc-llvm-codegen prune-import --input unit.opt.bc --output unit.bc scriptc-llvm-codegen prune-import --input unit.opt.bc --output unit.bc
``` ```
Both commands keep only what a program build can import: the importable bodies within the size limit at their call distance from any public function, and the constants they copy. Every other definition becomes a declaration, so a program imports exactly what it would from the whole unit while reading a fraction of it. Unpromoted unit-local symbols stay local, so release bitcode offers only bodies that need none of them. Import bitcode keeps only what a program build can import: the importable bodies within the size limit at their call distance from any public function, and the constants they copy. Every other definition becomes a declaration, so a program imports exactly what it would from the whole unit while reading a fraction of it. Unpromoted unit-local symbols stay local, so `prune-import` bitcode offers only bodies that need none of them.
## Linux release builds ## Linux release builds
+2 -1
View File
@@ -60,7 +60,8 @@ int version(int Argc, char **Argv) {
{"data_layout", Machine->createDataLayout().getStringRepresentation()}, {"data_layout", Machine->createDataLayout().getStringRepresentation()},
// Commands and options added within one protocol version, for callers // Commands and options added within one protocol version, for callers
// that also run against older helpers. // that also run against older helpers.
{"features", json::Array{"runtime-unit-profile", "profile-functions"}}, {"features", json::Array{"runtime-unit-profile", "profile-functions",
"runtime-unit-optional-bitcode"}},
}; };
outs() << formatv("{0}\n", json::Value(std::move(Response))); outs() << formatv("{0}\n", json::Value(std::move(Response)));
return 0; return 0;
+17 -14
View File
@@ -1,6 +1,7 @@
// `runtime-unit`: turn one optimized runtime translation unit into the pair // `runtime-unit`: turn one optimized runtime translation unit into the
// of artifacts a runtime pack ships for it: the native object and the // native object a runtime pack ships for it and, with --bitcode, the pruned
// bitcode the program build imports from (see runtime-import.cpp). // bitcode a build could import from (see runtime-import.cpp). Runtime packs
// ship no bitcode since program builds stopped importing it.
// //
// Both come from one module. Every translation-unit-local symbol that an // Both come from one module. Every translation-unit-local symbol that an
// imported body could need (functions, mutable state, address-significant // imported body could need (functions, mutable state, address-significant
@@ -106,7 +107,7 @@ std::optional<UnitOptions> parse(int Argc, char **Argv) {
return std::nullopt; return std::nullopt;
} }
if (Options.Input.empty() || Options.Object.empty() || if (Options.Input.empty() || Options.Object.empty() ||
Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty()) Options.Target.empty() || Options.Tag.empty())
return std::nullopt; return std::nullopt;
// A profile only guides an optimization the helper runs, and a unit is // A profile only guides an optimization the helper runs, and a unit is
// either instrumented or annotated. // either instrumented or annotated.
@@ -196,8 +197,8 @@ int runtimeUnit(int Argc, char **Argv) {
std::optional<UnitOptions> Options = parse(Argc, Argv); std::optional<UnitOptions> Options = parse(Argc, Argv);
if (!Options) if (!Options)
return fail("usage", return fail("usage",
"runtime-unit requires --input <bc> --object <o> --bitcode <bc> " "runtime-unit requires --input <bc> --object <o> "
"--target <triple> --tag <unit> and accepts " "--target <triple> --tag <unit> and accepts --bitcode <bc>, "
"--function-sections --data-sections, and --opt-level <level> " "--function-sections --data-sections, and --opt-level <level> "
"with either --profile-generate <pattern> or --profile-use " "with either --profile-generate <pattern> or --profile-use "
"<profdata>"); "<profdata>");
@@ -306,14 +307,16 @@ int runtimeUnit(int Argc, char **Argv) {
// The bitcode exists only to be imported from, so it carries only what // The bitcode exists only to be imported from, so it carries only what
// an import can take; program builds then read a fraction of the unit. // an import can take; program builds then read a fraction of the unit.
std::unique_ptr<Module> Import = CloneModule(*M); if (!Options->Bitcode.empty()) {
pruneRuntimeImport(*Import); std::unique_ptr<Module> Import = CloneModule(*M);
if (verifyModule(*Import, &VerifyStream)) pruneRuntimeImport(*Import);
return fail("verification_failed", VerifyStream.str()); if (verifyModule(*Import, &VerifyStream))
if (std::optional<std::string> Failure = writeFileAtomically( return fail("verification_failed", VerifyStream.str());
Options->Bitcode, if (std::optional<std::string> Failure = writeFileAtomically(
[&](raw_fd_ostream &Out) { WriteBitcodeToFile(*Import, Out); })) Options->Bitcode,
return fail("output_write_failed", *Failure); [&](raw_fd_ostream &Out) { WriteBitcodeToFile(*Import, Out); }))
return fail("output_write_failed", *Failure);
}
bool Unsupported = false; bool Unsupported = false;
std::optional<std::string> Failure = std::optional<std::string> Failure =
@@ -23,8 +23,6 @@ export interface RuntimePackVariant extends RuntimePackArtifact {
id: string; id: string;
when: Record<string, boolean>; when: Record<string, boolean>;
defines: string[]; defines: string[];
/** LLVM bitcode of the same unit, imported by the helper for inlining. */
bitcode?: RuntimePackArtifact;
} }
interface RuntimePackUnit { interface RuntimePackUnit {
@@ -212,7 +210,6 @@ export function parseRuntimePackManifest(value: unknown): RuntimePackManifest {
const when = object(variant?.when); const when = object(variant?.when);
return ( return (
validArtifact(variantRaw) && validArtifact(variantRaw) &&
(variant?.bitcode === undefined || validArtifact(variant.bitcode)) &&
typeof variant?.id === "string" && typeof variant?.id === "string" &&
when !== null && when !== null &&
Object.values(when).every((entry) => typeof entry === "boolean") && Object.values(when).every((entry) => typeof entry === "boolean") &&
@@ -426,8 +423,8 @@ export function validateRuntimePackIdentity(
export interface RuntimePackArtifacts { export interface RuntimePackArtifacts {
features: RuntimeFeatureSet; features: RuntimeFeatureSet;
/** Selected unit variants; each may carry its bitcode artifact. */ /** Selected unit variants. */
runtime: (RuntimePackArtifact & { bitcode?: RuntimePackArtifact })[]; runtime: RuntimePackArtifact[];
archives: RuntimePackArtifact[]; archives: RuntimePackArtifact[];
/** Files to install beside the executable (ICU's data file for Intl). */ /** Files to install beside the executable (ICU's data file for Intl). */
data: RuntimePackArtifact[]; data: RuntimePackArtifact[];
@@ -321,17 +321,6 @@ describe("runtime pack manifests", () => {
expect(fileHandle.archives.map((path) => path.split("/").at(-1))).toEqual(["zlib.a"]); expect(fileHandle.archives.map((path) => path.split("/").at(-1))).toEqual(["zlib.a"]);
}); });
test("runtime unit bitcode must stay inside the pack", async () => {
const { manifest } = await fixture();
const malformed = structuredClone(manifest);
malformed.flavors.release!.runtime_units[0]!.variants[0]!.bitcode = {
path: "../escape.bc",
sha256: "0".repeat(64),
size: 1,
};
expect(() => parseRuntimePackManifest(malformed)).toThrow("malformed");
});
test("a stale manifest reports its identity independently from the package", async () => { test("a stale manifest reports its identity independently from the package", async () => {
const { packagePath, manifest, root } = await fixture(); const { packagePath, manifest, root } = await fixture();
await writeFile( await writeFile(
@@ -9,7 +9,6 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util"; import { promisify } from "node:util";
import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs"; import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs";
import { import {
buildImportBitcode,
buildRuntimeUnit, buildRuntimeUnit,
resolveRuntimeUnitHelper, resolveRuntimeUnitHelper,
} from "../../runtime-pack-common/scripts/runtime-unit.mjs"; } from "../../runtime-pack-common/scripts/runtime-unit.mjs";
@@ -80,8 +79,8 @@ async function build() {
const ZLIB_BUILD = zlibInputs(RUNTIME_PACK_MATRIX.target.llvm_triple); const ZLIB_BUILD = zlibInputs(RUNTIME_PACK_MATRIX.target.llvm_triple);
// Speed-flavor executable units are emitted through the host LLVM helper // Speed-flavor executable units are emitted through the host LLVM helper
// so their object and import bitcode come from one promoted module. The // (runtime-unit.mjs), which promotes their local symbols and applies the
// helper also reduces release units' bitcode to what programs can import. // runtime profile.
const unitHelper = await resolveRuntimeUnitHelper( const unitHelper = await resolveRuntimeUnitHelper(
repoRoot, repoRoot,
RUNTIME_PACK_MATRIX.target.llvm_triple, RUNTIME_PACK_MATRIX.target.llvm_triple,
@@ -137,7 +136,7 @@ async function build() {
const flavors = {}; const flavors = {};
// The executable `speed` flavor (--optimization=speed) exists only when // The executable `speed` flavor (--optimization=speed) exists only when
// it differs from release: its static units are emitted through the host // it differs from release: its static units are emitted through the host
// LLVM helper with import bitcode. // LLVM helper.
const flavorPlan = [ const flavorPlan = [
...Object.entries(RUNTIME_PACK_MATRIX.flavors), ...Object.entries(RUNTIME_PACK_MATRIX.flavors),
...("release" in RUNTIME_PACK_MATRIX.flavors && unitHelper !== null ...("release" in RUNTIME_PACK_MATRIX.flavors && unitHelper !== null
@@ -183,8 +182,6 @@ async function build() {
...variant.defines.map((define) => `-D${define}`), ...variant.defines.map((define) => `-D${define}`),
...includeFlags, ...includeFlags,
]; ];
let bitcode;
const bitcodeOutput = output.replace(/\.o$/, ".bc");
if (flavor === "speed") await mkdir(dirname(output), { recursive: true }); if (flavor === "speed") await mkdir(dirname(output), { recursive: true });
if ( if (
flavor === "speed" && flavor === "speed" &&
@@ -201,7 +198,6 @@ async function build() {
optimized, optimized,
]), ]),
object: output, object: output,
bitcode: bitcodeOutput,
triple: RUNTIME_PACK_MATRIX.target.llvm_triple, triple: RUNTIME_PACK_MATRIX.target.llvm_triple,
tag: unit.source.replace(/\.c$/, ""), tag: unit.source.replace(/\.c$/, ""),
sections: sections:
@@ -210,38 +206,8 @@ async function build() {
), ),
})) }))
) { ) {
bitcode = {
path: artifactPath(bitcodeOutput),
sha256: await sha256(bitcodeOutput),
size: (await stat(bitcodeOutput)).size,
};
} else { } else {
await compile(join(runtimeSrc, unit.source), output, unitFlags); await compile(join(runtimeSrc, unit.source), output, unitFlags);
if (
flavor === "release" &&
unitHelper !== null &&
(await buildImportBitcode({
helper: unitHelper,
compileBitcode: (optimized) =>
run(compiler, [
...sourcePathFlags,
...unitFlags,
"-emit-llvm",
"-c",
join(runtimeSrc, unit.source),
"-o",
optimized,
]),
bitcode: bitcodeOutput,
tag: unit.source.replace(/\.c$/, ""),
}))
) {
bitcode = {
path: artifactPath(bitcodeOutput),
sha256: await sha256(bitcodeOutput),
size: (await stat(bitcodeOutput)).size,
};
}
} }
const built = { const built = {
id: variant.id, id: variant.id,
@@ -250,7 +216,6 @@ async function build() {
path: artifactPath(output), path: artifactPath(output),
sha256: await sha256(output), sha256: await sha256(output),
size: (await stat(output)).size, size: (await stat(output)).size,
...(bitcode === undefined ? {} : { bitcode }),
}; };
variants.push(built); variants.push(built);
} }
@@ -16,11 +16,7 @@ if (
throw new Error("runtime pack identity does not match package.json"); throw new Error("runtime pack identity does not match package.json");
const artifacts = [ const artifacts = [
...Object.values(manifest.flavors).flatMap((flavor) => ...Object.values(manifest.flavors).flatMap((flavor) =>
flavor.runtime_units.flatMap((unit) => flavor.runtime_units.flatMap((unit) => unit.variants),
unit.variants.flatMap((variant) =>
variant.bitcode ? [variant, variant.bitcode] : [variant],
),
),
), ),
...manifest.archives, ...manifest.archives,
]; ];
+4 -40
View File
@@ -15,7 +15,7 @@ import {
runtimeForbiddenStrings, runtimeForbiddenStrings,
} from "./artifact-policy.mjs"; } from "./artifact-policy.mjs";
import { createDeterministicArchive } from "./archive.mjs"; import { createDeterministicArchive } from "./archive.mjs";
import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs"; import { buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
import { installRuntimePack, withBuildLock } from "./build-state.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs";
import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
import { parallelMap } from "./parallel-map.mjs"; import { parallelMap } from "./parallel-map.mjs";
@@ -141,8 +141,8 @@ async function build() {
]); ]);
}; };
// Speed-flavor executable units are emitted through the host LLVM helper // Speed-flavor executable units are emitted through the host LLVM helper
// so their object and import bitcode come from one promoted module. The // (runtime-unit.mjs), which promotes their local symbols and applies the
// helper also reduces release units' bitcode to what programs can import. // runtime profile.
const unitHelper = const unitHelper =
config.bitcode === false config.bitcode === false
? null ? null
@@ -379,7 +379,7 @@ async function build() {
const flavors = {}; const flavors = {};
// The executable `speed` flavor (--optimization=speed) exists only when // The executable `speed` flavor (--optimization=speed) exists only when
// it differs from release: its static units are emitted through the host // it differs from release: its static units are emitted through the host
// LLVM helper with import bitcode. // LLVM helper.
const flavorPlan = [ const flavorPlan = [
...Object.entries(matrix.flavors), ...Object.entries(matrix.flavors),
...("release" in matrix.flavors && unitHelper !== null ...("release" in matrix.flavors && unitHelper !== null
@@ -463,8 +463,6 @@ async function build() {
...tlsFlags, ...tlsFlags,
...includeFlags, ...includeFlags,
]; ];
let bitcode;
const bitcodeOutput = output.replace(/\.o$/, ".bc");
if (flavor === "speed") await mkdir(dirname(output), { recursive: true }); if (flavor === "speed") await mkdir(dirname(output), { recursive: true });
let unitBuild = false; let unitBuild = false;
if ( if (
@@ -489,7 +487,6 @@ async function build() {
? ["--opt-level", "2"] ? ["--opt-level", "2"]
: (speedProfile?.args ?? []), : (speedProfile?.args ?? []),
object: output, object: output,
bitcode: bitcodeOutput,
triple: matrix.target.llvm_triple, triple: matrix.target.llvm_triple,
tag: unit.source.replace(/\.c$/, ""), tag: unit.source.replace(/\.c$/, ""),
sections: sections:
@@ -500,40 +497,8 @@ async function build() {
) { ) {
if (unitBuild.profile !== undefined) profileMatches.push(unitBuild.profile); if (unitBuild.profile !== undefined) profileMatches.push(unitBuild.profile);
for (const name of unitBuild.functions ?? []) profiledFunctions.add(name); for (const name of unitBuild.functions ?? []) profiledFunctions.add(name);
bitcode = {
path: artifactPath(bitcodeOutput),
sha256: await sha256(bitcodeOutput),
size: (await stat(bitcodeOutput)).size,
};
} else { } else {
await compile(join(runtimeSrc, unit.source), output, unitFlags); await compile(join(runtimeSrc, unit.source), output, unitFlags);
if (
flavor === "release" &&
unitHelper !== null &&
(await buildImportBitcode({
helper: unitHelper,
compileBitcode: (optimized) =>
run(compiler, [
...compilerArgs,
...sourcePathFlags,
...unitFlags,
"-g0",
"-emit-llvm",
"-c",
join(runtimeSrc, unit.source),
"-o",
optimized,
]),
bitcode: bitcodeOutput,
tag: unit.source.replace(/\.c$/, ""),
}))
) {
bitcode = {
path: artifactPath(bitcodeOutput),
sha256: await sha256(bitcodeOutput),
size: (await stat(bitcodeOutput)).size,
};
}
} }
const built = { const built = {
id: variant.id, id: variant.id,
@@ -542,7 +507,6 @@ async function build() {
path: artifactPath(output), path: artifactPath(output),
sha256: await sha256(output), sha256: await sha256(output),
size: (await stat(output)).size, size: (await stat(output)).size,
...(bitcode === undefined ? {} : { bitcode }),
}; };
variants.push(built); variants.push(built);
} }
@@ -5,26 +5,18 @@ import { promisify } from "node:util";
const run = promisify(execFile); const run = promisify(execFile);
/* Executable runtime units ship the bitcode the LLVM helper imports small /* Speed units are emitted through the host LLVM helper: the runtime
* runtime functions from (available_externally, so program code can inline * compiler compiles the C source to bitcode, and the helper's `runtime-unit`
* them while the unit's object stays the only linked definition). * command promotes translation-unit-local symbols to hidden unit-qualified
* * globals and emits the object. With the target's committed runtime profile,
* Speed units produce both artifacts from one optimized LLVM module: the * the runtime compiler emits the bitcode unoptimized and the helper also
* runtime compiler optimizes the C source to bitcode, and the host helper's * optimizes it with the profile (build.mjs), so one LLVM reads and writes
* `runtime-unit` command promotes translation-unit-local symbols to hidden * every profile. Runtime packs ship objects only: program builds do not
* unit-qualified globals and emits the object and the bitcode. With the * import runtime bitcode.
* target's committed runtime profile, the runtime compiler emits the bitcode
* unoptimized and the helper also optimizes it with the profile (build.mjs),
* so one LLVM reads and writes every profile. Release units keep the
* runtime compiler's own objects; their bitcode is the same compilation with
* -emit-llvm, which the helper's `prune-import` reduces to the bodies that
* need none of the unit's local state. Both bitcode forms
* keep only what a program build can import.
* *
* Every host helper supports every scriptc target, so packs keep building * Every host helper supports every scriptc target, so packs keep building
* on their usual hosts. Without a runnable host helper (or with * on their usual hosts. Without a runnable host helper (or with
* SCRIPTC_RUNTIME_BITCODE=0) units are compiled to objects directly and * SCRIPTC_RUNTIME_BITCODE=0) a pack has no speed flavor. */
* ship no bitcode; program builds then simply import nothing. */
function hostHelperPackage() { function hostHelperPackage() {
if (process.platform === "darwin") return `llvm-darwin-${process.arch}`; if (process.platform === "darwin") return `llvm-darwin-${process.arch}`;
@@ -63,7 +55,8 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
// Commands and options newer than the protocol version, such as // Commands and options newer than the protocol version, such as
// "runtime-unit-profile": `runtime-unit` takes --opt-level, // "runtime-unit-profile": `runtime-unit` takes --opt-level,
// --profile-use and --profile-generate (older helpers only promote and // --profile-use and --profile-generate (older helpers only promote and
// emit). // emit); "runtime-unit-optional-bitcode": it emits the object without
// the import bitcode.
features: Array.isArray(version.features) ? version.features : [], features: Array.isArray(version.features) ? version.features : [],
}; };
} catch { } catch {
@@ -72,14 +65,31 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
} }
} }
/** The helper's diagnostic from a failed command: its JSON line, which
* follows any LLVM warnings on the same streams. */
function helperFailure(error) {
const lines = [error.stdout, error.stderr]
.filter((text) => typeof text === "string")
.join("\n")
.split("\n")
.map((line) => line.trim())
.filter((line) => line !== "");
return (
lines.find((line) => line.startsWith("{")) ??
lines.find((line) => !line.includes("is not a recognized feature")) ??
lines[0] ??
error.message ??
String(error)
);
}
/** Compile one unit through the helper. `compileBitcode(output)` must run /** Compile one unit through the helper. `compileBitcode(output)` must run
* the runtime compiler with the unit's exact flags plus -emit-llvm. * the runtime compiler with the unit's exact flags plus -emit-llvm.
* `helperArgs` asks the helper to optimize the unit itself (`--opt-level`, * `helperArgs` asks the helper to optimize the unit itself (`--opt-level`,
* with `--profile-use` or `--profile-generate`), for bitcode the runtime * with `--profile-use` or `--profile-generate`), for bitcode the runtime
* compiler emitted unoptimized. * compiler emitted unoptimized.
* Returns false, leaving no artifacts behind, when the helper cannot emit * Returns false, leaving no artifacts behind, when the helper cannot emit
* this unit; the caller then compiles the object directly without bitcode, * this unit; the caller then compiles the object directly. Otherwise
* so a pack never fails to build because of the import path. Otherwise
* returns an object, with the helper's `profile` match counts when a profile * returns an object, with the helper's `profile` match counts when a profile
* guided the unit, or the profile names of the unit's `functions` when it * guided the unit, or the profile names of the unit's `functions` when it
* was instrumented. */ * was instrumented. */
@@ -88,12 +98,15 @@ export async function buildRuntimeUnit({
compileBitcode, compileBitcode,
helperArgs = [], helperArgs = [],
object, object,
bitcode,
triple, triple,
tag, tag,
sections, sections,
}) { }) {
const optimized = object.replace(/\.o$/, ".opt.bc"); const optimized = object.replace(/\.o$/, ".opt.bc");
// Helpers older than "runtime-unit-optional-bitcode" always write it.
const bitcode = helper.features.includes("runtime-unit-optional-bitcode")
? null
: object.replace(/\.o$/, ".import.bc");
// A runtime compiler failure is a real build error and surfaces as one. // A runtime compiler failure is a real build error and surfaces as one.
await compileBitcode(optimized); await compileBitcode(optimized);
try { try {
@@ -103,8 +116,7 @@ export async function buildRuntimeUnit({
optimized, optimized,
"--object", "--object",
object, object,
"--bitcode", ...(bitcode === null ? [] : ["--bitcode", bitcode]),
bitcode,
"--target", "--target",
triple, triple,
"--tag", "--tag",
@@ -118,31 +130,13 @@ export async function buildRuntimeUnit({
...(report.functions === undefined ? {} : { functions: report.functions }), ...(report.functions === undefined ? {} : { functions: report.functions }),
}; };
} catch (error) { } catch (error) {
const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0]; process.stderr.write(
process.stderr.write(`runtime-unit: ${tag}: emitting without import bitcode (${detail})\n`); `runtime-unit: ${tag}: compiling the object directly (${helperFailure(error)})\n`,
await Promise.all([rm(object, { force: true }), rm(bitcode, { force: true })]); );
return false; await rm(object, { force: true });
} finally {
await rm(optimized, { force: true });
}
}
/** Import bitcode for a unit whose object the runtime compiler emits
* itself (the release flavor). `compileBitcode(output)` must run the
* runtime compiler with the unit's exact flags plus -emit-llvm. A helper
* failure only drops the bitcode; the object is unaffected. */
export async function buildImportBitcode({ helper, compileBitcode, bitcode, tag }) {
const optimized = bitcode.replace(/\.bc$/, ".opt.bc");
await compileBitcode(optimized);
try {
await run(helper.path, ["prune-import", "--input", optimized, "--output", bitcode]);
return true;
} catch (error) {
const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0];
process.stderr.write(`prune-import: ${tag}: shipping without import bitcode (${detail})\n`);
await rm(bitcode, { force: true });
return false; return false;
} finally { } finally {
await rm(optimized, { force: true }); await rm(optimized, { force: true });
if (bitcode !== null) await rm(bitcode, { force: true });
} }
} }
@@ -3,8 +3,8 @@ import { join } from "node:path";
/** Points the artifacts of one runtime unit that hold the same bytes at one /** Points the artifacts of one runtime unit that hold the same bytes at one
* file and deletes the copies. Variants whose defines a unit does not read, * file and deletes the copies. Variants whose defines a unit does not read,
* and flavors that compile a unit the same way, produce identical objects * and flavors that compile a unit the same way, produce identical objects.
* and bitcode. Each link selects one flavor and one variant of each unit, so * Each link selects one flavor and one variant of each unit, so
* no link names a shared file twice. `root` is the directory artifact paths * no link names a shared file twice. `root` is the directory artifact paths
* are relative to; flavors keep the order they were built in, so the first * are relative to; flavors keep the order they were built in, so the first
* flavor's path holds the shared file. */ * flavor's path holds the shared file. */
@@ -18,16 +18,12 @@ export async function shareIdenticalArtifacts(flavors, root) {
firstPaths.set(unit.source, paths); firstPaths.set(unit.source, paths);
} }
for (const variant of unit.variants) { for (const variant of unit.variants) {
for (const artifact of variant.bitcode === undefined const first = paths.get(variant.sha256);
? [variant] if (first === undefined) {
: [variant, variant.bitcode]) { paths.set(variant.sha256, variant.path);
const first = paths.get(artifact.sha256); } else if (first !== variant.path) {
if (first === undefined) { await rm(join(root, variant.path));
paths.set(artifact.sha256, artifact.path); variant.path = first;
} else if (first !== artifact.path) {
await rm(join(root, artifact.path));
artifact.path = first;
}
} }
} }
} }
@@ -16,11 +16,7 @@ if (
throw new Error("runtime pack identity does not match package.json"); throw new Error("runtime pack identity does not match package.json");
const artifacts = [ const artifacts = [
...Object.values(manifest.flavors).flatMap((flavor) => ...Object.values(manifest.flavors).flatMap((flavor) =>
flavor.runtime_units.flatMap((unit) => flavor.runtime_units.flatMap((unit) => unit.variants),
unit.variants.flatMap((variant) =>
variant.bitcode ? [variant, variant.bitcode] : [variant],
),
),
), ),
...manifest.archives, ...manifest.archives,
...(manifest.data ?? []), ...(manifest.data ?? []),
@@ -5,7 +5,7 @@ import { dirname, join } from "node:path";
import { expect, test } from "vitest"; import { expect, test } from "vitest";
import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs"; import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs";
type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact }; type Artifact = { path: string; sha256: string; size: number };
async function artifact(root: string, path: string, sha256: string): Promise<Artifact> { async function artifact(root: string, path: string, sha256: string): Promise<Artifact> {
await mkdir(dirname(join(root, path)), { recursive: true }); await mkdir(dirname(join(root, path)), { recursive: true });
@@ -31,11 +31,6 @@ test("identical artifacts of one unit share the first flavor's file", async () =
await unit("release", "scr_b", { default: "b1" }), await unit("release", "scr_b", { default: "b1" }),
], ],
}; };
release.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/release/runtime/default/scr_b.bc",
"b-bitcode",
);
const speed = { const speed = {
runtime_units: [ runtime_units: [
await unit("speed", "scr_a", { default: "a2" }), await unit("speed", "scr_a", { default: "a2" }),
@@ -43,34 +38,20 @@ test("identical artifacts of one unit share the first flavor's file", async () =
await unit("speed", "scr_b", { default: "b1", other: "a1" }), await unit("speed", "scr_b", { default: "b1", other: "a1" }),
], ],
}; };
speed.runtime_units[1]!.variants[0]!.bitcode = await artifact(
root,
"artifacts/speed/runtime/default/scr_b.bc",
"b-bitcode",
);
const flavors = { release, speed }; const flavors = { release, speed };
await shareIdenticalArtifacts(flavors, root); await shareIdenticalArtifacts(flavors, root);
const paths = (flavor: typeof release) => const paths = (flavor: typeof release) =>
flavor.runtime_units.map((unit) => flavor.runtime_units.map((unit) => unit.variants.map((variant: Artifact) => variant.path));
unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]),
);
expect(paths(release)).toEqual([ expect(paths(release)).toEqual([
[ ["artifacts/release/runtime/default/scr_a.o", "artifacts/release/runtime/default/scr_a.o"],
["artifacts/release/runtime/default/scr_a.o", undefined], ["artifacts/release/runtime/default/scr_b.o"],
["artifacts/release/runtime/default/scr_a.o", undefined],
],
[["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]],
]); ]);
expect(paths(speed)).toEqual([ expect(paths(speed)).toEqual([
[["artifacts/speed/runtime/default/scr_a.o", undefined]], ["artifacts/speed/runtime/default/scr_a.o"],
[ ["artifacts/release/runtime/default/scr_b.o", "artifacts/speed/runtime/other/scr_b.o"],
["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"],
["artifacts/speed/runtime/other/scr_b.o", undefined],
],
]); ]);
expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false); expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false); expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false);
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true); expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true);
expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true); expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true);
} finally { } finally {
+30 -13
View File
@@ -549,7 +549,8 @@ export function compareCounts(baseline, candidate) {
/* ── runtime packs ─────────────────────────────────────────────────────── */ /* ── runtime packs ─────────────────────────────────────────────────────── */
/** Every file a runtime-pack.json lists (each flavor's runtime unit /** Every file a runtime-pack.json lists (each flavor's runtime unit
* variants, their bitcode, and the vendor archives), as path → sha256. */ * variants and the vendor archives, and the bitcode packs built before
* 0.4.7 listed), as path → sha256. */
export function runtimePackFiles(manifest) { export function runtimePackFiles(manifest) {
const files = new Map(); const files = new Map();
const visit = (value) => { const visit = (value) => {
@@ -586,20 +587,36 @@ export function linkedRuntimeFiles(diff) {
); );
} }
/** Defined functions of textual LLVM IR (llvm-dis output) as name → hash of /** Functions of an object's disassembly (llvm-objdump -d -r
* the body. Attribute group and metadata numbers are dropped first, since * --no-show-raw-insn --no-leading-addr) as name → hash of the body. Branch
* an unrelated change elsewhere in the unit renumbers them. */ * targets and relocations keep their symbol and offset but drop addresses,
export function llvmFunctionHashes(text) { * so a function that only moved within its section is not a change. */
export function objectFunctionHashes(text) {
const functions = new Map(); const functions = new Map();
const lines = text.split("\n"); let name = null;
for (let index = 0; index < lines.length; index++) { let body = [];
const head = /^define\b[^@]*@("(?:[^"\\]|\\.)*"|[\w.$-]+)\(/.exec(lines[index]); const flush = () => {
if (head === null) continue; if (name !== null) functions.set(name, hash(body.join("\n")));
const body = []; name = null;
for (; index < lines.length && lines[index] !== "}"; index++) body.push(lines[index]); body = [];
const normalized = body.join("\n").replace(/ #\d+/g, "").replace(/!\d+/g, "!"); };
functions.set(head[1].replace(/^"|"$/g, ""), hash(normalized)); for (const line of text.split("\n")) {
const head = /^(?:[0-9a-f]+ )?<(.+)>:$/.exec(line.trim());
if (head !== null) {
flush();
name = head[1];
} else if (line.trim() === "" || line.startsWith("Disassembly of section")) {
flush();
} else if (name !== null) {
body.push(
line
.trim()
.replace(/^[0-9a-f]+:\s+/, "")
.replace(/\b(?:0x)?[0-9a-f]+ <([^>]+)>/g, "<$1>"),
);
}
} }
flush();
return functions; return functions;
} }
+19 -18
View File
@@ -46,7 +46,7 @@ import {
definedFunctionNames, definedFunctionNames,
dynamicTierSymbols, dynamicTierSymbols,
linkedRuntimeFiles, linkedRuntimeFiles,
llvmFunctionHashes, objectFunctionHashes,
median, median,
icuSymbols, icuSymbols,
nodeIslandSymbols, nodeIslandSymbols,
@@ -465,21 +465,22 @@ async function instructionCounts(binary, args, out) {
const sha256 = (file) => createHash("sha256").update(readFileSync(file)).digest("hex"); const sha256 = (file) => createHash("sha256").update(readFileSync(file)).digest("hex");
/** llvm-dis from PATH or the pinned LLVM install, or null. */ /** llvm-objdump from PATH or the pinned LLVM install, or null. */
function llvmDis() { function llvmObjdump() {
for (const candidate of ["llvm-dis", "llvm-dis-22", "/usr/lib/llvm-22/bin/llvm-dis"]) for (const candidate of ["llvm-objdump", "llvm-objdump-22", "/usr/lib/llvm-22/bin/llvm-objdump"])
if (spawnSync(candidate, ["--version"]).status === 0) return candidate; if (spawnSync(candidate, ["--version"]).status === 0) return candidate;
const config = spawnSync("llvm-config", ["--bindir"], { encoding: "utf8" }); const config = spawnSync("llvm-config", ["--bindir"], { encoding: "utf8" });
const fromConfig = config.status === 0 ? join(config.stdout.trim(), "llvm-dis") : null; const fromConfig = config.status === 0 ? join(config.stdout.trim(), "llvm-objdump") : null;
return fromConfig !== null && existsSync(fromConfig) ? fromConfig : null; return fromConfig !== null && existsSync(fromConfig) ? fromConfig : null;
} }
/** What differs between the two checkouts' runtime packs for this host, or /** What differs between the two checkouts' runtime packs for this host, or
* null when no object or archive does: the changed pack files, whether the * null when no object or archive does: the changed pack files, whether the
* LLVM helper is the same, the symbols the changed objects define (on either * LLVM helper is the same, the symbols the changed objects define (on either
* side), and the functions that changed in the bitcode of the changed * side), and the functions whose code changed in the changed release and
* release and speed objects. Bitcode alone is not a change: it records the * speed objects. Bitcode, which packs built before 0.4.7 also list, is not
* checkout's source path, so it differs between any two checkouts. */ * a change: it records the checkout's source path, so it differs between
* any two checkouts. */
function runtimeChanges() { function runtimeChanges() {
const pack = (root) => const pack = (root) =>
readdirSync(join(root, "packages")) readdirSync(join(root, "packages"))
@@ -523,22 +524,22 @@ function runtimeChanges() {
}); });
for (const symbol of parseNm(listed.stdout ?? "")) symbols.add(symbol.name); for (const symbol of parseNm(listed.stdout ?? "")) symbols.add(symbol.name);
} }
// Changed functions, by unit, from the bitcode of the release and speed // Changed functions, by unit, from the disassembly of the release and
// flavors (the ones the gate builds with). // speed flavors' objects (the ones the gate builds with).
const dis = llvmDis(); const dis = llvmObjdump();
const functions = { changed: new Set(), added: new Set(), removed: new Set(), units: new Set() }; const functions = { changed: new Set(), added: new Set(), removed: new Set(), units: new Set() };
for (const object of objects) for (const object of objects)
functions.units.add(object.replace(/^.*\//, "").replace(/\.o$/, ".c")); functions.units.add(object.replace(/^.*\//, "").replace(/\.o$/, ".c"));
const bitcode = objects for (const path of objects.filter((path) => /^artifacts\/(release|speed)\//.test(path))) {
.filter((path) => /^artifacts\/(release|speed)\//.test(path))
.map((path) => path.replace(/\.o$/, ".bc"));
for (const path of bitcode) {
if (dis === null) continue; if (dis === null) continue;
const text = (root) => { const text = (root) => {
const file = join(packDir(root), path); const file = join(packDir(root), path);
if (!existsSync(file)) return new Map(); if (!existsSync(file)) return new Map();
const out = spawnSync(dis, [file, "-o", "-"], { encoding: "utf8", maxBuffer: 1 << 30 }); const out = spawnSync(dis, ["-d", "-r", "--no-show-raw-insn", "--no-leading-addr", file], {
return llvmFunctionHashes(out.stdout ?? ""); encoding: "utf8",
maxBuffer: 1 << 30,
});
return objectFunctionHashes(out.stdout ?? "");
}; };
const delta = changedFunctions(text(trees.baseline), text(trees.candidate)); const delta = changedFunctions(text(trees.baseline), text(trees.candidate));
for (const kind of ["changed", "added", "removed"]) for (const kind of ["changed", "added", "removed"])
@@ -698,7 +699,7 @@ if (runtime !== null) {
(runtime.helperSame ? "" : "; the LLVM helper differs too, so nothing is runtime-only"), (runtime.helperSame ? "" : "; the LLVM helper differs too, so nothing is runtime-only"),
); );
if (runtime.disassembler === null) if (runtime.disassembler === null)
console.log("runtime functions: llvm-dis not found, so changed functions are not listed"); console.log("runtime functions: llvm-objdump not found, so changed functions are not listed");
else else
console.log( console.log(
`runtime functions: ${runtime.functions.changed.length} changed` + `runtime functions: ${runtime.functions.changed.length} changed` +
+21 -12
View File
@@ -11,7 +11,7 @@ import {
definedFunctionNames, definedFunctionNames,
dynamicTierSymbols, dynamicTierSymbols,
linkedRuntimeFiles, linkedRuntimeFiles,
llvmFunctionHashes, objectFunctionHashes,
mergeReports, mergeReports,
nmOutput, nmOutput,
programInputsHash, programInputsHash,
@@ -279,21 +279,30 @@ test("the zero-cost gate names the runtime pack files and functions that changed
removed: [], removed: [],
}); });
const ll = (body: string, attrs: number) => // llvm-objdump -d -r --no-show-raw-insn --no-leading-addr output.
const objdump = (offset: number, call: string) =>
[ [
`define i32 @scr_same(i32 %x) #${attrs} {`, "scr_a.o:\tfile format elf64-x86-64",
" ret i32 %x, !dbg !" + attrs, "",
"}", "Disassembly of section .text:",
"define internal void @scr_changed() {", "",
` ${body}`, "<scr_same>:",
"}", ` \tjl\t0x${(offset + 0x12).toString(16)} <scr_same+0x12>`,
` \tcallq\t0x${(offset + 0xb).toString(16)} <scr_same+0xb>`,
"\t\tR_X86_64_PLT32\tscr_g-0x4",
" \tretq",
"",
"<scr_changed>:",
` \tcallq\t0x0 <scr_changed>`,
`\t\tR_X86_64_PLT32\t${call}-0x4`,
" \tretq",
"", "",
].join("\n"); ].join("\n");
const before = llvmFunctionHashes(ll("ret void", 0)); const before = objectFunctionHashes(objdump(0, "scr_g"));
const after = llvmFunctionHashes( const after = objectFunctionHashes(
ll("call void @f()\n ret void", 7) + 'define void @"scr new"() {\n}\n', objdump(0x40, "scr_h") + "<scr new>:\n \tretq\n",
); );
// Renumbered attribute groups and metadata are not changes. // A function that only moved within its section is not a change.
expect(changedFunctions(before, after)).toEqual({ expect(changedFunctions(before, after)).toEqual({
changed: ["scr_changed"], changed: ["scr_changed"],
added: ["scr new"], added: ["scr new"],