mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-31 07:23:06 +08:00
Stop building and shipping runtime import bitcode (#2583)
This commit is contained in:
Chris Tate
@@ -49,19 +49,19 @@ Runtime-pack builds emit each unit of the executable `speed` flavor through the
|
||||
|
||||
```console
|
||||
scriptc-llvm-codegen runtime-unit --input unit.opt.bc --object unit.o \
|
||||
--bitcode unit.bc --target x86_64-unknown-linux-gnu --tag scr_array \
|
||||
--target x86_64-unknown-linux-gnu --tag scr_array \
|
||||
[--function-sections --data-sections]
|
||||
```
|
||||
|
||||
The input is the runtime compiler's optimized bitcode. The command promotes unit-local symbols to hidden `<name>.scrunit.<tag>` globals and writes the object and the import bitcode from that one module, so imported bodies can reference the unit's private state and helpers.
|
||||
The input is the runtime compiler's bitcode for the unit. The command promotes unit-local symbols to hidden `<name>.scrunit.<tag>` globals and writes the object. With `--bitcode unit.bc` it also writes import bitcode from the same module, so imported bodies can reference the unit's private state and helpers. Runtime packs ship no bitcode, since program builds don't import it.
|
||||
|
||||
Release units keep the runtime compiler's own objects. Their import bitcode is the same compilation with `-emit-llvm`, reduced by the helper:
|
||||
`prune-import` reduces a unit's `-emit-llvm` bitcode to import bitcode without promotion:
|
||||
|
||||
```console
|
||||
scriptc-llvm-codegen prune-import --input unit.opt.bc --output unit.bc
|
||||
```
|
||||
|
||||
Both commands keep only what a program build can import: the importable bodies within the size limit at their call distance from any public function, and the constants they copy. Every other definition becomes a declaration, so a program imports exactly what it would from the whole unit while reading a fraction of it. Unpromoted unit-local symbols stay local, so release bitcode offers only bodies that need none of them.
|
||||
Import bitcode keeps only what a program build can import: the importable bodies within the size limit at their call distance from any public function, and the constants they copy. Every other definition becomes a declaration, so a program imports exactly what it would from the whole unit while reading a fraction of it. Unpromoted unit-local symbols stay local, so `prune-import` bitcode offers only bodies that need none of them.
|
||||
|
||||
## Linux release builds
|
||||
|
||||
|
||||
@@ -60,7 +60,8 @@ int version(int Argc, char **Argv) {
|
||||
{"data_layout", Machine->createDataLayout().getStringRepresentation()},
|
||||
// Commands and options added within one protocol version, for callers
|
||||
// that also run against older helpers.
|
||||
{"features", json::Array{"runtime-unit-profile", "profile-functions"}},
|
||||
{"features", json::Array{"runtime-unit-profile", "profile-functions",
|
||||
"runtime-unit-optional-bitcode"}},
|
||||
};
|
||||
outs() << formatv("{0}\n", json::Value(std::move(Response)));
|
||||
return 0;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// `runtime-unit`: turn one optimized runtime translation unit into the pair
|
||||
// of artifacts a runtime pack ships for it: the native object and the
|
||||
// bitcode the program build imports from (see runtime-import.cpp).
|
||||
// `runtime-unit`: turn one optimized runtime translation unit into the
|
||||
// native object a runtime pack ships for it and, with --bitcode, the pruned
|
||||
// bitcode a build could import from (see runtime-import.cpp). Runtime packs
|
||||
// ship no bitcode since program builds stopped importing it.
|
||||
//
|
||||
// Both come from one module. Every translation-unit-local symbol that an
|
||||
// imported body could need (functions, mutable state, address-significant
|
||||
@@ -106,7 +107,7 @@ std::optional<UnitOptions> parse(int Argc, char **Argv) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (Options.Input.empty() || Options.Object.empty() ||
|
||||
Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty())
|
||||
Options.Target.empty() || Options.Tag.empty())
|
||||
return std::nullopt;
|
||||
// A profile only guides an optimization the helper runs, and a unit is
|
||||
// either instrumented or annotated.
|
||||
@@ -196,8 +197,8 @@ int runtimeUnit(int Argc, char **Argv) {
|
||||
std::optional<UnitOptions> Options = parse(Argc, Argv);
|
||||
if (!Options)
|
||||
return fail("usage",
|
||||
"runtime-unit requires --input <bc> --object <o> --bitcode <bc> "
|
||||
"--target <triple> --tag <unit> and accepts "
|
||||
"runtime-unit requires --input <bc> --object <o> "
|
||||
"--target <triple> --tag <unit> and accepts --bitcode <bc>, "
|
||||
"--function-sections --data-sections, and --opt-level <level> "
|
||||
"with either --profile-generate <pattern> or --profile-use "
|
||||
"<profdata>");
|
||||
@@ -306,14 +307,16 @@ int runtimeUnit(int Argc, char **Argv) {
|
||||
|
||||
// The bitcode exists only to be imported from, so it carries only what
|
||||
// an import can take; program builds then read a fraction of the unit.
|
||||
std::unique_ptr<Module> Import = CloneModule(*M);
|
||||
pruneRuntimeImport(*Import);
|
||||
if (verifyModule(*Import, &VerifyStream))
|
||||
return fail("verification_failed", VerifyStream.str());
|
||||
if (std::optional<std::string> Failure = writeFileAtomically(
|
||||
Options->Bitcode,
|
||||
[&](raw_fd_ostream &Out) { WriteBitcodeToFile(*Import, Out); }))
|
||||
return fail("output_write_failed", *Failure);
|
||||
if (!Options->Bitcode.empty()) {
|
||||
std::unique_ptr<Module> Import = CloneModule(*M);
|
||||
pruneRuntimeImport(*Import);
|
||||
if (verifyModule(*Import, &VerifyStream))
|
||||
return fail("verification_failed", VerifyStream.str());
|
||||
if (std::optional<std::string> Failure = writeFileAtomically(
|
||||
Options->Bitcode,
|
||||
[&](raw_fd_ostream &Out) { WriteBitcodeToFile(*Import, Out); }))
|
||||
return fail("output_write_failed", *Failure);
|
||||
}
|
||||
|
||||
bool Unsupported = false;
|
||||
std::optional<std::string> Failure =
|
||||
|
||||
@@ -23,8 +23,6 @@ export interface RuntimePackVariant extends RuntimePackArtifact {
|
||||
id: string;
|
||||
when: Record<string, boolean>;
|
||||
defines: string[];
|
||||
/** LLVM bitcode of the same unit, imported by the helper for inlining. */
|
||||
bitcode?: RuntimePackArtifact;
|
||||
}
|
||||
|
||||
interface RuntimePackUnit {
|
||||
@@ -212,7 +210,6 @@ export function parseRuntimePackManifest(value: unknown): RuntimePackManifest {
|
||||
const when = object(variant?.when);
|
||||
return (
|
||||
validArtifact(variantRaw) &&
|
||||
(variant?.bitcode === undefined || validArtifact(variant.bitcode)) &&
|
||||
typeof variant?.id === "string" &&
|
||||
when !== null &&
|
||||
Object.values(when).every((entry) => typeof entry === "boolean") &&
|
||||
@@ -426,8 +423,8 @@ export function validateRuntimePackIdentity(
|
||||
|
||||
export interface RuntimePackArtifacts {
|
||||
features: RuntimeFeatureSet;
|
||||
/** Selected unit variants; each may carry its bitcode artifact. */
|
||||
runtime: (RuntimePackArtifact & { bitcode?: RuntimePackArtifact })[];
|
||||
/** Selected unit variants. */
|
||||
runtime: RuntimePackArtifact[];
|
||||
archives: RuntimePackArtifact[];
|
||||
/** Files to install beside the executable (ICU's data file for Intl). */
|
||||
data: RuntimePackArtifact[];
|
||||
|
||||
@@ -321,17 +321,6 @@ describe("runtime pack manifests", () => {
|
||||
expect(fileHandle.archives.map((path) => path.split("/").at(-1))).toEqual(["zlib.a"]);
|
||||
});
|
||||
|
||||
test("runtime unit bitcode must stay inside the pack", async () => {
|
||||
const { manifest } = await fixture();
|
||||
const malformed = structuredClone(manifest);
|
||||
malformed.flavors.release!.runtime_units[0]!.variants[0]!.bitcode = {
|
||||
path: "../escape.bc",
|
||||
sha256: "0".repeat(64),
|
||||
size: 1,
|
||||
};
|
||||
expect(() => parseRuntimePackManifest(malformed)).toThrow("malformed");
|
||||
});
|
||||
|
||||
test("a stale manifest reports its identity independently from the package", async () => {
|
||||
const { packagePath, manifest, root } = await fixture();
|
||||
await writeFile(
|
||||
|
||||
@@ -9,7 +9,6 @@ import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { RUNTIME_PACK_MATRIX } from "../runtime-pack-matrix.mjs";
|
||||
import {
|
||||
buildImportBitcode,
|
||||
buildRuntimeUnit,
|
||||
resolveRuntimeUnitHelper,
|
||||
} from "../../runtime-pack-common/scripts/runtime-unit.mjs";
|
||||
@@ -80,8 +79,8 @@ async function build() {
|
||||
const ZLIB_BUILD = zlibInputs(RUNTIME_PACK_MATRIX.target.llvm_triple);
|
||||
|
||||
// Speed-flavor executable units are emitted through the host LLVM helper
|
||||
// so their object and import bitcode come from one promoted module. The
|
||||
// helper also reduces release units' bitcode to what programs can import.
|
||||
// (runtime-unit.mjs), which promotes their local symbols and applies the
|
||||
// runtime profile.
|
||||
const unitHelper = await resolveRuntimeUnitHelper(
|
||||
repoRoot,
|
||||
RUNTIME_PACK_MATRIX.target.llvm_triple,
|
||||
@@ -137,7 +136,7 @@ async function build() {
|
||||
const flavors = {};
|
||||
// The executable `speed` flavor (--optimization=speed) exists only when
|
||||
// it differs from release: its static units are emitted through the host
|
||||
// LLVM helper with import bitcode.
|
||||
// LLVM helper.
|
||||
const flavorPlan = [
|
||||
...Object.entries(RUNTIME_PACK_MATRIX.flavors),
|
||||
...("release" in RUNTIME_PACK_MATRIX.flavors && unitHelper !== null
|
||||
@@ -183,8 +182,6 @@ async function build() {
|
||||
...variant.defines.map((define) => `-D${define}`),
|
||||
...includeFlags,
|
||||
];
|
||||
let bitcode;
|
||||
const bitcodeOutput = output.replace(/\.o$/, ".bc");
|
||||
if (flavor === "speed") await mkdir(dirname(output), { recursive: true });
|
||||
if (
|
||||
flavor === "speed" &&
|
||||
@@ -201,7 +198,6 @@ async function build() {
|
||||
optimized,
|
||||
]),
|
||||
object: output,
|
||||
bitcode: bitcodeOutput,
|
||||
triple: RUNTIME_PACK_MATRIX.target.llvm_triple,
|
||||
tag: unit.source.replace(/\.c$/, ""),
|
||||
sections:
|
||||
@@ -210,38 +206,8 @@ async function build() {
|
||||
),
|
||||
}))
|
||||
) {
|
||||
bitcode = {
|
||||
path: artifactPath(bitcodeOutput),
|
||||
sha256: await sha256(bitcodeOutput),
|
||||
size: (await stat(bitcodeOutput)).size,
|
||||
};
|
||||
} else {
|
||||
await compile(join(runtimeSrc, unit.source), output, unitFlags);
|
||||
if (
|
||||
flavor === "release" &&
|
||||
unitHelper !== null &&
|
||||
(await buildImportBitcode({
|
||||
helper: unitHelper,
|
||||
compileBitcode: (optimized) =>
|
||||
run(compiler, [
|
||||
...sourcePathFlags,
|
||||
...unitFlags,
|
||||
"-emit-llvm",
|
||||
"-c",
|
||||
join(runtimeSrc, unit.source),
|
||||
"-o",
|
||||
optimized,
|
||||
]),
|
||||
bitcode: bitcodeOutput,
|
||||
tag: unit.source.replace(/\.c$/, ""),
|
||||
}))
|
||||
) {
|
||||
bitcode = {
|
||||
path: artifactPath(bitcodeOutput),
|
||||
sha256: await sha256(bitcodeOutput),
|
||||
size: (await stat(bitcodeOutput)).size,
|
||||
};
|
||||
}
|
||||
}
|
||||
const built = {
|
||||
id: variant.id,
|
||||
@@ -250,7 +216,6 @@ async function build() {
|
||||
path: artifactPath(output),
|
||||
sha256: await sha256(output),
|
||||
size: (await stat(output)).size,
|
||||
...(bitcode === undefined ? {} : { bitcode }),
|
||||
};
|
||||
variants.push(built);
|
||||
}
|
||||
|
||||
@@ -16,11 +16,7 @@ if (
|
||||
throw new Error("runtime pack identity does not match package.json");
|
||||
const artifacts = [
|
||||
...Object.values(manifest.flavors).flatMap((flavor) =>
|
||||
flavor.runtime_units.flatMap((unit) =>
|
||||
unit.variants.flatMap((variant) =>
|
||||
variant.bitcode ? [variant, variant.bitcode] : [variant],
|
||||
),
|
||||
),
|
||||
flavor.runtime_units.flatMap((unit) => unit.variants),
|
||||
),
|
||||
...manifest.archives,
|
||||
];
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
runtimeForbiddenStrings,
|
||||
} from "./artifact-policy.mjs";
|
||||
import { createDeterministicArchive } from "./archive.mjs";
|
||||
import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
|
||||
import { buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs";
|
||||
import { installRuntimePack, withBuildLock } from "./build-state.mjs";
|
||||
import { shareIdenticalArtifacts } from "./share-artifacts.mjs";
|
||||
import { parallelMap } from "./parallel-map.mjs";
|
||||
@@ -141,8 +141,8 @@ async function build() {
|
||||
]);
|
||||
};
|
||||
// Speed-flavor executable units are emitted through the host LLVM helper
|
||||
// so their object and import bitcode come from one promoted module. The
|
||||
// helper also reduces release units' bitcode to what programs can import.
|
||||
// (runtime-unit.mjs), which promotes their local symbols and applies the
|
||||
// runtime profile.
|
||||
const unitHelper =
|
||||
config.bitcode === false
|
||||
? null
|
||||
@@ -379,7 +379,7 @@ async function build() {
|
||||
const flavors = {};
|
||||
// The executable `speed` flavor (--optimization=speed) exists only when
|
||||
// it differs from release: its static units are emitted through the host
|
||||
// LLVM helper with import bitcode.
|
||||
// LLVM helper.
|
||||
const flavorPlan = [
|
||||
...Object.entries(matrix.flavors),
|
||||
...("release" in matrix.flavors && unitHelper !== null
|
||||
@@ -463,8 +463,6 @@ async function build() {
|
||||
...tlsFlags,
|
||||
...includeFlags,
|
||||
];
|
||||
let bitcode;
|
||||
const bitcodeOutput = output.replace(/\.o$/, ".bc");
|
||||
if (flavor === "speed") await mkdir(dirname(output), { recursive: true });
|
||||
let unitBuild = false;
|
||||
if (
|
||||
@@ -489,7 +487,6 @@ async function build() {
|
||||
? ["--opt-level", "2"]
|
||||
: (speedProfile?.args ?? []),
|
||||
object: output,
|
||||
bitcode: bitcodeOutput,
|
||||
triple: matrix.target.llvm_triple,
|
||||
tag: unit.source.replace(/\.c$/, ""),
|
||||
sections:
|
||||
@@ -500,40 +497,8 @@ async function build() {
|
||||
) {
|
||||
if (unitBuild.profile !== undefined) profileMatches.push(unitBuild.profile);
|
||||
for (const name of unitBuild.functions ?? []) profiledFunctions.add(name);
|
||||
bitcode = {
|
||||
path: artifactPath(bitcodeOutput),
|
||||
sha256: await sha256(bitcodeOutput),
|
||||
size: (await stat(bitcodeOutput)).size,
|
||||
};
|
||||
} else {
|
||||
await compile(join(runtimeSrc, unit.source), output, unitFlags);
|
||||
if (
|
||||
flavor === "release" &&
|
||||
unitHelper !== null &&
|
||||
(await buildImportBitcode({
|
||||
helper: unitHelper,
|
||||
compileBitcode: (optimized) =>
|
||||
run(compiler, [
|
||||
...compilerArgs,
|
||||
...sourcePathFlags,
|
||||
...unitFlags,
|
||||
"-g0",
|
||||
"-emit-llvm",
|
||||
"-c",
|
||||
join(runtimeSrc, unit.source),
|
||||
"-o",
|
||||
optimized,
|
||||
]),
|
||||
bitcode: bitcodeOutput,
|
||||
tag: unit.source.replace(/\.c$/, ""),
|
||||
}))
|
||||
) {
|
||||
bitcode = {
|
||||
path: artifactPath(bitcodeOutput),
|
||||
sha256: await sha256(bitcodeOutput),
|
||||
size: (await stat(bitcodeOutput)).size,
|
||||
};
|
||||
}
|
||||
}
|
||||
const built = {
|
||||
id: variant.id,
|
||||
@@ -542,7 +507,6 @@ async function build() {
|
||||
path: artifactPath(output),
|
||||
sha256: await sha256(output),
|
||||
size: (await stat(output)).size,
|
||||
...(bitcode === undefined ? {} : { bitcode }),
|
||||
};
|
||||
variants.push(built);
|
||||
}
|
||||
|
||||
@@ -5,26 +5,18 @@ import { promisify } from "node:util";
|
||||
|
||||
const run = promisify(execFile);
|
||||
|
||||
/* Executable runtime units ship the bitcode the LLVM helper imports small
|
||||
* runtime functions from (available_externally, so program code can inline
|
||||
* them while the unit's object stays the only linked definition).
|
||||
*
|
||||
* Speed units produce both artifacts from one optimized LLVM module: the
|
||||
* runtime compiler optimizes the C source to bitcode, and the host helper's
|
||||
* `runtime-unit` command promotes translation-unit-local symbols to hidden
|
||||
* unit-qualified globals and emits the object and the bitcode. With the
|
||||
* target's committed runtime profile, the runtime compiler emits the bitcode
|
||||
* unoptimized and the helper also optimizes it with the profile (build.mjs),
|
||||
* so one LLVM reads and writes every profile. Release units keep the
|
||||
* runtime compiler's own objects; their bitcode is the same compilation with
|
||||
* -emit-llvm, which the helper's `prune-import` reduces to the bodies that
|
||||
* need none of the unit's local state. Both bitcode forms
|
||||
* keep only what a program build can import.
|
||||
/* Speed units are emitted through the host LLVM helper: the runtime
|
||||
* compiler compiles the C source to bitcode, and the helper's `runtime-unit`
|
||||
* command promotes translation-unit-local symbols to hidden unit-qualified
|
||||
* globals and emits the object. With the target's committed runtime profile,
|
||||
* the runtime compiler emits the bitcode unoptimized and the helper also
|
||||
* optimizes it with the profile (build.mjs), so one LLVM reads and writes
|
||||
* every profile. Runtime packs ship objects only: program builds do not
|
||||
* import runtime bitcode.
|
||||
*
|
||||
* Every host helper supports every scriptc target, so packs keep building
|
||||
* on their usual hosts. Without a runnable host helper (or with
|
||||
* SCRIPTC_RUNTIME_BITCODE=0) units are compiled to objects directly and
|
||||
* ship no bitcode; program builds then simply import nothing. */
|
||||
* SCRIPTC_RUNTIME_BITCODE=0) a pack has no speed flavor. */
|
||||
|
||||
function hostHelperPackage() {
|
||||
if (process.platform === "darwin") return `llvm-darwin-${process.arch}`;
|
||||
@@ -63,7 +55,8 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
|
||||
// Commands and options newer than the protocol version, such as
|
||||
// "runtime-unit-profile": `runtime-unit` takes --opt-level,
|
||||
// --profile-use and --profile-generate (older helpers only promote and
|
||||
// emit).
|
||||
// emit); "runtime-unit-optional-bitcode": it emits the object without
|
||||
// the import bitcode.
|
||||
features: Array.isArray(version.features) ? version.features : [],
|
||||
};
|
||||
} catch {
|
||||
@@ -72,14 +65,31 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
|
||||
}
|
||||
}
|
||||
|
||||
/** The helper's diagnostic from a failed command: its JSON line, which
|
||||
* follows any LLVM warnings on the same streams. */
|
||||
function helperFailure(error) {
|
||||
const lines = [error.stdout, error.stderr]
|
||||
.filter((text) => typeof text === "string")
|
||||
.join("\n")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line !== "");
|
||||
return (
|
||||
lines.find((line) => line.startsWith("{")) ??
|
||||
lines.find((line) => !line.includes("is not a recognized feature")) ??
|
||||
lines[0] ??
|
||||
error.message ??
|
||||
String(error)
|
||||
);
|
||||
}
|
||||
|
||||
/** Compile one unit through the helper. `compileBitcode(output)` must run
|
||||
* the runtime compiler with the unit's exact flags plus -emit-llvm.
|
||||
* `helperArgs` asks the helper to optimize the unit itself (`--opt-level`,
|
||||
* with `--profile-use` or `--profile-generate`), for bitcode the runtime
|
||||
* compiler emitted unoptimized.
|
||||
* Returns false, leaving no artifacts behind, when the helper cannot emit
|
||||
* this unit; the caller then compiles the object directly without bitcode,
|
||||
* so a pack never fails to build because of the import path. Otherwise
|
||||
* this unit; the caller then compiles the object directly. Otherwise
|
||||
* returns an object, with the helper's `profile` match counts when a profile
|
||||
* guided the unit, or the profile names of the unit's `functions` when it
|
||||
* was instrumented. */
|
||||
@@ -88,12 +98,15 @@ export async function buildRuntimeUnit({
|
||||
compileBitcode,
|
||||
helperArgs = [],
|
||||
object,
|
||||
bitcode,
|
||||
triple,
|
||||
tag,
|
||||
sections,
|
||||
}) {
|
||||
const optimized = object.replace(/\.o$/, ".opt.bc");
|
||||
// Helpers older than "runtime-unit-optional-bitcode" always write it.
|
||||
const bitcode = helper.features.includes("runtime-unit-optional-bitcode")
|
||||
? null
|
||||
: object.replace(/\.o$/, ".import.bc");
|
||||
// A runtime compiler failure is a real build error and surfaces as one.
|
||||
await compileBitcode(optimized);
|
||||
try {
|
||||
@@ -103,8 +116,7 @@ export async function buildRuntimeUnit({
|
||||
optimized,
|
||||
"--object",
|
||||
object,
|
||||
"--bitcode",
|
||||
bitcode,
|
||||
...(bitcode === null ? [] : ["--bitcode", bitcode]),
|
||||
"--target",
|
||||
triple,
|
||||
"--tag",
|
||||
@@ -118,31 +130,13 @@ export async function buildRuntimeUnit({
|
||||
...(report.functions === undefined ? {} : { functions: report.functions }),
|
||||
};
|
||||
} catch (error) {
|
||||
const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0];
|
||||
process.stderr.write(`runtime-unit: ${tag}: emitting without import bitcode (${detail})\n`);
|
||||
await Promise.all([rm(object, { force: true }), rm(bitcode, { force: true })]);
|
||||
return false;
|
||||
} finally {
|
||||
await rm(optimized, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Import bitcode for a unit whose object the runtime compiler emits
|
||||
* itself (the release flavor). `compileBitcode(output)` must run the
|
||||
* runtime compiler with the unit's exact flags plus -emit-llvm. A helper
|
||||
* failure only drops the bitcode; the object is unaffected. */
|
||||
export async function buildImportBitcode({ helper, compileBitcode, bitcode, tag }) {
|
||||
const optimized = bitcode.replace(/\.bc$/, ".opt.bc");
|
||||
await compileBitcode(optimized);
|
||||
try {
|
||||
await run(helper.path, ["prune-import", "--input", optimized, "--output", bitcode]);
|
||||
return true;
|
||||
} catch (error) {
|
||||
const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0];
|
||||
process.stderr.write(`prune-import: ${tag}: shipping without import bitcode (${detail})\n`);
|
||||
await rm(bitcode, { force: true });
|
||||
process.stderr.write(
|
||||
`runtime-unit: ${tag}: compiling the object directly (${helperFailure(error)})\n`,
|
||||
);
|
||||
await rm(object, { force: true });
|
||||
return false;
|
||||
} finally {
|
||||
await rm(optimized, { force: true });
|
||||
if (bitcode !== null) await rm(bitcode, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ import { join } from "node:path";
|
||||
|
||||
/** Points the artifacts of one runtime unit that hold the same bytes at one
|
||||
* file and deletes the copies. Variants whose defines a unit does not read,
|
||||
* and flavors that compile a unit the same way, produce identical objects
|
||||
* and bitcode. Each link selects one flavor and one variant of each unit, so
|
||||
* and flavors that compile a unit the same way, produce identical objects.
|
||||
* Each link selects one flavor and one variant of each unit, so
|
||||
* no link names a shared file twice. `root` is the directory artifact paths
|
||||
* are relative to; flavors keep the order they were built in, so the first
|
||||
* flavor's path holds the shared file. */
|
||||
@@ -18,16 +18,12 @@ export async function shareIdenticalArtifacts(flavors, root) {
|
||||
firstPaths.set(unit.source, paths);
|
||||
}
|
||||
for (const variant of unit.variants) {
|
||||
for (const artifact of variant.bitcode === undefined
|
||||
? [variant]
|
||||
: [variant, variant.bitcode]) {
|
||||
const first = paths.get(artifact.sha256);
|
||||
if (first === undefined) {
|
||||
paths.set(artifact.sha256, artifact.path);
|
||||
} else if (first !== artifact.path) {
|
||||
await rm(join(root, artifact.path));
|
||||
artifact.path = first;
|
||||
}
|
||||
const first = paths.get(variant.sha256);
|
||||
if (first === undefined) {
|
||||
paths.set(variant.sha256, variant.path);
|
||||
} else if (first !== variant.path) {
|
||||
await rm(join(root, variant.path));
|
||||
variant.path = first;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,7 @@ if (
|
||||
throw new Error("runtime pack identity does not match package.json");
|
||||
const artifacts = [
|
||||
...Object.values(manifest.flavors).flatMap((flavor) =>
|
||||
flavor.runtime_units.flatMap((unit) =>
|
||||
unit.variants.flatMap((variant) =>
|
||||
variant.bitcode ? [variant, variant.bitcode] : [variant],
|
||||
),
|
||||
),
|
||||
flavor.runtime_units.flatMap((unit) => unit.variants),
|
||||
),
|
||||
...manifest.archives,
|
||||
...(manifest.data ?? []),
|
||||
|
||||
@@ -5,7 +5,7 @@ import { dirname, join } from "node:path";
|
||||
import { expect, test } from "vitest";
|
||||
import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs";
|
||||
|
||||
type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact };
|
||||
type Artifact = { path: string; sha256: string; size: number };
|
||||
|
||||
async function artifact(root: string, path: string, sha256: string): Promise<Artifact> {
|
||||
await mkdir(dirname(join(root, path)), { recursive: true });
|
||||
@@ -31,11 +31,6 @@ test("identical artifacts of one unit share the first flavor's file", async () =
|
||||
await unit("release", "scr_b", { default: "b1" }),
|
||||
],
|
||||
};
|
||||
release.runtime_units[1]!.variants[0]!.bitcode = await artifact(
|
||||
root,
|
||||
"artifacts/release/runtime/default/scr_b.bc",
|
||||
"b-bitcode",
|
||||
);
|
||||
const speed = {
|
||||
runtime_units: [
|
||||
await unit("speed", "scr_a", { default: "a2" }),
|
||||
@@ -43,34 +38,20 @@ test("identical artifacts of one unit share the first flavor's file", async () =
|
||||
await unit("speed", "scr_b", { default: "b1", other: "a1" }),
|
||||
],
|
||||
};
|
||||
speed.runtime_units[1]!.variants[0]!.bitcode = await artifact(
|
||||
root,
|
||||
"artifacts/speed/runtime/default/scr_b.bc",
|
||||
"b-bitcode",
|
||||
);
|
||||
const flavors = { release, speed };
|
||||
await shareIdenticalArtifacts(flavors, root);
|
||||
const paths = (flavor: typeof release) =>
|
||||
flavor.runtime_units.map((unit) =>
|
||||
unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]),
|
||||
);
|
||||
flavor.runtime_units.map((unit) => unit.variants.map((variant: Artifact) => variant.path));
|
||||
expect(paths(release)).toEqual([
|
||||
[
|
||||
["artifacts/release/runtime/default/scr_a.o", undefined],
|
||||
["artifacts/release/runtime/default/scr_a.o", undefined],
|
||||
],
|
||||
[["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]],
|
||||
["artifacts/release/runtime/default/scr_a.o", "artifacts/release/runtime/default/scr_a.o"],
|
||||
["artifacts/release/runtime/default/scr_b.o"],
|
||||
]);
|
||||
expect(paths(speed)).toEqual([
|
||||
[["artifacts/speed/runtime/default/scr_a.o", undefined]],
|
||||
[
|
||||
["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"],
|
||||
["artifacts/speed/runtime/other/scr_b.o", undefined],
|
||||
],
|
||||
["artifacts/speed/runtime/default/scr_a.o"],
|
||||
["artifacts/release/runtime/default/scr_b.o", "artifacts/speed/runtime/other/scr_b.o"],
|
||||
]);
|
||||
expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false);
|
||||
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false);
|
||||
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false);
|
||||
expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true);
|
||||
expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true);
|
||||
} finally {
|
||||
|
||||
@@ -549,7 +549,8 @@ export function compareCounts(baseline, candidate) {
|
||||
/* ── runtime packs ─────────────────────────────────────────────────────── */
|
||||
|
||||
/** Every file a runtime-pack.json lists (each flavor's runtime unit
|
||||
* variants, their bitcode, and the vendor archives), as path → sha256. */
|
||||
* variants and the vendor archives, and the bitcode packs built before
|
||||
* 0.4.7 listed), as path → sha256. */
|
||||
export function runtimePackFiles(manifest) {
|
||||
const files = new Map();
|
||||
const visit = (value) => {
|
||||
@@ -586,20 +587,36 @@ export function linkedRuntimeFiles(diff) {
|
||||
);
|
||||
}
|
||||
|
||||
/** Defined functions of textual LLVM IR (llvm-dis output) as name → hash of
|
||||
* the body. Attribute group and metadata numbers are dropped first, since
|
||||
* an unrelated change elsewhere in the unit renumbers them. */
|
||||
export function llvmFunctionHashes(text) {
|
||||
/** Functions of an object's disassembly (llvm-objdump -d -r
|
||||
* --no-show-raw-insn --no-leading-addr) as name → hash of the body. Branch
|
||||
* targets and relocations keep their symbol and offset but drop addresses,
|
||||
* so a function that only moved within its section is not a change. */
|
||||
export function objectFunctionHashes(text) {
|
||||
const functions = new Map();
|
||||
const lines = text.split("\n");
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
const head = /^define\b[^@]*@("(?:[^"\\]|\\.)*"|[\w.$-]+)\(/.exec(lines[index]);
|
||||
if (head === null) continue;
|
||||
const body = [];
|
||||
for (; index < lines.length && lines[index] !== "}"; index++) body.push(lines[index]);
|
||||
const normalized = body.join("\n").replace(/ #\d+/g, "").replace(/!\d+/g, "!");
|
||||
functions.set(head[1].replace(/^"|"$/g, ""), hash(normalized));
|
||||
let name = null;
|
||||
let body = [];
|
||||
const flush = () => {
|
||||
if (name !== null) functions.set(name, hash(body.join("\n")));
|
||||
name = null;
|
||||
body = [];
|
||||
};
|
||||
for (const line of text.split("\n")) {
|
||||
const head = /^(?:[0-9a-f]+ )?<(.+)>:$/.exec(line.trim());
|
||||
if (head !== null) {
|
||||
flush();
|
||||
name = head[1];
|
||||
} else if (line.trim() === "" || line.startsWith("Disassembly of section")) {
|
||||
flush();
|
||||
} else if (name !== null) {
|
||||
body.push(
|
||||
line
|
||||
.trim()
|
||||
.replace(/^[0-9a-f]+:\s+/, "")
|
||||
.replace(/\b(?:0x)?[0-9a-f]+ <([^>]+)>/g, "<$1>"),
|
||||
);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return functions;
|
||||
}
|
||||
|
||||
|
||||
+19
-18
@@ -46,7 +46,7 @@ import {
|
||||
definedFunctionNames,
|
||||
dynamicTierSymbols,
|
||||
linkedRuntimeFiles,
|
||||
llvmFunctionHashes,
|
||||
objectFunctionHashes,
|
||||
median,
|
||||
icuSymbols,
|
||||
nodeIslandSymbols,
|
||||
@@ -465,21 +465,22 @@ async function instructionCounts(binary, args, out) {
|
||||
|
||||
const sha256 = (file) => createHash("sha256").update(readFileSync(file)).digest("hex");
|
||||
|
||||
/** llvm-dis from PATH or the pinned LLVM install, or null. */
|
||||
function llvmDis() {
|
||||
for (const candidate of ["llvm-dis", "llvm-dis-22", "/usr/lib/llvm-22/bin/llvm-dis"])
|
||||
/** llvm-objdump from PATH or the pinned LLVM install, or null. */
|
||||
function llvmObjdump() {
|
||||
for (const candidate of ["llvm-objdump", "llvm-objdump-22", "/usr/lib/llvm-22/bin/llvm-objdump"])
|
||||
if (spawnSync(candidate, ["--version"]).status === 0) return candidate;
|
||||
const config = spawnSync("llvm-config", ["--bindir"], { encoding: "utf8" });
|
||||
const fromConfig = config.status === 0 ? join(config.stdout.trim(), "llvm-dis") : null;
|
||||
const fromConfig = config.status === 0 ? join(config.stdout.trim(), "llvm-objdump") : null;
|
||||
return fromConfig !== null && existsSync(fromConfig) ? fromConfig : null;
|
||||
}
|
||||
|
||||
/** What differs between the two checkouts' runtime packs for this host, or
|
||||
* null when no object or archive does: the changed pack files, whether the
|
||||
* LLVM helper is the same, the symbols the changed objects define (on either
|
||||
* side), and the functions that changed in the bitcode of the changed
|
||||
* release and speed objects. Bitcode alone is not a change: it records the
|
||||
* checkout's source path, so it differs between any two checkouts. */
|
||||
* side), and the functions whose code changed in the changed release and
|
||||
* speed objects. Bitcode, which packs built before 0.4.7 also list, is not
|
||||
* a change: it records the checkout's source path, so it differs between
|
||||
* any two checkouts. */
|
||||
function runtimeChanges() {
|
||||
const pack = (root) =>
|
||||
readdirSync(join(root, "packages"))
|
||||
@@ -523,22 +524,22 @@ function runtimeChanges() {
|
||||
});
|
||||
for (const symbol of parseNm(listed.stdout ?? "")) symbols.add(symbol.name);
|
||||
}
|
||||
// Changed functions, by unit, from the bitcode of the release and speed
|
||||
// flavors (the ones the gate builds with).
|
||||
const dis = llvmDis();
|
||||
// Changed functions, by unit, from the disassembly of the release and
|
||||
// speed flavors' objects (the ones the gate builds with).
|
||||
const dis = llvmObjdump();
|
||||
const functions = { changed: new Set(), added: new Set(), removed: new Set(), units: new Set() };
|
||||
for (const object of objects)
|
||||
functions.units.add(object.replace(/^.*\//, "").replace(/\.o$/, ".c"));
|
||||
const bitcode = objects
|
||||
.filter((path) => /^artifacts\/(release|speed)\//.test(path))
|
||||
.map((path) => path.replace(/\.o$/, ".bc"));
|
||||
for (const path of bitcode) {
|
||||
for (const path of objects.filter((path) => /^artifacts\/(release|speed)\//.test(path))) {
|
||||
if (dis === null) continue;
|
||||
const text = (root) => {
|
||||
const file = join(packDir(root), path);
|
||||
if (!existsSync(file)) return new Map();
|
||||
const out = spawnSync(dis, [file, "-o", "-"], { encoding: "utf8", maxBuffer: 1 << 30 });
|
||||
return llvmFunctionHashes(out.stdout ?? "");
|
||||
const out = spawnSync(dis, ["-d", "-r", "--no-show-raw-insn", "--no-leading-addr", file], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1 << 30,
|
||||
});
|
||||
return objectFunctionHashes(out.stdout ?? "");
|
||||
};
|
||||
const delta = changedFunctions(text(trees.baseline), text(trees.candidate));
|
||||
for (const kind of ["changed", "added", "removed"])
|
||||
@@ -698,7 +699,7 @@ if (runtime !== null) {
|
||||
(runtime.helperSame ? "" : "; the LLVM helper differs too, so nothing is runtime-only"),
|
||||
);
|
||||
if (runtime.disassembler === null)
|
||||
console.log("runtime functions: llvm-dis not found, so changed functions are not listed");
|
||||
console.log("runtime functions: llvm-objdump not found, so changed functions are not listed");
|
||||
else
|
||||
console.log(
|
||||
`runtime functions: ${runtime.functions.changed.length} changed` +
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
definedFunctionNames,
|
||||
dynamicTierSymbols,
|
||||
linkedRuntimeFiles,
|
||||
llvmFunctionHashes,
|
||||
objectFunctionHashes,
|
||||
mergeReports,
|
||||
nmOutput,
|
||||
programInputsHash,
|
||||
@@ -279,21 +279,30 @@ test("the zero-cost gate names the runtime pack files and functions that changed
|
||||
removed: [],
|
||||
});
|
||||
|
||||
const ll = (body: string, attrs: number) =>
|
||||
// llvm-objdump -d -r --no-show-raw-insn --no-leading-addr output.
|
||||
const objdump = (offset: number, call: string) =>
|
||||
[
|
||||
`define i32 @scr_same(i32 %x) #${attrs} {`,
|
||||
" ret i32 %x, !dbg !" + attrs,
|
||||
"}",
|
||||
"define internal void @scr_changed() {",
|
||||
` ${body}`,
|
||||
"}",
|
||||
"scr_a.o:\tfile format elf64-x86-64",
|
||||
"",
|
||||
"Disassembly of section .text:",
|
||||
"",
|
||||
"<scr_same>:",
|
||||
` \tjl\t0x${(offset + 0x12).toString(16)} <scr_same+0x12>`,
|
||||
` \tcallq\t0x${(offset + 0xb).toString(16)} <scr_same+0xb>`,
|
||||
"\t\tR_X86_64_PLT32\tscr_g-0x4",
|
||||
" \tretq",
|
||||
"",
|
||||
"<scr_changed>:",
|
||||
` \tcallq\t0x0 <scr_changed>`,
|
||||
`\t\tR_X86_64_PLT32\t${call}-0x4`,
|
||||
" \tretq",
|
||||
"",
|
||||
].join("\n");
|
||||
const before = llvmFunctionHashes(ll("ret void", 0));
|
||||
const after = llvmFunctionHashes(
|
||||
ll("call void @f()\n ret void", 7) + 'define void @"scr new"() {\n}\n',
|
||||
const before = objectFunctionHashes(objdump(0, "scr_g"));
|
||||
const after = objectFunctionHashes(
|
||||
objdump(0x40, "scr_h") + "<scr new>:\n \tretq\n",
|
||||
);
|
||||
// Renumbered attribute groups and metadata are not changes.
|
||||
// A function that only moved within its section is not a change.
|
||||
expect(changedFunctions(before, after)).toEqual({
|
||||
changed: ["scr_changed"],
|
||||
added: ["scr new"],
|
||||
|
||||
Reference in New Issue
Block a user