diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 98e07e9ce..6a2574538 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -1340,6 +1340,44 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, } // namespace +std::optional optimizeModule(Module &M, TargetMachine &Machine, + const std::string &OptLevel, + bool Instrument, + const std::string &ProfileGenerate, + const std::string &ProfileUse, + ModuleProfileMatch &Result) { + OptimizationLevel Level = optimizationLevel(OptLevel); + std::optional Profile; + if (Instrument) { + Profile = PGOOptions(ProfileGenerate, "", "", "", PGOOptions::IRInstr); + } else if (!ProfileUse.empty()) { + EmitOptions Inspected; + Inspected.ProfileUse = ProfileUse; + if (std::optional Failure = inspectProfile(Inspected)) + return Failure->Message; + reportMissingProfiles(); + Profile = PGOOptions(ProfileUse, "", "", "", PGOOptions::IRUse); + } + bool Annotating = Profile && Profile->Action == PGOOptions::IRUse; + ProfileMatch Match; + if (Annotating) + M.getContext().setDiagnosticHandler( + std::make_unique(Match)); + { + Pipeline P(Machine, Level, std::move(Profile)); + if (Annotating) + P.countAnnotated(Match.Annotated); + P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); + } + Result.Annotated = Match.Annotated.load(); + Result.Mismatched = Match.Mismatched.load(); + Result.Missing = Match.Missing.load(); + if (std::optional Failure = + verify(M, "post_optimization_verification_failed")) + return Failure->Message; + return std::nullopt; +} + int emit(const EmitOptions &Options) { if (!supportsTarget(Options.Target)) return reportError("unsupported_target", diff --git a/native/llvm-codegen/src/emit.h b/native/llvm-codegen/src/emit.h index 1bdc578df..dd02cfd98 100644 --- a/native/llvm-codegen/src/emit.h +++ b/native/llvm-codegen/src/emit.h @@ -1,9 +1,15 @@ #pragma once +#include #include #include #include +namespace llvm { +class Module; +class TargetMachine; +} // namespace llvm + namespace scriptc { struct EmitOptions { @@ -36,4 +42,27 @@ struct EmitOptions { std::optional parseEmitOptions(int Argc, char **Argv); int emit(const EmitOptions &Options); +// How a profile matched the functions of one optimized module: functions +// that received its counts, functions whose control flow changed since it +// was collected, and functions it has no record of. +struct ModuleProfileMatch { + uint64_t Annotated = 0; + uint64_t Mismatched = 0; + uint64_t Missing = 0; +}; + +// Optimizes `M` with the per-module default pipeline at `OptLevel`, as `emit` +// does for a whole program, with IR instrumentation when `Instrument` is set +// or annotation from the indexed `ProfileUse` profile. Instrumentation writes +// raw profiles to the `ProfileGenerate` pattern; an empty pattern names no +// file, so the program that links the module names it. Returns an error +// message, or nothing on success. +std::optional optimizeModule(llvm::Module &M, + llvm::TargetMachine &Machine, + const std::string &OptLevel, + bool Instrument, + const std::string &ProfileGenerate, + const std::string &ProfileUse, + ModuleProfileMatch &Match); + } // namespace scriptc diff --git a/native/llvm-codegen/src/main.cpp b/native/llvm-codegen/src/main.cpp index 57416106a..d9481395d 100644 --- a/native/llvm-codegen/src/main.cpp +++ b/native/llvm-codegen/src/main.cpp @@ -58,6 +58,9 @@ int version(int Argc, char **Argv) { {"supported_targets", std::move(SupportedTargets)}, {"default_target", scriptc::DefaultTarget}, {"data_layout", Machine->createDataLayout().getStringRepresentation()}, + // Commands and options added within one protocol version, for callers + // that also run against older helpers. + {"features", json::Array{"runtime-unit-profile", "profile-functions"}}, }; outs() << formatv("{0}\n", json::Value(std::move(Response))); return 0; @@ -80,6 +83,8 @@ int main(int Argc, char **Argv) { return scriptc::pruneImport(Argc, Argv); if (Argc >= 2 && StringRef(Argv[1]) == "merge-profiles") return scriptc::mergeProfiles(Argc, Argv); + if (Argc >= 2 && StringRef(Argv[1]) == "profile-functions") + return scriptc::profileFunctions(Argc, Argv); if (Argc >= 2 && StringRef(Argv[1]) == "emit") { std::optional Options = scriptc::parseEmitOptions(Argc, Argv); @@ -95,5 +100,6 @@ int main(int Argc, char **Argv) { } return scriptc::reportError("usage", "expected 'version --format=json', 'emit', 'runtime-unit', " - "'prune-import', or 'merge-profiles'"); + "'prune-import', 'merge-profiles', or " + "'profile-functions'"); } diff --git a/native/llvm-codegen/src/profile.cpp b/native/llvm-codegen/src/profile.cpp index b7df1445d..e06d051c9 100644 --- a/native/llvm-codegen/src/profile.cpp +++ b/native/llvm-codegen/src/profile.cpp @@ -2,10 +2,14 @@ #include "diagnostics.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/ProfileData/InstrProfWriter.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" @@ -19,6 +23,7 @@ namespace scriptc { int mergeProfiles(int Argc, char **Argv) { std::string Output; bool Sparse = false; + std::string KeepFunctions; std::vector Inputs; for (int I = 2; I < Argc; ++I) { StringRef Arg(Argv[I]); @@ -26,17 +31,35 @@ int mergeProfiles(int Argc, char **Argv) { Output = Argv[++I]; else if (Arg == "--sparse") Sparse = true; + else if (Arg == "--keep-functions" && I + 1 < Argc) + KeepFunctions = Argv[++I]; else if (!Arg.starts_with("--")) Inputs.push_back(Arg.str()); else return reportError("usage", "merge-profiles requires --output " "and one or more input profiles, and " - "accepts --sparse"); + "accepts --sparse and --keep-functions " + ""); } if (Output.empty() || Inputs.empty()) return reportError("usage", "merge-profiles requires --output and " "one or more input profiles"); + // --keep-functions names the functions to keep, one profile name per + // line; the records of every other function are left out. + StringSet<> Kept; + if (!KeepFunctions.empty()) { + ErrorOr> List = + MemoryBuffer::getFile(KeepFunctions); + if (!List) + return reportError("profile_read_failed", + KeepFunctions + ": " + List.getError().message()); + SmallVector Names; + (*List)->getBuffer().split(Names, '\n', -1, /*KeepEmpty=*/false); + for (StringRef Name : Names) + Kept.insert(Name); + } + // Sparse profiles leave out the functions that never ran, which then // optimize as without a profile instead of as cold. InstrProfWriter Writer(Sparse); @@ -52,6 +75,8 @@ int mergeProfiles(int Argc, char **Argv) { Path + ": " + toString(std::move(E))); std::string RecordError; for (NamedInstrProfRecord &Record : **Reader) { + if (!KeepFunctions.empty() && !Kept.contains(Record.Name)) + continue; Writer.addRecord(std::move(Record), /*Weight=*/1, [&](Error E) { if (RecordError.empty()) RecordError = toString(std::move(E)); @@ -100,4 +125,26 @@ int mergeProfiles(int Argc, char **Argv) { return 0; } +int profileFunctions(int Argc, char **Argv) { + if (Argc != 4 || StringRef(Argv[2]) != "--input") + return reportError("usage", + "profile-functions requires --input "); + std::string Path = Argv[3]; + IntrusiveRefCntPtr FS = vfs::getRealFileSystem(); + Expected> Reader = + InstrProfReader::create(Path, *FS); + if (!Reader) + return reportError("profile_read_failed", + Path + ": " + toString(Reader.takeError())); + json::Array Functions; + for (const NamedInstrProfRecord &Record : **Reader) + Functions.push_back(Record.Name.str()); + if ((*Reader)->hasError()) + return reportError("profile_read_failed", + Path + ": " + toString((*Reader)->getError())); + json::Object Response{{"ok", true}, {"functions", std::move(Functions)}}; + outs() << formatv("{0}\n", json::Value(std::move(Response))); + return 0; +} + } // namespace scriptc diff --git a/native/llvm-codegen/src/profile.h b/native/llvm-codegen/src/profile.h index 1cad1c54b..8602889a3 100644 --- a/native/llvm-codegen/src/profile.h +++ b/native/llvm-codegen/src/profile.h @@ -2,9 +2,14 @@ namespace scriptc { -// `merge-profiles --output [--sparse] ...`: merges raw -// (.profraw) or indexed (.profdata) IR profiles into one indexed profile. -// --sparse leaves out functions that never ran. +// `merge-profiles --output [--sparse] [--keep-functions +// ] ...`: merges raw (.profraw) or indexed (.profdata) IR +// profiles into one indexed profile. --sparse leaves out functions that never +// ran, and --keep-functions every function the file does not name. int mergeProfiles(int Argc, char **Argv); +// `profile-functions --input `: prints the names of the +// functions an indexed profile has records for, as {"ok":true,"functions":[]}. +int profileFunctions(int Argc, char **Argv); + } // namespace scriptc diff --git a/native/llvm-codegen/src/runtime-unit.cpp b/native/llvm-codegen/src/runtime-unit.cpp index e92c384ee..d97638c84 100644 --- a/native/llvm-codegen/src/runtime-unit.cpp +++ b/native/llvm-codegen/src/runtime-unit.cpp @@ -13,6 +13,7 @@ #include "runtime-unit.h" #include "diagnostics.h" +#include "emit.h" #include "runtime-import.h" #include "target.h" @@ -27,12 +28,16 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" #include "llvm/IRReader/IRReader.h" +#include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/SizeOpts.h" #include #include @@ -53,6 +58,16 @@ struct UnitOptions { std::string Tag; bool FunctionSections = false; bool DataSections = false; + // When set, the input is unoptimized and the helper optimizes it at this + // level, with the profile-guided optimization below; otherwise the input + // is already optimized and is only promoted and emitted. + std::string OptLevel; + // Instruments the unit for IR profiling; raw profiles go to this pattern, + // or, when it is empty, to the file the program that links the unit names. + bool ProfileGenerate = false; + std::string ProfileGeneratePattern; + // Indexed profile (.profdata) that guides the unit's optimization. + std::string ProfileUse; }; std::optional parse(int Argc, char **Argv) { @@ -80,12 +95,26 @@ std::optional parse(int Argc, char **Argv) { Options.Target = Value.str(); else if (Arg == "--tag") Options.Tag = Value.str(); + else if (Arg == "--opt-level") + Options.OptLevel = Value.str(); + else if (Arg == "--profile-generate") { + Options.ProfileGenerate = true; + Options.ProfileGeneratePattern = Value.str(); + } else if (Arg == "--profile-use") + Options.ProfileUse = Value.str(); else return std::nullopt; } if (Options.Input.empty() || Options.Object.empty() || Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty()) return std::nullopt; + // A profile only guides an optimization the helper runs, and a unit is + // either instrumented or annotated. + if ((Options.ProfileGenerate || !Options.ProfileUse.empty()) && + Options.OptLevel.empty()) + return std::nullopt; + if (Options.ProfileGenerate && !Options.ProfileUse.empty()) + return std::nullopt; return Options; } @@ -98,10 +127,22 @@ bool keepLocal(const GlobalValue &GV) { return false; } +// The profile's counters and per-function records (instrumented units only) +// stay private to the unit: the profile runtime finds them by section. +bool profileData(const GlobalValue &GV) { + if (const auto *Object = dyn_cast(&GV)) + if (Object->getSection().starts_with("__llvm_prf")) + return true; + return GV.getName().starts_with("__llvm_prf") || + GV.getName().starts_with("__profc_") || + GV.getName().starts_with("__profd_") || + GV.getName().starts_with("__profvp_"); +} + void promoteLocals(Module &M, StringRef Tag) { unsigned Anonymous = 0; auto Promote = [&](GlobalValue &GV) { - if (!GV.hasLocalLinkage() || keepLocal(GV)) + if (!GV.hasLocalLinkage() || keepLocal(GV) || profileData(GV)) return; std::string Base = GV.hasName() ? GV.getName().str() : ("anon." + Twine(Anonymous++)).str(); @@ -157,7 +198,9 @@ int runtimeUnit(int Argc, char **Argv) { return fail("usage", "runtime-unit requires --input --object --bitcode " "--target --tag and accepts " - "--function-sections --data-sections"); + "--function-sections --data-sections, and --opt-level " + "with either --profile-generate or --profile-use " + ""); LLVMContext Context; SMDiagnostic Diagnostic; std::unique_ptr M = parseIRFile(Options->Input, Diagnostic, Context); @@ -208,6 +251,42 @@ int runtimeUnit(int Argc, char **Argv) { if (F.getFnAttribute("probe-stack").getValueAsString() == "__chkstk_darwin") F.addFnAttr("probe-stack", "inline-asm"); + // Profiles name a unit's internal functions after its source file. The + // unit's own name keeps those names the same in every checkout, so a + // committed profile matches wherever the pack is built. + if (!Options->OptLevel.empty()) { + M->setSourceFileName(Options->Tag + ".c"); + // The runtime profile comes from a fixed set of training programs, and a + // runtime function they rarely call can be hot in another program (a + // long UTF-8 decode, a busy worker channel). Code the profile finds cold + // is therefore optimized for speed as without a profile, not for size: + // the profile still guides inlining, block layout and branch weights. + EnablePGSO.setValue(false); + // The profile names of the functions an instrumented unit defines, so + // a merged profile can keep the runtime's records and drop the + // training programs'. + json::Array Functions; + if (Options->ProfileGenerate) + for (const Function &F : *M) + if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage()) + Functions.push_back(getIRPGOFuncName(F)); + ModuleProfileMatch Match; + if (std::optional Failure = optimizeModule( + *M, *Machine, Options->OptLevel, Options->ProfileGenerate, + Options->ProfileGeneratePattern, Options->ProfileUse, Match)) + return fail("optimization_failed", *Failure); + if (!Options->ProfileUse.empty()) { + json::Object Profile{{"annotated", Match.Annotated}, + {"mismatched", Match.Mismatched}, + {"missing", Match.Missing}}; + json::Object Response{{"ok", true}, {"profile", std::move(Profile)}}; + outs() << formatv("{0}\n", json::Value(std::move(Response))); + } else if (Options->ProfileGenerate) { + json::Object Response{{"ok", true}, {"functions", std::move(Functions)}}; + outs() << formatv("{0}\n", json::Value(std::move(Response))); + } + } + promoteLocals(*M, Options->Tag); std::string VerifyError; raw_string_ostream VerifyStream(VerifyError); diff --git a/packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz b/packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz new file mode 100644 index 000000000..6d4a0d942 Binary files /dev/null and b/packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz differ diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index b6305555e..07de35369 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -2,10 +2,12 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { availableParallelism, tmpdir } from "node:os"; +import { existsSync } from "node:fs"; import { copyFile, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { dirname, join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { gunzipSync } from "node:zlib"; import { createRuntimePackMatrix } from "../runtime-pack-matrix.mjs"; import { RUNTIME_COMPILE_FLAGS, @@ -118,6 +120,44 @@ async function build() { config.bitcode === false ? null : await resolveRuntimeUnitHelper(repoRoot, matrix.target.llvm_triple); + // Speed units that the LLVM helper optimizes with the target's committed + // runtime profile (profiles/.profdata.gz), or instruments for + // scripts/refresh-runtime-profile.mjs (SCRIPTC_RUNTIME_PROFILE_GENERATE=1). + // The runtime compiler then emits their bitcode unoptimized, and one LLVM, + // the helper's, instruments, annotates and optimizes every profiled unit. + // Release and library flavors keep the runtime compiler's optimization. + const committedProfile = join( + repoRoot, + "packages", + "runtime-pack-common", + "profiles", + `${config.target.name}.profdata.gz`, + ); + // The committed profile is unpacked into the build directory once it exists. + const unpackedProfile = join(buildRoot, "runtime.profdata"); + // Without a helper that optimizes units itself, speed units are built as + // before, without the profile. + const unitProfiles = unitHelper?.features.includes("runtime-unit-profile") === true; + if (process.env.SCRIPTC_RUNTIME_PROFILE_GENERATE === "1" && !unitProfiles) + throw new Error( + "SCRIPTC_RUNTIME_PROFILE_GENERATE=1 needs an LLVM helper with runtime-unit profiles", + ); + const speedProfile = !unitProfiles + ? null + : process.env.SCRIPTC_RUNTIME_PROFILE_GENERATE === "1" + ? { args: ["--opt-level", "2", "--profile-generate", ""] } + : existsSync(committedProfile) + ? { args: ["--opt-level", "2", "--profile-use", unpackedProfile], unpack: true } + : null; + // Speed units the helper optimizes without the profile (at the same level). + // scr_message.c: with the profile on its decoding, a program whose workers + // each receive many small messages ran 5% slower in its workers' own code, + // although that code and its data were unchanged; without it, the program + // runs as fast as with an unprofiled runtime. + const unprofiledUnits = new Set(["scr_message.c"]); + // The instrumented units' functions, which SCRIPTC_RUNTIME_PROFILE_FUNCTIONS + // names a file for; the refresh script keeps only their records. + const profiledFunctions = new Set(); const parallel = async (items, task) => { const width = Math.max(1, Math.min(8, availableParallelism())); return parallelMap(items, width, task); @@ -156,6 +196,8 @@ async function build() { await rm(buildRoot, { recursive: true, force: true }); try { await mkdir(stagedOutputRoot, { recursive: true }); + if (speedProfile?.unpack) + await writeFile(unpackedProfile, gunzipSync(await readFile(committedProfile))); // Zig 0.16 creates a zero-byte `a.o` in its working directory for // `zig cc --version`. Probe from a private temporary directory so a // runtime-pack build never leaves that compiler byproduct in the package. @@ -197,6 +239,7 @@ async function build() { ...(config.platform === "linux" ? ["-fomit-frame-pointer"] : []), ] : []; + const profileMatches = []; const units = await parallel( flavorSpec.runtime_units ?? matrix.runtime_units, async (unit) => { @@ -249,9 +292,10 @@ async function build() { let bitcode; const bitcodeOutput = output.replace(/\.o$/, ".bc"); if (flavor === "speed") await mkdir(dirname(output), { recursive: true }); + let unitBuild = false; if ( flavor === "speed" && - (await buildRuntimeUnit({ + (unitBuild = await buildRuntimeUnit({ helper: unitHelper, compileBitcode: (optimized) => run(compiler, [ @@ -259,12 +303,17 @@ async function build() { ...sourcePathFlags, ...unitFlags, "-g0", + ...(speedProfile === null ? [] : ["-Xclang", "-disable-llvm-passes"]), "-emit-llvm", "-c", join(runtimeSrc, unit.source), "-o", optimized, ]), + helperArgs: + speedProfile?.unpack && unprofiledUnits.has(unit.source) + ? ["--opt-level", "2"] + : (speedProfile?.args ?? []), object: output, bitcode: bitcodeOutput, triple: matrix.target.llvm_triple, @@ -275,6 +324,8 @@ async function build() { ), })) ) { + if (unitBuild.profile !== undefined) profileMatches.push(unitBuild.profile); + for (const name of unitBuild.functions ?? []) profiledFunctions.add(name); bitcode = { path: artifactPath(bitcodeOutput), sha256: await sha256(bitcodeOutput), @@ -324,7 +375,21 @@ async function build() { return { source: unit.source, predicate: unit.predicate, variants }; }, ); - flavors[flavor] = { optimization: flavorSpec.optimization, runtime_units: units }; + flavors[flavor] = { + optimization: flavorSpec.optimization, + runtime_units: units, + // How the committed runtime profile matched the units it guided + // (scripts/refresh-runtime-profile.mjs refreshes it). + ...(profileMatches.length === 0 + ? {} + : { + runtime_profile: { + annotated: profileMatches.reduce((sum, match) => sum + match.annotated, 0), + mismatched: profileMatches.reduce((sum, match) => sum + match.mismatched, 0), + missing: profileMatches.reduce((sum, match) => sum + match.missing, 0), + }, + }), + }; } const mbedtlsSources = (await readdir(join(mbedtls, "library"))) .filter((name) => !name.startsWith(".") && name.endsWith(".c")) @@ -505,6 +570,14 @@ async function build() { backupRoot: join(packageRoot, `.runtime-pack-artifacts-backup-${suffix}`), backupManifestPath: join(packageRoot, `.runtime-pack-manifest-backup-${suffix}`), }); + if (process.env.SCRIPTC_RUNTIME_PROFILE_FUNCTIONS !== undefined) + await writeFile( + process.env.SCRIPTC_RUNTIME_PROFILE_FUNCTIONS, + [...profiledFunctions] + .sort() + .map((name) => `${name}\n`) + .join(""), + ); process.stdout.write( `built ${packageManifest.name}@${packageManifest.version}: ${Object.keys(flavors).length} flavors, ${archives.length} vendor archives\n`, ); diff --git a/packages/runtime-pack-common/scripts/runtime-unit.mjs b/packages/runtime-pack-common/scripts/runtime-unit.mjs index 1d4b549b6..68bc62381 100644 --- a/packages/runtime-pack-common/scripts/runtime-unit.mjs +++ b/packages/runtime-pack-common/scripts/runtime-unit.mjs @@ -12,10 +12,13 @@ const run = promisify(execFile); * Speed units produce both artifacts from one optimized LLVM module: the * runtime compiler optimizes the C source to bitcode, and the host helper's * `runtime-unit` command promotes translation-unit-local symbols to hidden - * unit-qualified globals and emits the object and the bitcode. Release - * units keep the runtime compiler's own objects; their bitcode is the same - * compilation with -emit-llvm, which the helper's `prune-import` reduces to - * the bodies that need none of the unit's local state. Both bitcode forms + * unit-qualified globals and emits the object and the bitcode. With the + * target's committed runtime profile, the runtime compiler emits the bitcode + * unoptimized and the helper also optimizes it with the profile (build.mjs), + * so one LLVM reads and writes every profile. Release units keep the + * runtime compiler's own objects; their bitcode is the same compilation with + * -emit-llvm, which the helper's `prune-import` reduces to the bodies that + * need none of the unit's local state. Both bitcode forms * keep only what a program build can import. * * Every host helper supports every scriptc target, so packs keep building @@ -54,7 +57,15 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) { const { stdout } = await run(helper, ["version", "--format=json"]); const version = JSON.parse(stdout); if (!version.supported_targets?.includes(triple)) return null; - return { path: helper, identity: `scriptc-llvm-codegen ${version.llvm_version}` }; + return { + path: helper, + identity: `scriptc-llvm-codegen ${version.llvm_version}`, + // Commands and options newer than the protocol version, such as + // "runtime-unit-profile": `runtime-unit` takes --opt-level, + // --profile-use and --profile-generate (older helpers only promote and + // emit). + features: Array.isArray(version.features) ? version.features : [], + }; } catch { if (explicit !== undefined) throw new Error(`SCRIPTC_LLVM_CODEGEN is not usable: ${helper}`); return null; @@ -63,12 +74,19 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) { /** Compile one unit through the helper. `compileBitcode(output)` must run * the runtime compiler with the unit's exact flags plus -emit-llvm. + * `helperArgs` asks the helper to optimize the unit itself (`--opt-level`, + * with `--profile-use` or `--profile-generate`), for bitcode the runtime + * compiler emitted unoptimized. * Returns false, leaving no artifacts behind, when the helper cannot emit * this unit; the caller then compiles the object directly without bitcode, - * so a pack never fails to build because of the import path. */ + * so a pack never fails to build because of the import path. Otherwise + * returns an object, with the helper's `profile` match counts when a profile + * guided the unit, or the profile names of the unit's `functions` when it + * was instrumented. */ export async function buildRuntimeUnit({ helper, compileBitcode, + helperArgs = [], object, bitcode, triple, @@ -79,7 +97,7 @@ export async function buildRuntimeUnit({ // A runtime compiler failure is a real build error and surfaces as one. await compileBitcode(optimized); try { - await run(helper.path, [ + const { stdout } = await run(helper.path, [ "runtime-unit", "--input", optimized, @@ -92,8 +110,13 @@ export async function buildRuntimeUnit({ "--tag", tag, ...(sections ? ["--function-sections", "--data-sections"] : []), + ...helperArgs, ]); - return true; + const report = stdout.trim() === "" ? {} : JSON.parse(stdout.trim().split("\n").pop()); + return { + ...(report.profile === undefined ? {} : { profile: report.profile }), + ...(report.functions === undefined ? {} : { functions: report.functions }), + }; } catch (error) { const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0]; process.stderr.write(`runtime-unit: ${tag}: emitting without import bitcode (${detail})\n`); diff --git a/packages/runtime-pack-common/test/runtime-profile.test.ts b/packages/runtime-pack-common/test/runtime-profile.test.ts new file mode 100644 index 000000000..0b2bf19a4 --- /dev/null +++ b/packages/runtime-pack-common/test/runtime-profile.test.ts @@ -0,0 +1,98 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { gunzipSync } from "node:zlib"; +import { expect, test } from "vitest"; +import { resolveRuntimeUnitHelper } from "../scripts/runtime-unit.mjs"; + +const repoRoot = fileURLToPath(new URL("../../..", import.meta.url)); +const committedProfile = join( + repoRoot, + "packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz", +); +const manifestPath = join(repoRoot, "packages/runtime-linux-x64-gnu/runtime-pack.json"); +const runtimeRoot = join(repoRoot, "packages/runtime"); + +// Both tests read the committed profile through an LLVM helper new enough to +// apply it; the pack test also needs the Linux x64 pack built with it. +const helper = + process.platform === "linux" && process.arch === "x64" && existsSync(committedProfile) + ? await resolveRuntimeUnitHelper(repoRoot, "x86_64-unknown-linux-gnu") + : null; +const built = + helper?.features.includes("runtime-unit-profile") === true && existsSync(manifestPath); + +// A profile older than the runtime sources loses the counts of every function +// whose control flow changed since it was collected. More than this share of +// the functions it describes means it is due for scripts/refresh-runtime-profile.mjs. +const MAX_MISMATCHED_SHARE = 0.1; + +test.skipIf(!built)("the committed runtime profile still matches the runtime's speed units", () => { + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + const match = manifest.flavors?.speed?.runtime_profile; + expect(match, "the speed flavor was built without the committed runtime profile").toBeDefined(); + const described = match.annotated + match.mismatched; + expect(match.annotated).toBeGreaterThan(0); + expect( + match.mismatched / described, + `${match.mismatched} of the ${described} runtime functions the profile describes changed since it was collected; run node scripts/refresh-runtime-profile.mjs`, + ).toBeLessThanOrEqual(MAX_MISMATCHED_SHARE); +}); + +// The committed profile holds the runtime's records only. Training programs' +// functions, and the source paths their names can carry, stay out of the +// repository: refresh-runtime-profile.mjs keeps the functions the +// instrumented runtime units define. +test.skipIf(helper?.features.includes("profile-functions") !== true)( + "the committed runtime profile names only runtime functions", + () => { + const profile = gunzipSync(readFileSync(committedProfile)); + const scratch = mkdtempSync(join(tmpdir(), "scriptc-runtime-profile-")); + let functions: string[]; + try { + const unpacked = join(scratch, "runtime.profdata"); + writeFileSync(unpacked, profile); + ({ functions } = JSON.parse( + execFileSync(helper!.path, ["profile-functions", "--input", unpacked], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }), + )); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + expect(functions.length).toBeGreaterThan(0); + + const sources = new Set(readdirSync(join(runtimeRoot, "src"))); + const identifiers = new Set(); + const collect = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) collect(path); + else if (/\.(c|h|inc|def)$/.test(entry.name)) + for (const name of readFileSync(path, "latin1").match(/[A-Za-z_]\w*/g) ?? []) + identifiers.add(name); + } + }; + collect(join(runtimeRoot, "src")); + collect(join(runtimeRoot, "vendor")); + + // A unit's internal functions are named ".c;" after its + // runtime source, and its external ones by their C names, which are never + // a program's `main` or generated `sc_` symbols. + const foreign = functions.filter((name) => { + if (/[/\\]/.test(name)) return true; + const local = /^([\w-]+\.c);(.+)$/.exec(name); + if (local !== null) return !sources.has(local[1]!); + return name === "main" || name.startsWith("sc_") || !identifiers.has(name); + }); + expect(foreign, "records that are not runtime functions").toEqual([]); + + // Nor does any other part of the file hold a path, such as a checkout's. + const text = profile.toString("latin1"); + const paths = text.match(/(?:\/[\w.-]+){3,}|[A-Za-z]:\\[\w.-]+\\|\.tsx?;/g) ?? []; + expect(paths, "path strings in the committed profile").toEqual([]); + }, +); diff --git a/scripts/bench-runtime.mjs b/scripts/bench-runtime.mjs index 0845d57b8..63f3fa2d6 100644 --- a/scripts/bench-runtime.mjs +++ b/scripts/bench-runtime.mjs @@ -44,6 +44,7 @@ const { values } = parseArgs({ "candidate-target-cpu": { type: "string" }, "baseline-cpu-dispatch": { type: "boolean", default: false }, "candidate-cpu-dispatch": { type: "boolean", default: false }, + "candidate-profile-generate": { type: "string" }, json: { type: "string" }, "no-lock": { type: "boolean", default: false }, node: { type: "boolean", default: true }, @@ -57,6 +58,7 @@ if (values.help) { [--baseline-optimization=release] [--candidate-optimization=release] [--baseline-target-cpu=] [--candidate-target-cpu=] [--baseline-cpu-dispatch] [--candidate-cpu-dispatch] + [--candidate-profile-generate=] [--no-lock] [--no-node] [--keep] Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native @@ -79,7 +81,10 @@ release. --baseline-target-cpu/--candidate-target-cpu pass --target-cpu to each contender's builds, e.g. --baseline=. --candidate-target-cpu=x86-64-v3 compares x86-64-v3 code with the baseline. --baseline-cpu-dispatch/--candidate-cpu-dispatch pass --cpu-dispatch, and SCRIPTC_CPU_LEVEL in the environment reaches every timed -run. cold_build_ms records each contender's first, uncached build.`); +run. cold_build_ms records each contender's first, uncached build. +--candidate-profile-generate builds the candidate's workloads with +--profile-generate=, so its runs write raw profiles there +(scripts/refresh-runtime-profile.mjs trains the runtime profile this way).`); process.exit(0); } const runs = Number(values.runs); @@ -448,7 +453,16 @@ function revision(root) { }); return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : ""); } -function build(label, root, workload, layout, optimization, targetCpu, cpuDispatch) { +function build( + label, + root, + workload, + layout, + optimization, + targetCpu, + cpuDispatch, + profileGenerate, +) { const cli = cliFor(root); const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`); mkdirSync(dirname(out), { recursive: true }); @@ -471,6 +485,7 @@ function build(label, root, workload, layout, optimization, targetCpu, cpuDispat `--optimization=${optimization}`, ...(targetCpu === undefined ? [] : [`--target-cpu=${targetCpu}`]), ...(cpuDispatch ? ["--cpu-dispatch"] : []), + ...(profileGenerate === undefined ? [] : [`--profile-generate=${profileGenerate}`]), "-o", out, ], @@ -507,6 +522,7 @@ const contenders = [ optimization: values["candidate-optimization"], targetCpu: values["candidate-target-cpu"], cpuDispatch: values["candidate-cpu-dispatch"], + profileGenerate: values["candidate-profile-generate"], }, ]; const report = { @@ -613,7 +629,16 @@ try { entry.results[c.label] = result; result.binaries = []; for (let layout = 0; layout < layouts && result.status === undefined; layout++) { - const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu, c.cpuDispatch); + const built = build( + c.label, + c.root, + w, + layout, + c.optimization, + c.targetCpu, + c.cpuDispatch, + c.profileGenerate, + ); result.build_ms += Math.round(built.buildMs); if (layout === 0) result.cold_build_ms = Math.round(built.buildMs); if (!built.ok) { diff --git a/scripts/refresh-runtime-profile.mjs b/scripts/refresh-runtime-profile.mjs new file mode 100644 index 000000000..608abb5de --- /dev/null +++ b/scripts/refresh-runtime-profile.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +/* Refreshes packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz, + * the profile that the Linux x64 runtime pack's speed units are optimized + * with (runtime-pack-common/scripts/build.mjs). The LLVM helper instruments, + * annotates and optimizes those units, so the profile is in the helper's + * format and merged by the helper's merge-profiles, like every program + * profile. + * + * Builds an instrumented runtime pack (SCRIPTC_RUNTIME_PROFILE_GENERATE=1), + * then runs the training programs built against it with --profile-generate: + * every runtime benchmark workload at --optimization=speed (through + * scripts/bench-runtime.mjs, with its inputs and load clients), and, with + * --tsc-ts, an instrumented tsc-ts checking its own sources and each + * --project. The raw profiles hold the runtime's counters beside the + * programs'; the merged profile keeps the runtime functions that ran: the + * instrumented pack lists its functions, and the programs' records are left + * out. The pack is then rebuilt with the new profile. A stale profile only + * loses part of its gain: functions whose code changed are optimized as + * without one (the pack test reports how many). + * + * node scripts/refresh-runtime-profile.mjs [--tsc-ts [--project [:]]...] + */ +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { constants, gzipSync } from "node:zlib"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const { values } = parseArgs({ + options: { + "tsc-ts": { type: "string" }, + project: { type: "string", multiple: true, default: [] }, + output: { + type: "string", + default: join(root, "packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz"), + }, + work: { type: "string" }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help) { + console.log( + "Usage: node scripts/refresh-runtime-profile.mjs [--tsc-ts [--project [:]]...] [--output ] [--work ]", + ); + process.exit(0); +} +if (process.platform !== "linux" || process.arch !== "x64") + throw new Error("the runtime profile is collected on Linux x64"); +if (values.project.length !== 0 && values["tsc-ts"] === undefined) + throw new Error("--project needs --tsc-ts"); +const cli = join(root, "packages/cli/dist/main.js"); +if (!existsSync(cli)) throw new Error("build the compiler first (pnpm build)"); +const helper = join(root, "packages/llvm-linux-x64-gnu/bin/scriptc-llvm-codegen"); +if (!existsSync(helper)) + throw new Error( + "build the LLVM helper first (pnpm --filter @scriptc/llvm-linux-x64-gnu build:native)", + ); +const output = resolve(values.output); +mkdirSync(dirname(output), { recursive: true }); +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-runtime-profile-"))); +const raw = join(work, "raw"); +rmSync(raw, { recursive: true, force: true }); +mkdirSync(raw, { recursive: true }); +const env = { ...process.env }; +delete env.SCRIPTC_LLVM_HELPER; + +const buildPack = (extra) => + run("pnpm", ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"], { + cwd: root, + env: { ...env, ...extra }, + }); + +const functions = join(work, "runtime-functions.txt"); +console.log("building the instrumented runtime pack"); +buildPack({ SCRIPTC_RUNTIME_PROFILE_GENERATE: "1", SCRIPTC_RUNTIME_PROFILE_FUNCTIONS: functions }); +try { + console.log("training on the runtime benchmark workloads"); + run( + process.execPath, + [ + join(root, "scripts/bench-runtime.mjs"), + `--candidate=${root}`, + "--candidate-optimization=speed", + `--candidate-profile-generate=${join(raw, "bench-%m-%p.profraw")}`, + "--runs=3", + "--warmup=0", + "--no-node", + "--no-lock", + `--json=${join(work, "bench.json")}`, + ], + { cwd: root, env }, + ); + if (values["tsc-ts"] !== undefined) trainTscTs(resolve(values["tsc-ts"])); + const profiles = readdirSync(raw).map((name) => join(raw, name)); + if (profiles.length === 0) throw new Error(`no raw profiles in ${raw}`); + // `scriptc merge-profiles` runs this merge too; called directly, it keeps + // only the runtime's records, so the training programs' stay out of the repo. + const merged = join(work, "runtime.profdata"); + run( + helper, + ["merge-profiles", "--output", merged, "--sparse", "--keep-functions", functions, ...profiles], + { cwd: root, env }, + ); + writeFileSync(output, gzipSync(readFileSync(merged), { level: constants.Z_BEST_COMPRESSION })); + console.log(`wrote ${output} from ${profiles.length} raw profiles`); +} finally { + // The pack goes back to an ordinary build, now with the new profile. + console.log("rebuilding the runtime pack"); + buildPack({}); +} +if (values.work === undefined) rmSync(work, { recursive: true, force: true }); + +// An instrumented tsc-ts, built against the instrumented pack, checking its +// own sources and each --project at --checkers 1. +function trainTscTs(checkout) { + const pkg = join(checkout, "packages/tsc-ts"); + if (!existsSync(join(pkg, "src/main.ts"))) + throw new Error(`${checkout} is not a tsc-ts checkout`); + // The upstream lib files, where npm run fetch:upstream puts them (older + // tsc-ts checkouts used the second directory). + const lib = [ + "upstream/typescript-go/internal/bundled/libs", + "upstream/TypeScript/tsc/internal/bundled/libs", + ] + .map((dir) => join(checkout, dir)) + .find((dir) => existsSync(join(dir, "lib.d.ts"))); + if (lib === undefined) + throw new Error( + `no lib files in ${checkout}/upstream: run npm run fetch:upstream in the tsc-ts checkout`, + ); + const tsc = join(work, "tsc-ts"); + console.log("building an instrumented tsc-ts"); + run( + process.execPath, + [ + cli, + "build", + "src/main.ts", + "--optimization=speed", + "--detached-exit", + `--profile-generate=${join(raw, "tsc-ts-%p.profraw")}`, + "-o", + tsc, + ], + { cwd: pkg, env }, + ); + const projects = [ + { dir: pkg, config: "tsconfig.json" }, + ...values.project.map((spec) => { + const [dir, config = "tsconfig.json"] = spec.split(":"); + return { dir: resolve(dir), config }; + }), + ]; + for (const { dir, config } of projects) { + // Projects with type errors exit with 1 or 2; any other status is a failed run. + const result = spawnSync( + tsc, + ["-p", config, "--noEmit", "--incremental", "false", "--checkers", "1"], + { + cwd: dir, + stdio: "ignore", + env: { ...env, TSC_TS_LIB_DIR: lib }, + }, + ); + if (result.status === null || result.status > 2) + throw new Error( + `the instrumented tsc-ts failed on ${join(dir, config)} (${result.signal ?? result.status})`, + ); + console.log(`profiled ${join(dir, config)}`); + } +} + +function run(command, args, options) { + const result = spawnSync(command, args, { ...options, stdio: "inherit" }); + if (result.status !== 0) + throw new Error(`${command} ${args.join(" ")} exited with ${result.status}`); +}