Optimize the speed runtime units with a committed runtime profile

The LLVM helper's runtime-unit now optimizes a unit itself (--opt-level),
with IR instrumentation (--profile-generate) or a profile (--profile-use),
through the same pipeline as program code. The Linux x64 pack's speed
units are compiled to unoptimized bitcode and optimized there with
packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz, so one
LLVM writes and reads every profile. Release and library flavors are
unchanged. The pack manifest records how the profile matched the units.

Code the runtime profile finds cold is optimized for speed, not size: a
runtime function the training programs rarely call can be hot in another
program. scr_message.c is optimized without the profile.

scripts/refresh-runtime-profile.mjs trains the profile on the runtime
benchmark workloads and, optionally, a tsc-ts checkout, and keeps only the
records of functions the instrumented runtime units define
(merge-profiles --keep-functions). Tests check that the profile still
matches the runtime and names only runtime functions.

native/llvm-codegen/src/emit.cpp

@@ -1340,6 +1340,44 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
 
 } // namespace
 
+std::optional<std::string> optimizeModule(Module &M, TargetMachine &Machine,
+                                          const std::string &OptLevel,
+                                          bool Instrument,
+                                          const std::string &ProfileGenerate,
+                                          const std::string &ProfileUse,
+                                          ModuleProfileMatch &Result) {
+  OptimizationLevel Level = optimizationLevel(OptLevel);
+  std::optional<PGOOptions> Profile;
+  if (Instrument) {
+    Profile = PGOOptions(ProfileGenerate, "", "", "", PGOOptions::IRInstr);
+  } else if (!ProfileUse.empty()) {
+    EmitOptions Inspected;
+    Inspected.ProfileUse = ProfileUse;
+    if (std::optional<EmitFailure> Failure = inspectProfile(Inspected))
+      return Failure->Message;
+    reportMissingProfiles();
+    Profile = PGOOptions(ProfileUse, "", "", "", PGOOptions::IRUse);
+  }
+  bool Annotating = Profile && Profile->Action == PGOOptions::IRUse;
+  ProfileMatch Match;
+  if (Annotating)
+    M.getContext().setDiagnosticHandler(
+        std::make_unique<ProfileMatchHandler>(Match));
+  {
+    Pipeline P(Machine, Level, std::move(Profile));
+    if (Annotating)
+      P.countAnnotated(Match.Annotated);
+    P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
+  }
+  Result.Annotated = Match.Annotated.load();
+  Result.Mismatched = Match.Mismatched.load();
+  Result.Missing = Match.Missing.load();
+  if (std::optional<EmitFailure> Failure =
+          verify(M, "post_optimization_verification_failed"))
+    return Failure->Message;
+  return std::nullopt;
+}
+
 int emit(const EmitOptions &Options) {
   if (!supportsTarget(Options.Target))
     return reportError("unsupported_target",

native/llvm-codegen/src/emit.h

@@ -1,9 +1,15 @@
 #pragma once
 
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <vector>
 
+namespace llvm {
+class Module;
+class TargetMachine;
+} // namespace llvm
+
 namespace scriptc {
 
 struct EmitOptions {
@@ -36,4 +42,27 @@ struct EmitOptions {
 std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv);
 int emit(const EmitOptions &Options);
 
+// How a profile matched the functions of one optimized module: functions
+// that received its counts, functions whose control flow changed since it
+// was collected, and functions it has no record of.
+struct ModuleProfileMatch {
+  uint64_t Annotated = 0;
+  uint64_t Mismatched = 0;
+  uint64_t Missing = 0;
+};
+
+// Optimizes `M` with the per-module default pipeline at `OptLevel`, as `emit`
+// does for a whole program, with IR instrumentation when `Instrument` is set
+// or annotation from the indexed `ProfileUse` profile. Instrumentation writes
+// raw profiles to the `ProfileGenerate` pattern; an empty pattern names no
+// file, so the program that links the module names it. Returns an error
+// message, or nothing on success.
+std::optional<std::string> optimizeModule(llvm::Module &M,
+                                          llvm::TargetMachine &Machine,
+                                          const std::string &OptLevel,
+                                          bool Instrument,
+                                          const std::string &ProfileGenerate,
+                                          const std::string &ProfileUse,
+                                          ModuleProfileMatch &Match);
+
 } // namespace scriptc

native/llvm-codegen/src/main.cpp

@@ -58,6 +58,9 @@ int version(int Argc, char **Argv) {
       {"supported_targets", std::move(SupportedTargets)},
       {"default_target", scriptc::DefaultTarget},
       {"data_layout", Machine->createDataLayout().getStringRepresentation()},
+      // Commands and options added within one protocol version, for callers
+      // that also run against older helpers.
+      {"features", json::Array{"runtime-unit-profile", "profile-functions"}},
   };
   outs() << formatv("{0}\n", json::Value(std::move(Response)));
   return 0;
@@ -80,6 +83,8 @@ int main(int Argc, char **Argv) {
     return scriptc::pruneImport(Argc, Argv);
   if (Argc >= 2 && StringRef(Argv[1]) == "merge-profiles")
     return scriptc::mergeProfiles(Argc, Argv);
+  if (Argc >= 2 && StringRef(Argv[1]) == "profile-functions")
+    return scriptc::profileFunctions(Argc, Argv);
   if (Argc >= 2 && StringRef(Argv[1]) == "emit") {
     std::optional<scriptc::EmitOptions> Options =
         scriptc::parseEmitOptions(Argc, Argv);
@@ -95,5 +100,6 @@ int main(int Argc, char **Argv) {
   }
   return scriptc::reportError("usage",
                               "expected 'version --format=json', 'emit', 'runtime-unit', "
-                              "'prune-import', or 'merge-profiles'");
+                              "'prune-import', 'merge-profiles', or "
+                              "'profile-functions'");
 }

native/llvm-codegen/src/profile.cpp

@@ -2,10 +2,14 @@
 #include "diagnostics.h"
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/InstrProfWriter.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -19,6 +23,7 @@ namespace scriptc {
 int mergeProfiles(int Argc, char **Argv) {
   std::string Output;
   bool Sparse = false;
+  std::string KeepFunctions;
   std::vector<std::string> Inputs;
   for (int I = 2; I < Argc; ++I) {
     StringRef Arg(Argv[I]);
@@ -26,17 +31,35 @@ int mergeProfiles(int Argc, char **Argv) {
       Output = Argv[++I];
     else if (Arg == "--sparse")
       Sparse = true;
+    else if (Arg == "--keep-functions" && I + 1 < Argc)
+      KeepFunctions = Argv[++I];
     else if (!Arg.starts_with("--"))
       Inputs.push_back(Arg.str());
     else
       return reportError("usage", "merge-profiles requires --output <file> "
                                   "and one or more input profiles, and "
-                                  "accepts --sparse");
+                                  "accepts --sparse and --keep-functions "
+                                  "<file>");
   }
   if (Output.empty() || Inputs.empty())
     return reportError("usage", "merge-profiles requires --output <file> and "
                                 "one or more input profiles");
 
+  // --keep-functions names the functions to keep, one profile name per
+  // line; the records of every other function are left out.
+  StringSet<> Kept;
+  if (!KeepFunctions.empty()) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> List =
+        MemoryBuffer::getFile(KeepFunctions);
+    if (!List)
+      return reportError("profile_read_failed",
+                         KeepFunctions + ": " + List.getError().message());
+    SmallVector<StringRef, 0> Names;
+    (*List)->getBuffer().split(Names, '\n', -1, /*KeepEmpty=*/false);
+    for (StringRef Name : Names)
+      Kept.insert(Name);
+  }
+
   // Sparse profiles leave out the functions that never ran, which then
   // optimize as without a profile instead of as cold.
   InstrProfWriter Writer(Sparse);
@@ -52,6 +75,8 @@ int mergeProfiles(int Argc, char **Argv) {
                          Path + ": " + toString(std::move(E)));
     std::string RecordError;
     for (NamedInstrProfRecord &Record : **Reader) {
+      if (!KeepFunctions.empty() && !Kept.contains(Record.Name))
+        continue;
       Writer.addRecord(std::move(Record), /*Weight=*/1, [&](Error E) {
         if (RecordError.empty())
           RecordError = toString(std::move(E));
@@ -100,4 +125,26 @@ int mergeProfiles(int Argc, char **Argv) {
   return 0;
 }
 
+int profileFunctions(int Argc, char **Argv) {
+  if (Argc != 4 || StringRef(Argv[2]) != "--input")
+    return reportError("usage",
+                       "profile-functions requires --input <file.profdata>");
+  std::string Path = Argv[3];
+  IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getRealFileSystem();
+  Expected<std::unique_ptr<InstrProfReader>> Reader =
+      InstrProfReader::create(Path, *FS);
+  if (!Reader)
+    return reportError("profile_read_failed",
+                       Path + ": " + toString(Reader.takeError()));
+  json::Array Functions;
+  for (const NamedInstrProfRecord &Record : **Reader)
+    Functions.push_back(Record.Name.str());
+  if ((*Reader)->hasError())
+    return reportError("profile_read_failed",
+                       Path + ": " + toString((*Reader)->getError()));
+  json::Object Response{{"ok", true}, {"functions", std::move(Functions)}};
+  outs() << formatv("{0}\n", json::Value(std::move(Response)));
+  return 0;
+}
+
 } // namespace scriptc

native/llvm-codegen/src/profile.h

@@ -2,9 +2,14 @@
 
 namespace scriptc {
 
-// `merge-profiles --output <file.profdata> [--sparse] <input>...`: merges raw
-// (.profraw) or indexed (.profdata) IR profiles into one indexed profile.
-// --sparse leaves out functions that never ran.
+// `merge-profiles --output <file.profdata> [--sparse] [--keep-functions
+// <file>] <input>...`: merges raw (.profraw) or indexed (.profdata) IR
+// profiles into one indexed profile. --sparse leaves out functions that never
+// ran, and --keep-functions every function the file does not name.
 int mergeProfiles(int Argc, char **Argv);
 
+// `profile-functions --input <file.profdata>`: prints the names of the
+// functions an indexed profile has records for, as {"ok":true,"functions":[]}.
+int profileFunctions(int Argc, char **Argv);
+
 } // namespace scriptc

native/llvm-codegen/src/runtime-unit.cpp

@@ -13,6 +13,7 @@
 #include "runtime-unit.h"
 
 #include "diagnostics.h"
+#include "emit.h"
 #include "runtime-import.h"
 #include "target.h"
 
@@ -27,12 +28,16 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/SizeOpts.h"
 
 #include <memory>
 #include <optional>
@@ -53,6 +58,16 @@ struct UnitOptions {
   std::string Tag;
   bool FunctionSections = false;
   bool DataSections = false;
+  // When set, the input is unoptimized and the helper optimizes it at this
+  // level, with the profile-guided optimization below; otherwise the input
+  // is already optimized and is only promoted and emitted.
+  std::string OptLevel;
+  // Instruments the unit for IR profiling; raw profiles go to this pattern,
+  // or, when it is empty, to the file the program that links the unit names.
+  bool ProfileGenerate = false;
+  std::string ProfileGeneratePattern;
+  // Indexed profile (.profdata) that guides the unit's optimization.
+  std::string ProfileUse;
 };
 
 std::optional<UnitOptions> parse(int Argc, char **Argv) {
@@ -80,12 +95,26 @@ std::optional<UnitOptions> parse(int Argc, char **Argv) {
       Options.Target = Value.str();
     else if (Arg == "--tag")
       Options.Tag = Value.str();
+    else if (Arg == "--opt-level")
+      Options.OptLevel = Value.str();
+    else if (Arg == "--profile-generate") {
+      Options.ProfileGenerate = true;
+      Options.ProfileGeneratePattern = Value.str();
+    } else if (Arg == "--profile-use")
+      Options.ProfileUse = Value.str();
     else
       return std::nullopt;
   }
   if (Options.Input.empty() || Options.Object.empty() ||
       Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty())
     return std::nullopt;
+  // A profile only guides an optimization the helper runs, and a unit is
+  // either instrumented or annotated.
+  if ((Options.ProfileGenerate || !Options.ProfileUse.empty()) &&
+      Options.OptLevel.empty())
+    return std::nullopt;
+  if (Options.ProfileGenerate && !Options.ProfileUse.empty())
+    return std::nullopt;
   return Options;
 }
 
@@ -98,10 +127,22 @@ bool keepLocal(const GlobalValue &GV) {
   return false;
 }
 
+// The profile's counters and per-function records (instrumented units only)
+// stay private to the unit: the profile runtime finds them by section.
+bool profileData(const GlobalValue &GV) {
+  if (const auto *Object = dyn_cast<GlobalObject>(&GV))
+    if (Object->getSection().starts_with("__llvm_prf"))
+      return true;
+  return GV.getName().starts_with("__llvm_prf") ||
+         GV.getName().starts_with("__profc_") ||
+         GV.getName().starts_with("__profd_") ||
+         GV.getName().starts_with("__profvp_");
+}
+
 void promoteLocals(Module &M, StringRef Tag) {
   unsigned Anonymous = 0;
   auto Promote = [&](GlobalValue &GV) {
-    if (!GV.hasLocalLinkage() || keepLocal(GV))
+    if (!GV.hasLocalLinkage() || keepLocal(GV) || profileData(GV))
       return;
     std::string Base =
         GV.hasName() ? GV.getName().str() : ("anon." + Twine(Anonymous++)).str();
@@ -157,7 +198,9 @@ int runtimeUnit(int Argc, char **Argv) {
     return fail("usage",
                 "runtime-unit requires --input <bc> --object <o> --bitcode <bc> "
                 "--target <triple> --tag <unit> and accepts "
-                "--function-sections --data-sections");
+                "--function-sections --data-sections, and --opt-level <level> "
+                "with either --profile-generate <pattern> or --profile-use "
+                "<profdata>");
   LLVMContext Context;
   SMDiagnostic Diagnostic;
   std::unique_ptr<Module> M = parseIRFile(Options->Input, Diagnostic, Context);
@@ -208,6 +251,42 @@ int runtimeUnit(int Argc, char **Argv) {
     if (F.getFnAttribute("probe-stack").getValueAsString() == "__chkstk_darwin")
       F.addFnAttr("probe-stack", "inline-asm");
 
+  // Profiles name a unit's internal functions after its source file. The
+  // unit's own name keeps those names the same in every checkout, so a
+  // committed profile matches wherever the pack is built.
+  if (!Options->OptLevel.empty()) {
+    M->setSourceFileName(Options->Tag + ".c");
+    // The runtime profile comes from a fixed set of training programs, and a
+    // runtime function they rarely call can be hot in another program (a
+    // long UTF-8 decode, a busy worker channel). Code the profile finds cold
+    // is therefore optimized for speed as without a profile, not for size:
+    // the profile still guides inlining, block layout and branch weights.
+    EnablePGSO.setValue(false);
+    // The profile names of the functions an instrumented unit defines, so
+    // a merged profile can keep the runtime's records and drop the
+    // training programs'.
+    json::Array Functions;
+    if (Options->ProfileGenerate)
+      for (const Function &F : *M)
+        if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage())
+          Functions.push_back(getIRPGOFuncName(F));
+    ModuleProfileMatch Match;
+    if (std::optional<std::string> Failure = optimizeModule(
+            *M, *Machine, Options->OptLevel, Options->ProfileGenerate,
+            Options->ProfileGeneratePattern, Options->ProfileUse, Match))
+      return fail("optimization_failed", *Failure);
+    if (!Options->ProfileUse.empty()) {
+      json::Object Profile{{"annotated", Match.Annotated},
+                           {"mismatched", Match.Mismatched},
+                           {"missing", Match.Missing}};
+      json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    } else if (Options->ProfileGenerate) {
+      json::Object Response{{"ok", true}, {"functions", std::move(Functions)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    }
+  }
+
   promoteLocals(*M, Options->Tag);
   std::string VerifyError;
   raw_string_ostream VerifyStream(VerifyError);

packages/runtime-pack-common/scripts/build.mjs

@@ -2,10 +2,12 @@
 import { execFile } from "node:child_process";
 import { createHash } from "node:crypto";
 import { availableParallelism, tmpdir } from "node:os";
+import { existsSync } from "node:fs";
 import { copyFile, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { dirname, join, relative, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { promisify } from "node:util";
+import { gunzipSync } from "node:zlib";
 import { createRuntimePackMatrix } from "../runtime-pack-matrix.mjs";
 import {
   RUNTIME_COMPILE_FLAGS,
@@ -118,6 +120,44 @@ async function build() {
     config.bitcode === false
       ? null
       : await resolveRuntimeUnitHelper(repoRoot, matrix.target.llvm_triple);
+  // Speed units that the LLVM helper optimizes with the target's committed
+  // runtime profile (profiles/<target>.profdata.gz), or instruments for
+  // scripts/refresh-runtime-profile.mjs (SCRIPTC_RUNTIME_PROFILE_GENERATE=1).
+  // The runtime compiler then emits their bitcode unoptimized, and one LLVM,
+  // the helper's, instruments, annotates and optimizes every profiled unit.
+  // Release and library flavors keep the runtime compiler's optimization.
+  const committedProfile = join(
+    repoRoot,
+    "packages",
+    "runtime-pack-common",
+    "profiles",
+    `${config.target.name}.profdata.gz`,
+  );
+  // The committed profile is unpacked into the build directory once it exists.
+  const unpackedProfile = join(buildRoot, "runtime.profdata");
+  // Without a helper that optimizes units itself, speed units are built as
+  // before, without the profile.
+  const unitProfiles = unitHelper?.features.includes("runtime-unit-profile") === true;
+  if (process.env.SCRIPTC_RUNTIME_PROFILE_GENERATE === "1" && !unitProfiles)
+    throw new Error(
+      "SCRIPTC_RUNTIME_PROFILE_GENERATE=1 needs an LLVM helper with runtime-unit profiles",
+    );
+  const speedProfile = !unitProfiles
+    ? null
+    : process.env.SCRIPTC_RUNTIME_PROFILE_GENERATE === "1"
+      ? { args: ["--opt-level", "2", "--profile-generate", ""] }
+      : existsSync(committedProfile)
+        ? { args: ["--opt-level", "2", "--profile-use", unpackedProfile], unpack: true }
+        : null;
+  // Speed units the helper optimizes without the profile (at the same level).
+  // scr_message.c: with the profile on its decoding, a program whose workers
+  // each receive many small messages ran 5% slower in its workers' own code,
+  // although that code and its data were unchanged; without it, the program
+  // runs as fast as with an unprofiled runtime.
+  const unprofiledUnits = new Set(["scr_message.c"]);
+  // The instrumented units' functions, which SCRIPTC_RUNTIME_PROFILE_FUNCTIONS
+  // names a file for; the refresh script keeps only their records.
+  const profiledFunctions = new Set();
   const parallel = async (items, task) => {
     const width = Math.max(1, Math.min(8, availableParallelism()));
     return parallelMap(items, width, task);
@@ -156,6 +196,8 @@ async function build() {
   await rm(buildRoot, { recursive: true, force: true });
   try {
     await mkdir(stagedOutputRoot, { recursive: true });
+    if (speedProfile?.unpack)
+      await writeFile(unpackedProfile, gunzipSync(await readFile(committedProfile)));
     // Zig 0.16 creates a zero-byte `a.o` in its working directory for
     // `zig cc --version`. Probe from a private temporary directory so a
     // runtime-pack build never leaves that compiler byproduct in the package.
@@ -197,6 +239,7 @@ async function build() {
               ...(config.platform === "linux" ? ["-fomit-frame-pointer"] : []),
             ]
           : [];
+      const profileMatches = [];
       const units = await parallel(
         flavorSpec.runtime_units ?? matrix.runtime_units,
         async (unit) => {
@@ -249,9 +292,10 @@ async function build() {
             let bitcode;
             const bitcodeOutput = output.replace(/\.o$/, ".bc");
             if (flavor === "speed") await mkdir(dirname(output), { recursive: true });
+            let unitBuild = false;
             if (
               flavor === "speed" &&
-              (await buildRuntimeUnit({
+              (unitBuild = await buildRuntimeUnit({
                 helper: unitHelper,
                 compileBitcode: (optimized) =>
                   run(compiler, [
@@ -259,12 +303,17 @@ async function build() {
                     ...sourcePathFlags,
                     ...unitFlags,
                     "-g0",
+                    ...(speedProfile === null ? [] : ["-Xclang", "-disable-llvm-passes"]),
                     "-emit-llvm",
                     "-c",
                     join(runtimeSrc, unit.source),
                     "-o",
                     optimized,
                   ]),
+                helperArgs:
+                  speedProfile?.unpack && unprofiledUnits.has(unit.source)
+                    ? ["--opt-level", "2"]
+                    : (speedProfile?.args ?? []),
                 object: output,
                 bitcode: bitcodeOutput,
                 triple: matrix.target.llvm_triple,
@@ -275,6 +324,8 @@ async function build() {
                   ),
               }))
             ) {
+              if (unitBuild.profile !== undefined) profileMatches.push(unitBuild.profile);
+              for (const name of unitBuild.functions ?? []) profiledFunctions.add(name);
               bitcode = {
                 path: artifactPath(bitcodeOutput),
                 sha256: await sha256(bitcodeOutput),
@@ -324,7 +375,21 @@ async function build() {
           return { source: unit.source, predicate: unit.predicate, variants };
         },
       );
-      flavors[flavor] = { optimization: flavorSpec.optimization, runtime_units: units };
+      flavors[flavor] = {
+        optimization: flavorSpec.optimization,
+        runtime_units: units,
+        // How the committed runtime profile matched the units it guided
+        // (scripts/refresh-runtime-profile.mjs refreshes it).
+        ...(profileMatches.length === 0
+          ? {}
+          : {
+              runtime_profile: {
+                annotated: profileMatches.reduce((sum, match) => sum + match.annotated, 0),
+                mismatched: profileMatches.reduce((sum, match) => sum + match.mismatched, 0),
+                missing: profileMatches.reduce((sum, match) => sum + match.missing, 0),
+              },
+            }),
+      };
     }
     const mbedtlsSources = (await readdir(join(mbedtls, "library")))
       .filter((name) => !name.startsWith(".") && name.endsWith(".c"))
@@ -505,6 +570,14 @@ async function build() {
       backupRoot: join(packageRoot, `.runtime-pack-artifacts-backup-${suffix}`),
       backupManifestPath: join(packageRoot, `.runtime-pack-manifest-backup-${suffix}`),
     });
+    if (process.env.SCRIPTC_RUNTIME_PROFILE_FUNCTIONS !== undefined)
+      await writeFile(
+        process.env.SCRIPTC_RUNTIME_PROFILE_FUNCTIONS,
+        [...profiledFunctions]
+          .sort()
+          .map((name) => `${name}\n`)
+          .join(""),
+      );
     process.stdout.write(
       `built ${packageManifest.name}@${packageManifest.version}: ${Object.keys(flavors).length} flavors, ${archives.length} vendor archives\n`,
     );

packages/runtime-pack-common/scripts/runtime-unit.mjs

@@ -12,10 +12,13 @@ const run = promisify(execFile);
  * Speed units produce both artifacts from one optimized LLVM module: the
  * runtime compiler optimizes the C source to bitcode, and the host helper's
  * `runtime-unit` command promotes translation-unit-local symbols to hidden
- * unit-qualified globals and emits the object and the bitcode. Release
- * units keep the runtime compiler's own objects; their bitcode is the same
- * compilation with -emit-llvm, which the helper's `prune-import` reduces to
- * the bodies that need none of the unit's local state. Both bitcode forms
+ * unit-qualified globals and emits the object and the bitcode. With the
+ * target's committed runtime profile, the runtime compiler emits the bitcode
+ * unoptimized and the helper also optimizes it with the profile (build.mjs),
+ * so one LLVM reads and writes every profile. Release units keep the
+ * runtime compiler's own objects; their bitcode is the same compilation with
+ * -emit-llvm, which the helper's `prune-import` reduces to the bodies that
+ * need none of the unit's local state. Both bitcode forms
  * keep only what a program build can import.
  *
  * Every host helper supports every scriptc target, so packs keep building
@@ -54,7 +57,15 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
     const { stdout } = await run(helper, ["version", "--format=json"]);
     const version = JSON.parse(stdout);
     if (!version.supported_targets?.includes(triple)) return null;
-    return { path: helper, identity: `scriptc-llvm-codegen ${version.llvm_version}` };
+    return {
+      path: helper,
+      identity: `scriptc-llvm-codegen ${version.llvm_version}`,
+      // Commands and options newer than the protocol version, such as
+      // "runtime-unit-profile": `runtime-unit` takes --opt-level,
+      // --profile-use and --profile-generate (older helpers only promote and
+      // emit).
+      features: Array.isArray(version.features) ? version.features : [],
+    };
   } catch {
     if (explicit !== undefined) throw new Error(`SCRIPTC_LLVM_CODEGEN is not usable: ${helper}`);
     return null;
@@ -63,12 +74,19 @@ export async function resolveRuntimeUnitHelper(repoRoot, triple) {
 
 /** Compile one unit through the helper. `compileBitcode(output)` must run
  * the runtime compiler with the unit's exact flags plus -emit-llvm.
+ * `helperArgs` asks the helper to optimize the unit itself (`--opt-level`,
+ * with `--profile-use` or `--profile-generate`), for bitcode the runtime
+ * compiler emitted unoptimized.
  * Returns false, leaving no artifacts behind, when the helper cannot emit
  * this unit; the caller then compiles the object directly without bitcode,
- * so a pack never fails to build because of the import path. */
+ * so a pack never fails to build because of the import path. Otherwise
+ * returns an object, with the helper's `profile` match counts when a profile
+ * guided the unit, or the profile names of the unit's `functions` when it
+ * was instrumented. */
 export async function buildRuntimeUnit({
   helper,
   compileBitcode,
+  helperArgs = [],
   object,
   bitcode,
   triple,
@@ -79,7 +97,7 @@ export async function buildRuntimeUnit({
   // A runtime compiler failure is a real build error and surfaces as one.
   await compileBitcode(optimized);
   try {
-    await run(helper.path, [
+    const { stdout } = await run(helper.path, [
       "runtime-unit",
       "--input",
       optimized,
@@ -92,8 +110,13 @@ export async function buildRuntimeUnit({
       "--tag",
       tag,
       ...(sections ? ["--function-sections", "--data-sections"] : []),
+      ...helperArgs,
     ]);
-    return true;
+    const report = stdout.trim() === "" ? {} : JSON.parse(stdout.trim().split("\n").pop());
+    return {
+      ...(report.profile === undefined ? {} : { profile: report.profile }),
+      ...(report.functions === undefined ? {} : { functions: report.functions }),
+    };
   } catch (error) {
     const detail = (error.stderr || error.message || String(error)).trim().split("\n")[0];
     process.stderr.write(`runtime-unit: ${tag}: emitting without import bitcode (${detail})\n`);

packages/runtime-pack-common/test/runtime-profile.test.ts

@@ -0,0 +1,98 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { gunzipSync } from "node:zlib";
+import { expect, test } from "vitest";
+import { resolveRuntimeUnitHelper } from "../scripts/runtime-unit.mjs";
+
+const repoRoot = fileURLToPath(new URL("../../..", import.meta.url));
+const committedProfile = join(
+  repoRoot,
+  "packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz",
+);
+const manifestPath = join(repoRoot, "packages/runtime-linux-x64-gnu/runtime-pack.json");
+const runtimeRoot = join(repoRoot, "packages/runtime");
+
+// Both tests read the committed profile through an LLVM helper new enough to
+// apply it; the pack test also needs the Linux x64 pack built with it.
+const helper =
+  process.platform === "linux" && process.arch === "x64" && existsSync(committedProfile)
+    ? await resolveRuntimeUnitHelper(repoRoot, "x86_64-unknown-linux-gnu")
+    : null;
+const built =
+  helper?.features.includes("runtime-unit-profile") === true && existsSync(manifestPath);
+
+// A profile older than the runtime sources loses the counts of every function
+// whose control flow changed since it was collected. More than this share of
+// the functions it describes means it is due for scripts/refresh-runtime-profile.mjs.
+const MAX_MISMATCHED_SHARE = 0.1;
+
+test.skipIf(!built)("the committed runtime profile still matches the runtime's speed units", () => {
+  const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
+  const match = manifest.flavors?.speed?.runtime_profile;
+  expect(match, "the speed flavor was built without the committed runtime profile").toBeDefined();
+  const described = match.annotated + match.mismatched;
+  expect(match.annotated).toBeGreaterThan(0);
+  expect(
+    match.mismatched / described,
+    `${match.mismatched} of the ${described} runtime functions the profile describes changed since it was collected; run node scripts/refresh-runtime-profile.mjs`,
+  ).toBeLessThanOrEqual(MAX_MISMATCHED_SHARE);
+});
+
+// The committed profile holds the runtime's records only. Training programs'
+// functions, and the source paths their names can carry, stay out of the
+// repository: refresh-runtime-profile.mjs keeps the functions the
+// instrumented runtime units define.
+test.skipIf(helper?.features.includes("profile-functions") !== true)(
+  "the committed runtime profile names only runtime functions",
+  () => {
+    const profile = gunzipSync(readFileSync(committedProfile));
+    const scratch = mkdtempSync(join(tmpdir(), "scriptc-runtime-profile-"));
+    let functions: string[];
+    try {
+      const unpacked = join(scratch, "runtime.profdata");
+      writeFileSync(unpacked, profile);
+      ({ functions } = JSON.parse(
+        execFileSync(helper!.path, ["profile-functions", "--input", unpacked], {
+          encoding: "utf8",
+          maxBuffer: 64 * 1024 * 1024,
+        }),
+      ));
+    } finally {
+      rmSync(scratch, { recursive: true, force: true });
+    }
+    expect(functions.length).toBeGreaterThan(0);
+
+    const sources = new Set(readdirSync(join(runtimeRoot, "src")));
+    const identifiers = new Set<string>();
+    const collect = (dir: string) => {
+      for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory()) collect(path);
+        else if (/\.(c|h|inc|def)$/.test(entry.name))
+          for (const name of readFileSync(path, "latin1").match(/[A-Za-z_]\w*/g) ?? [])
+            identifiers.add(name);
+      }
+    };
+    collect(join(runtimeRoot, "src"));
+    collect(join(runtimeRoot, "vendor"));
+
+    // A unit's internal functions are named "<unit>.c;<name>" after its
+    // runtime source, and its external ones by their C names, which are never
+    // a program's `main` or generated `sc_` symbols.
+    const foreign = functions.filter((name) => {
+      if (/[/\\]/.test(name)) return true;
+      const local = /^([\w-]+\.c);(.+)$/.exec(name);
+      if (local !== null) return !sources.has(local[1]!);
+      return name === "main" || name.startsWith("sc_") || !identifiers.has(name);
+    });
+    expect(foreign, "records that are not runtime functions").toEqual([]);
+
+    // Nor does any other part of the file hold a path, such as a checkout's.
+    const text = profile.toString("latin1");
+    const paths = text.match(/(?:\/[\w.-]+){3,}|[A-Za-z]:\\[\w.-]+\\|\.tsx?;/g) ?? [];
+    expect(paths, "path strings in the committed profile").toEqual([]);
+  },
+);

scripts/bench-runtime.mjs

@@ -44,6 +44,7 @@ const { values } = parseArgs({
     "candidate-target-cpu": { type: "string" },
     "baseline-cpu-dispatch": { type: "boolean", default: false },
     "candidate-cpu-dispatch": { type: "boolean", default: false },
+    "candidate-profile-generate": { type: "string" },
     json: { type: "string" },
     "no-lock": { type: "boolean", default: false },
     node: { type: "boolean", default: true },
@@ -57,6 +58,7 @@ if (values.help) {
        [--baseline-optimization=release] [--candidate-optimization=release]
        [--baseline-target-cpu=<level>] [--candidate-target-cpu=<level>]
        [--baseline-cpu-dispatch] [--candidate-cpu-dispatch]
+       [--candidate-profile-generate=<raw profile pattern>]
        [--no-lock] [--no-node] [--keep]
 
 Each checkout must have a built CLI (packages/cli/dist/bootstrap.js) and native
@@ -79,7 +81,10 @@ release. --baseline-target-cpu/--candidate-target-cpu pass --target-cpu to each
 contender's builds, e.g. --baseline=. --candidate-target-cpu=x86-64-v3 compares
 x86-64-v3 code with the baseline. --baseline-cpu-dispatch/--candidate-cpu-dispatch
 pass --cpu-dispatch, and SCRIPTC_CPU_LEVEL in the environment reaches every timed
-run. cold_build_ms records each contender's first, uncached build.`);
+run. cold_build_ms records each contender's first, uncached build.
+--candidate-profile-generate builds the candidate's workloads with
+--profile-generate=<pattern>, so its runs write raw profiles there
+(scripts/refresh-runtime-profile.mjs trains the runtime profile this way).`);
   process.exit(0);
 }
 const runs = Number(values.runs);
@@ -448,7 +453,16 @@ function revision(root) {
   });
   return r.stdout.trim() + (dirty.stdout.trim() ? "+dirty" : "");
 }
-function build(label, root, workload, layout, optimization, targetCpu, cpuDispatch) {
+function build(
+  label,
+  root,
+  workload,
+  layout,
+  optimization,
+  targetCpu,
+  cpuDispatch,
+  profileGenerate,
+) {
   const cli = cliFor(root);
   const out = join(work, label, layout === 0 ? workload.name : `${workload.name}.layout${layout}`);
   mkdirSync(dirname(out), { recursive: true });
@@ -471,6 +485,7 @@ function build(label, root, workload, layout, optimization, targetCpu, cpuDispat
       `--optimization=${optimization}`,
       ...(targetCpu === undefined ? [] : [`--target-cpu=${targetCpu}`]),
       ...(cpuDispatch ? ["--cpu-dispatch"] : []),
+      ...(profileGenerate === undefined ? [] : [`--profile-generate=${profileGenerate}`]),
       "-o",
       out,
     ],
@@ -507,6 +522,7 @@ const contenders = [
     optimization: values["candidate-optimization"],
     targetCpu: values["candidate-target-cpu"],
     cpuDispatch: values["candidate-cpu-dispatch"],
+    profileGenerate: values["candidate-profile-generate"],
   },
 ];
 const report = {
@@ -613,7 +629,16 @@ try {
       entry.results[c.label] = result;
       result.binaries = [];
       for (let layout = 0; layout < layouts && result.status === undefined; layout++) {
-        const built = build(c.label, c.root, w, layout, c.optimization, c.targetCpu, c.cpuDispatch);
+        const built = build(
+          c.label,
+          c.root,
+          w,
+          layout,
+          c.optimization,
+          c.targetCpu,
+          c.cpuDispatch,
+          c.profileGenerate,
+        );
         result.build_ms += Math.round(built.buildMs);
         if (layout === 0) result.cold_build_ms = Math.round(built.buildMs);
         if (!built.ok) {

scripts/refresh-runtime-profile.mjs

@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+/* Refreshes packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz,
+ * the profile that the Linux x64 runtime pack's speed units are optimized
+ * with (runtime-pack-common/scripts/build.mjs). The LLVM helper instruments,
+ * annotates and optimizes those units, so the profile is in the helper's
+ * format and merged by the helper's merge-profiles, like every program
+ * profile.
+ *
+ * Builds an instrumented runtime pack (SCRIPTC_RUNTIME_PROFILE_GENERATE=1),
+ * then runs the training programs built against it with --profile-generate:
+ * every runtime benchmark workload at --optimization=speed (through
+ * scripts/bench-runtime.mjs, with its inputs and load clients), and, with
+ * --tsc-ts, an instrumented tsc-ts checking its own sources and each
+ * --project. The raw profiles hold the runtime's counters beside the
+ * programs'; the merged profile keeps the runtime functions that ran: the
+ * instrumented pack lists its functions, and the programs' records are left
+ * out. The pack is then rebuilt with the new profile. A stale profile only
+ * loses part of its gain: functions whose code changed are optimized as
+ * without one (the pack test reports how many).
+ *
+ *   node scripts/refresh-runtime-profile.mjs [--tsc-ts <checkout> [--project <dir>[:<tsconfig>]]...]
+ */
+import { spawnSync } from "node:child_process";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { constants, gzipSync } from "node:zlib";
+
+const root = fileURLToPath(new URL("..", import.meta.url));
+const { values } = parseArgs({
+  options: {
+    "tsc-ts": { type: "string" },
+    project: { type: "string", multiple: true, default: [] },
+    output: {
+      type: "string",
+      default: join(root, "packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz"),
+    },
+    work: { type: "string" },
+    help: { type: "boolean", default: false },
+  },
+});
+if (values.help) {
+  console.log(
+    "Usage: node scripts/refresh-runtime-profile.mjs [--tsc-ts <checkout> [--project <dir>[:<tsconfig>]]...] [--output <profdata.gz>] [--work <dir>]",
+  );
+  process.exit(0);
+}
+if (process.platform !== "linux" || process.arch !== "x64")
+  throw new Error("the runtime profile is collected on Linux x64");
+if (values.project.length !== 0 && values["tsc-ts"] === undefined)
+  throw new Error("--project needs --tsc-ts");
+const cli = join(root, "packages/cli/dist/main.js");
+if (!existsSync(cli)) throw new Error("build the compiler first (pnpm build)");
+const helper = join(root, "packages/llvm-linux-x64-gnu/bin/scriptc-llvm-codegen");
+if (!existsSync(helper))
+  throw new Error(
+    "build the LLVM helper first (pnpm --filter @scriptc/llvm-linux-x64-gnu build:native)",
+  );
+const output = resolve(values.output);
+mkdirSync(dirname(output), { recursive: true });
+const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-runtime-profile-")));
+const raw = join(work, "raw");
+rmSync(raw, { recursive: true, force: true });
+mkdirSync(raw, { recursive: true });
+const env = { ...process.env };
+delete env.SCRIPTC_LLVM_HELPER;
+
+const buildPack = (extra) =>
+  run("pnpm", ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"], {
+    cwd: root,
+    env: { ...env, ...extra },
+  });
+
+const functions = join(work, "runtime-functions.txt");
+console.log("building the instrumented runtime pack");
+buildPack({ SCRIPTC_RUNTIME_PROFILE_GENERATE: "1", SCRIPTC_RUNTIME_PROFILE_FUNCTIONS: functions });
+try {
+  console.log("training on the runtime benchmark workloads");
+  run(
+    process.execPath,
+    [
+      join(root, "scripts/bench-runtime.mjs"),
+      `--candidate=${root}`,
+      "--candidate-optimization=speed",
+      `--candidate-profile-generate=${join(raw, "bench-%m-%p.profraw")}`,
+      "--runs=3",
+      "--warmup=0",
+      "--no-node",
+      "--no-lock",
+      `--json=${join(work, "bench.json")}`,
+    ],
+    { cwd: root, env },
+  );
+  if (values["tsc-ts"] !== undefined) trainTscTs(resolve(values["tsc-ts"]));
+  const profiles = readdirSync(raw).map((name) => join(raw, name));
+  if (profiles.length === 0) throw new Error(`no raw profiles in ${raw}`);
+  // `scriptc merge-profiles` runs this merge too; called directly, it keeps
+  // only the runtime's records, so the training programs' stay out of the repo.
+  const merged = join(work, "runtime.profdata");
+  run(
+    helper,
+    ["merge-profiles", "--output", merged, "--sparse", "--keep-functions", functions, ...profiles],
+    { cwd: root, env },
+  );
+  writeFileSync(output, gzipSync(readFileSync(merged), { level: constants.Z_BEST_COMPRESSION }));
+  console.log(`wrote ${output} from ${profiles.length} raw profiles`);
+} finally {
+  // The pack goes back to an ordinary build, now with the new profile.
+  console.log("rebuilding the runtime pack");
+  buildPack({});
+}
+if (values.work === undefined) rmSync(work, { recursive: true, force: true });
+
+// An instrumented tsc-ts, built against the instrumented pack, checking its
+// own sources and each --project at --checkers 1.
+function trainTscTs(checkout) {
+  const pkg = join(checkout, "packages/tsc-ts");
+  if (!existsSync(join(pkg, "src/main.ts")))
+    throw new Error(`${checkout} is not a tsc-ts checkout`);
+  // The upstream lib files, where npm run fetch:upstream puts them (older
+  // tsc-ts checkouts used the second directory).
+  const lib = [
+    "upstream/typescript-go/internal/bundled/libs",
+    "upstream/TypeScript/tsc/internal/bundled/libs",
+  ]
+    .map((dir) => join(checkout, dir))
+    .find((dir) => existsSync(join(dir, "lib.d.ts")));
+  if (lib === undefined)
+    throw new Error(
+      `no lib files in ${checkout}/upstream: run npm run fetch:upstream in the tsc-ts checkout`,
+    );
+  const tsc = join(work, "tsc-ts");
+  console.log("building an instrumented tsc-ts");
+  run(
+    process.execPath,
+    [
+      cli,
+      "build",
+      "src/main.ts",
+      "--optimization=speed",
+      "--detached-exit",
+      `--profile-generate=${join(raw, "tsc-ts-%p.profraw")}`,
+      "-o",
+      tsc,
+    ],
+    { cwd: pkg, env },
+  );
+  const projects = [
+    { dir: pkg, config: "tsconfig.json" },
+    ...values.project.map((spec) => {
+      const [dir, config = "tsconfig.json"] = spec.split(":");
+      return { dir: resolve(dir), config };
+    }),
+  ];
+  for (const { dir, config } of projects) {
+    // Projects with type errors exit with 1 or 2; any other status is a failed run.
+    const result = spawnSync(
+      tsc,
+      ["-p", config, "--noEmit", "--incremental", "false", "--checkers", "1"],
+      {
+        cwd: dir,
+        stdio: "ignore",
+        env: { ...env, TSC_TS_LIB_DIR: lib },
+      },
+    );
+    if (result.status === null || result.status > 2)
+      throw new Error(
+        `the instrumented tsc-ts failed on ${join(dir, config)} (${result.signal ?? result.status})`,
+      );
+    console.log(`profiled ${join(dir, config)}`);
+  }
+}
+
+function run(command, args, options) {
+  const result = spawnSync(command, args, { ...options, stdio: "inherit" });
+  if (result.status !== 0)
+    throw new Error(`${command} ${args.join(" ")} exited with ${result.status}`);
+}