Optimize the speed runtime units with a committed runtime profile

The LLVM helper's runtime-unit now optimizes a unit itself (--opt-level),
with IR instrumentation (--profile-generate) or a profile (--profile-use),
through the same pipeline as program code. The Linux x64 pack's speed
units are compiled to unoptimized bitcode and optimized there with
packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz, so one
LLVM writes and reads every profile. Release and library flavors are
unchanged. The pack manifest records how the profile matched the units.

Code the runtime profile finds cold is optimized for speed, not size: a
runtime function the training programs rarely call can be hot in another
program. scr_message.c is optimized without the profile.

scripts/refresh-runtime-profile.mjs trains the profile on the runtime
benchmark workloads and, optionally, a tsc-ts checkout, and keeps only the
records of functions the instrumented runtime units define
(merge-profiles --keep-functions). Tests check that the profile still
matches the runtime and names only runtime functions.

native/llvm-codegen/src/runtime-unit.cpp

@@ -13,6 +13,7 @@
 #include "runtime-unit.h"
 
 #include "diagnostics.h"
+#include "emit.h"
 #include "runtime-import.h"
 #include "target.h"
 
@@ -27,12 +28,16 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/SizeOpts.h"
 
 #include <memory>
 #include <optional>
@@ -53,6 +58,16 @@ struct UnitOptions {
   std::string Tag;
   bool FunctionSections = false;
   bool DataSections = false;
+  // When set, the input is unoptimized and the helper optimizes it at this
+  // level, with the profile-guided optimization below; otherwise the input
+  // is already optimized and is only promoted and emitted.
+  std::string OptLevel;
+  // Instruments the unit for IR profiling; raw profiles go to this pattern,
+  // or, when it is empty, to the file the program that links the unit names.
+  bool ProfileGenerate = false;
+  std::string ProfileGeneratePattern;
+  // Indexed profile (.profdata) that guides the unit's optimization.
+  std::string ProfileUse;
 };
 
 std::optional<UnitOptions> parse(int Argc, char **Argv) {
@@ -80,12 +95,26 @@ std::optional<UnitOptions> parse(int Argc, char **Argv) {
       Options.Target = Value.str();
     else if (Arg == "--tag")
       Options.Tag = Value.str();
+    else if (Arg == "--opt-level")
+      Options.OptLevel = Value.str();
+    else if (Arg == "--profile-generate") {
+      Options.ProfileGenerate = true;
+      Options.ProfileGeneratePattern = Value.str();
+    } else if (Arg == "--profile-use")
+      Options.ProfileUse = Value.str();
     else
       return std::nullopt;
   }
   if (Options.Input.empty() || Options.Object.empty() ||
       Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty())
     return std::nullopt;
+  // A profile only guides an optimization the helper runs, and a unit is
+  // either instrumented or annotated.
+  if ((Options.ProfileGenerate || !Options.ProfileUse.empty()) &&
+      Options.OptLevel.empty())
+    return std::nullopt;
+  if (Options.ProfileGenerate && !Options.ProfileUse.empty())
+    return std::nullopt;
   return Options;
 }
 
@@ -98,10 +127,22 @@ bool keepLocal(const GlobalValue &GV) {
   return false;
 }
 
+// The profile's counters and per-function records (instrumented units only)
+// stay private to the unit: the profile runtime finds them by section.
+bool profileData(const GlobalValue &GV) {
+  if (const auto *Object = dyn_cast<GlobalObject>(&GV))
+    if (Object->getSection().starts_with("__llvm_prf"))
+      return true;
+  return GV.getName().starts_with("__llvm_prf") ||
+         GV.getName().starts_with("__profc_") ||
+         GV.getName().starts_with("__profd_") ||
+         GV.getName().starts_with("__profvp_");
+}
+
 void promoteLocals(Module &M, StringRef Tag) {
   unsigned Anonymous = 0;
   auto Promote = [&](GlobalValue &GV) {
-    if (!GV.hasLocalLinkage() || keepLocal(GV))
+    if (!GV.hasLocalLinkage() || keepLocal(GV) || profileData(GV))
       return;
     std::string Base =
         GV.hasName() ? GV.getName().str() : ("anon." + Twine(Anonymous++)).str();
@@ -157,7 +198,9 @@ int runtimeUnit(int Argc, char **Argv) {
     return fail("usage",
                 "runtime-unit requires --input <bc> --object <o> --bitcode <bc> "
                 "--target <triple> --tag <unit> and accepts "
-                "--function-sections --data-sections");
+                "--function-sections --data-sections, and --opt-level <level> "
+                "with either --profile-generate <pattern> or --profile-use "
+                "<profdata>");
   LLVMContext Context;
   SMDiagnostic Diagnostic;
   std::unique_ptr<Module> M = parseIRFile(Options->Input, Diagnostic, Context);
@@ -208,6 +251,42 @@ int runtimeUnit(int Argc, char **Argv) {
     if (F.getFnAttribute("probe-stack").getValueAsString() == "__chkstk_darwin")
       F.addFnAttr("probe-stack", "inline-asm");
 
+  // Profiles name a unit's internal functions after its source file. The
+  // unit's own name keeps those names the same in every checkout, so a
+  // committed profile matches wherever the pack is built.
+  if (!Options->OptLevel.empty()) {
+    M->setSourceFileName(Options->Tag + ".c");
+    // The runtime profile comes from a fixed set of training programs, and a
+    // runtime function they rarely call can be hot in another program (a
+    // long UTF-8 decode, a busy worker channel). Code the profile finds cold
+    // is therefore optimized for speed as without a profile, not for size:
+    // the profile still guides inlining, block layout and branch weights.
+    EnablePGSO.setValue(false);
+    // The profile names of the functions an instrumented unit defines, so
+    // a merged profile can keep the runtime's records and drop the
+    // training programs'.
+    json::Array Functions;
+    if (Options->ProfileGenerate)
+      for (const Function &F : *M)
+        if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage())
+          Functions.push_back(getIRPGOFuncName(F));
+    ModuleProfileMatch Match;
+    if (std::optional<std::string> Failure = optimizeModule(
+            *M, *Machine, Options->OptLevel, Options->ProfileGenerate,
+            Options->ProfileGeneratePattern, Options->ProfileUse, Match))
+      return fail("optimization_failed", *Failure);
+    if (!Options->ProfileUse.empty()) {
+      json::Object Profile{{"annotated", Match.Annotated},
+                           {"mismatched", Match.Mismatched},
+                           {"missing", Match.Missing}};
+      json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    } else if (Options->ProfileGenerate) {
+      json::Object Response{{"ok", true}, {"functions", std::move(Functions)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    }
+  }
+
   promoteLocals(*M, Options->Tag);
   std::string VerifyError;
   raw_string_ostream VerifyStream(VerifyError);