Report how well the native compiler's profile matches its functions

The native compiler is built with a committed optimization profile. A
function whose control flow changed since the profile was collected is
optimized as without one, so the profile loses its gain as compiler code
changes, with nothing to show it.

With --profile-use, the helper now counts what the profile annotation
reports while it applies the profile: functions that took the profile's
counts, functions whose control flow no longer matches (hash mismatch),
and functions the profile has no record of. It sums the counts over
partitions, keeps each partition's counts beside its cached summary, and
prints them as one line of JSON after a successful emit. The compiler
reads that line (an older helper prints none) and returns it as the
executable artifact's profileMatch.

build-native-cli prints the share of profiled functions that match, and
warns below 90% that the profile should be refreshed. On main aa1c26845, a
profile refreshed on that commit matches 92.7% of the compiler's profiled
functions, and the one committed about 200 commits earlier 87.2%; that one
kept about half of the fresh profile's gain on a cold tsc-ts build.

native/llvm-codegen/src/emit.cpp

@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/IR/DiagnosticHandler.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/LegacyPassManager.h"
@@ -41,11 +42,13 @@
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/Threading.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/SplitModule.h"
 
+#include <atomic>
 #include <mutex>
 #include <optional>
 #include <string>
@@ -170,6 +173,108 @@ void limitProfileImports() {
   (void)Limited;
 }
 
+// How the --profile-use profile matched the functions it annotated, summed
+// over partitions: functions that received its counts, functions whose
+// control flow changed since it was collected (hash mismatch), and functions
+// it has no record of (new, or never run in training: the merged profile is
+// sparse). A partition reused from the cache without its counts is counted
+// in Unknown.
+struct ProfileMatch {
+  std::atomic<uint64_t> Annotated{0};
+  std::atomic<uint64_t> Mismatched{0};
+  std::atomic<uint64_t> Missing{0};
+  std::atomic<uint64_t> Unknown{0};
+
+  void add(const ProfileMatch &Other) {
+    Annotated += Other.Annotated;
+    Mismatched += Other.Mismatched;
+    Missing += Other.Missing;
+  }
+
+  std::string serialize() const {
+    return formatv("{0} {1} {2}", Annotated.load(), Mismatched.load(),
+                   Missing.load())
+        .str();
+  }
+
+  bool deserialize(StringRef Text) {
+    SmallVector<StringRef, 3> Parts;
+    Text.split(Parts, ' ');
+    uint64_t Values[3];
+    if (Parts.size() != 3)
+      return false;
+    for (size_t I = 0; I < 3; ++I)
+      if (Parts[I].getAsInteger(10, Values[I]))
+        return false;
+    Annotated += Values[0];
+    Mismatched += Values[1];
+    Missing += Values[2];
+    return true;
+  }
+};
+
+// Counts the profile annotation's hash-mismatch and missing-function
+// warnings instead of printing them. Every other diagnostic takes the
+// context's default handling.
+class ProfileMatchHandler : public DiagnosticHandler {
+public:
+  explicit ProfileMatchHandler(ProfileMatch &Match) : Match(Match) {}
+
+  bool handleDiagnostics(const DiagnosticInfo &Info) override {
+    if (Info.getKind() != DK_PGOProfile)
+      return false;
+    std::string Message;
+    raw_string_ostream Stream(Message);
+    DiagnosticPrinterRawOStream Printer(Stream);
+    Info.print(Printer);
+    if (StringRef(Message).contains("hash mismatch"))
+      ++Match.Mismatched;
+    else if (StringRef(Message).contains_insensitive("no profile data"))
+      ++Match.Missing;
+    else
+      return false;
+    return true;
+  }
+
+private:
+  ProfileMatch &Match;
+};
+
+// The annotation warns about functions the profile does not describe only
+// when asked; the match counts need those warnings. Process-wide, set once.
+void reportMissingProfiles() {
+  static const bool Reported = [] {
+    if (cl::Option *Option =
+            cl::getRegisteredOptions().lookup("pgo-warn-missing-function"))
+      Option->addOccurrence(0, "pgo-warn-missing-function", "true");
+    return true;
+  }();
+  (void)Reported;
+}
+
+// Functions of `M` that the profile annotated with counts.
+uint64_t annotatedFunctions(const Module &M) {
+  uint64_t Count = 0;
+  for (const Function &F : M) {
+    if (F.isDeclaration())
+      continue;
+    std::optional<Function::ProfileCount> Entry = F.getEntryCount();
+    if (Entry && !Entry->isSynthetic())
+      ++Count;
+  }
+  return Count;
+}
+
+// Prints the profile match as the successful emit's one line of JSON.
+void reportProfileMatch(const ProfileMatch &Match) {
+  json::Object Profile{{"annotated", Match.Annotated.load()},
+                       {"mismatched", Match.Mismatched.load()},
+                       {"missing", Match.Missing.load()},
+                       {"unknown_partitions", Match.Unknown.load()}};
+  json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
+  outs() << formatv("{0}\n", json::Value(std::move(Response)));
+}
+
 // Reads the profile's header once, before any partition needs it: it must
 // hold IR-level counts.
 std::optional<EmitFailure> inspectProfile(const EmitOptions &Options) {
@@ -387,9 +492,16 @@ std::optional<EmitFailure> publish(StringRef TemporaryPath,
 std::optional<EmitFailure>
 emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level,
            std::optional<PGOOptions> Profile, CodeGenFileType Type,
-           StringRef OutputPath, std::vector<SmallString<256>> &Temporaries) {
+           StringRef OutputPath, std::vector<SmallString<256>> &Temporaries,
+           ProfileMatch &Match) {
+  bool Annotating = Profile && Profile->Action == PGOOptions::IRUse;
+  if (Annotating)
+    M.getContext().setDiagnosticHandler(
+        std::make_unique<ProfileMatchHandler>(Match));
   Pipeline P(Machine, Level, std::move(Profile));
   P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
+  if (Annotating)
+    Match.Annotated += annotatedFunctions(M);
   if (std::optional<EmitFailure> Failure =
           verify(M, "post_optimization_verification_failed"))
     return Failure;
@@ -555,20 +667,32 @@ std::optional<EmitFailure>
 compileOptimizedPartitions(const EmitOptions &Options,
                            const PartitionPlan &Plan, OptimizationLevel Level,
                            StringRef Salt, const ArtifactCache &Cache,
-                           std::vector<SmallVector<char, 0>> &Objects) {
+                           std::vector<SmallVector<char, 0>> &Objects,
+                           ProfileMatch &Match) {
   size_t Count = Plan.Modules.size();
+  bool Annotating = !Options.ProfileUse.empty();
   std::vector<std::unique_ptr<MemoryBuffer>> Summaries(Count);
   SharedFailure Failures;
   forEachPartition(Count, [&](size_t I) {
     std::string Key = hashParts({Salt, "summary", Plan.Modules[I]});
-    if ((Summaries[I] = Cache.lookup(Key, "bc")))
+    if ((Summaries[I] = Cache.lookup(Key, "bc"))) {
+      if (Annotating) {
+        std::unique_ptr<MemoryBuffer> Counts = Cache.lookup(Key, "match");
+        if (!Counts || !Match.deserialize(Counts->getBuffer()))
+          ++Match.Unknown;
+      }
       return;
+    }
     std::string LookupError;
     std::unique_ptr<TargetMachine> Machine =
         createTargetMachine(Options.Target, Options.OptLevel, LookupError);
     if (!Machine)
       return Failures.set({"target_machine_failed", LookupError});
     LLVMContext Context;
+    ProfileMatch PartitionMatch;
+    if (Annotating)
+      Context.setDiagnosticHandler(
+          std::make_unique<ProfileMatchHandler>(PartitionMatch));
     Expected<std::unique_ptr<Module>> Loaded =
         loadPartition(Options, Plan.Modules[I], I, Context, *Machine);
     if (!Loaded)
@@ -598,6 +722,12 @@ compileOptimizedPartitions(const EmitOptions &Options,
                                     /*EmitModuleHash=*/true));
       MPM.run(Mod, P.MAM);
     }
+    if (Annotating) {
+      PartitionMatch.Annotated += annotatedFunctions(Mod);
+      Match.add(PartitionMatch);
+      // Stored before the summary, so a reused summary finds its counts.
+      Cache.store(Key, "match", PartitionMatch.serialize());
+    }
     StringRef Contents(Bitcode.data(), Bitcode.size());
     Cache.store(Key, "bc", Contents);
     Summaries[I] = MemoryBuffer::getMemBufferCopy(Contents);
@@ -744,7 +874,8 @@ compileOptimizedPartitions(const EmitOptions &Options,
 
 std::optional<EmitFailure>
 compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan,
-                  std::vector<SmallVector<char, 0>> &Objects) {
+                  std::vector<SmallVector<char, 0>> &Objects,
+                  ProfileMatch &Match) {
   // Everything besides a partition's own bytes that determines its
   // artifacts. The caller's cache directory already identifies the exact
   // helper build; source content and partition count arrive per module.
@@ -778,7 +909,7 @@ compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan,
     return compileUnoptimizedPartitions(Options, Plan, Salt, Cache, Objects);
   return compileOptimizedPartitions(Options, Plan,
                                     optimizationLevel(Options.OptLevel), Salt,
-                                    Cache, Objects);
+                                    Cache, Objects, Match);
 }
 
 // Publishes in-memory partition objects through private sibling files.
@@ -819,6 +950,8 @@ int emitPartitions(const EmitOptions &Options) {
   std::vector<SmallVector<char, 0>> Objects;
   std::optional<EmitFailure> Failure;
   bool Compiled = false;
+  // A failed attempt's counts are discarded with its objects.
+  auto Match = std::make_unique<ProfileMatch>();
   {
     ErrorOr<std::unique_ptr<MemoryBuffer>> Input =
         MemoryBuffer::getFile(Options.Input, /*IsText=*/false,
@@ -830,15 +963,16 @@ int emitPartitions(const EmitOptions &Options) {
         Input->reset();
         PartitionPlan Plan{std::move(Text->Sources),
                            std::move(Text->PublicSymbols)};
-        Compiled = !compilePartitions(Options, Plan, Objects);
+        Compiled = !compilePartitions(Options, Plan, Objects, *Match);
       }
     }
   }
   if (!Compiled) {
+    Match = std::make_unique<ProfileMatch>();
     PartitionPlan Plan;
     Failure = splitParsedModule(Options, Plan);
     if (!Failure)
-      Failure = compilePartitions(Options, Plan, Objects);
+      Failure = compilePartitions(Options, Plan, Objects, *Match);
   }
   std::vector<SmallString<256>> Temporaries;
   if (!Failure)
@@ -851,6 +985,8 @@ int emitPartitions(const EmitOptions &Options) {
     return reportError(Failure->Code, Failure->Message,
                        Options.DiagnosticFormat);
   }
+  if (!Options.ProfileUse.empty())
+    reportProfileMatch(*Match);
   return 0;
 }
 
@@ -895,6 +1031,8 @@ int emit(const EmitOptions &Options) {
     return reportError("invalid_import",
                        "runtime bitcode import requires an optimized build",
                        Options.DiagnosticFormat);
+  if (!Options.ProfileUse.empty())
+    reportMissingProfiles();
   if (std::optional<EmitFailure> Failure = inspectProfile(Options))
     return reportError(Failure->Code, Failure->Message,
                        Options.DiagnosticFormat);
@@ -957,9 +1095,10 @@ int emit(const EmitOptions &Options) {
                              ? CodeGenFileType::ObjectFile
                              : CodeGenFileType::AssemblyFile;
   std::vector<SmallString<256>> Temporaries;
+  ProfileMatch Match;
   std::optional<EmitFailure> Failure =
       emitModule(*Mod, *Machine, Level, profileOptions(Options), Type,
-                 Options.Outputs[0], Temporaries);
+                 Options.Outputs[0], Temporaries, Match);
   if (!Failure)
     Failure = publish(Temporaries[0], Options.Outputs[0]);
   if (Failure) {
@@ -968,6 +1107,8 @@ int emit(const EmitOptions &Options) {
     return reportError(Failure->Code, Failure->Message,
                        Options.DiagnosticFormat);
   }
+  if (!Options.ProfileUse.empty())
+    reportProfileMatch(Match);
   return 0;
 }
 

packages/compiler/src/backend/native-codegen.ts

@@ -30,6 +30,8 @@ import { compilerReleaseVersion } from "../library/sidecar.js";
 import {
   nativeProfileHelperArgs,
   nativeProfileKey,
+  parseNativeProfileMatch,
+  type NativeProfileMatch,
   type NativeProfileOptions,
 } from "./native-profile.js";
 import { writeTextParts } from "../text-parts.js";
@@ -69,6 +71,8 @@ export interface NativeCodegenArtifact {
   dependencies: NativeArtifactDependency[];
   /** Every emitted file, starting with the requested output path. */
   outputPaths: string[];
+  /** How the build's profile matched, when it used one and the helper ran. */
+  profileMatch?: NativeProfileMatch;
 }
 
 const resolvedHelperCache = new Map<string, Promise<ResolvedHelper>>();
@@ -458,7 +462,7 @@ export async function emitNativeArtifact(
   try {
     if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input);
     else await writeLlvmSource(input, options.llvm, 0o600);
-    await invoke(helper.binaryPath, [
+    const { stdout } = await invoke(helper.binaryPath, [
       "emit",
       "--input",
       input,
@@ -502,7 +506,9 @@ export async function emitNativeArtifact(
     for (let index = 0; index < stages.length; index++)
       await rename(stages[index]!, outputs[index]!);
     await pruneBuildCache(root);
-    return artifact;
+    const profileMatch =
+      options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout);
+    return profileMatch === null ? artifact : { ...artifact, profileMatch };
   } finally {
     await Promise.all([
       ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)),

packages/compiler/src/backend/native-profile.test.ts

@@ -1,6 +1,7 @@
 import { expect, test } from "vitest";
 import { executableLinkInputs } from "./link-plan-core.js";
 import {
+  parseNativeProfileMatch,
   nativeProfileHelperArgs,
   nativeProfileKey,
   nativeProfileRefusal,
@@ -71,3 +72,15 @@ test("only the instrumented link adds the profile runtime", () => {
     "-fprofile-generate",
   );
 });
+
+test("the helper's profile match is read from its output", () => {
+  expect(
+    parseNativeProfileMatch(
+      '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":1}}\n',
+    ),
+  ).toEqual({ annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 1 });
+  // A helper that predates the report prints nothing.
+  expect(parseNativeProfileMatch("")).toBeNull();
+  expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull();
+  expect(parseNativeProfileMatch('{"ok":true,"profile":{"annotated":-1}}\n')).toBeNull();
+});

packages/compiler/src/backend/native-profile.ts

@@ -44,6 +44,45 @@ export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefine
   return [];
 }
 
+/** How an optimized build's profile matched the program's functions, as the
+ * helper reports it: functions that took the profile's counts, functions
+ * whose control flow changed since the profile was collected, functions the
+ * profile has no record of (new, or never run while it was collected), and
+ * partitions reused from the cache without their counts. */
+export interface NativeProfileMatch {
+  annotated: number;
+  mismatched: number;
+  missing: number;
+  unknownPartitions: number;
+}
+
+/** The profile match in a successful helper emit's output, or null when the
+ * helper reports none (a helper that predates the report). */
+export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | null {
+  for (const line of stdout.split("\n")) {
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const profile = (parsed as { profile?: Record<string, unknown> } | null)?.profile;
+    if (typeof profile !== "object" || profile === null) continue;
+    const count = (key: string): number | null => {
+      const value = profile[key];
+      return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null;
+    };
+    const annotated = count("annotated");
+    const mismatched = count("mismatched");
+    const missing = count("missing");
+    const unknownPartitions = count("unknown_partitions");
+    if (annotated === null || mismatched === null || missing === null || unknownPartitions === null)
+      continue;
+    return { annotated, mismatched, missing, unknownPartitions };
+  }
+  return null;
+}
+
 /** Cache-key part; empty without a profile so other keys are unchanged. */
 export function nativeProfileKey(profile: NativeProfileOptions | undefined): string {
   if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`;

packages/compiler/src/compile-types.ts

@@ -2,6 +2,7 @@ import type { TargetCpu } from "./backend/target-cpu.js";
 import type { WindowsSubsystem } from "./backend/targets.js";
 import type { NativeLinkInfo } from "./backend/native-link-info.js";
 import type { NativeOptimization } from "./backend/optimization.js";
+import type { NativeProfileMatch } from "./backend/native-profile.js";
 import type { ScrDiagnostic } from "./diagnostics/diagnostic.js";
 import type { CoverageInput } from "./coverage/report.js";
 import type { TierMode } from "./frontend/tier-analysis.js";
@@ -100,6 +101,8 @@ export type CompileArtifact =
       path: string;
       translationUnitPath: string;
       backend: "llvm";
+      /** How a `profileUse` build's profile matched the program's functions. */
+      profileMatch?: NativeProfileMatch;
     };
 
 export type CompileFailure = {

packages/compiler/src/index.ts

@@ -94,6 +94,7 @@ import { createNativeLinkPlan } from "./backend/link-plan.js";
 import {
   nativeProfileKey,
   nativeProfileRefusal,
+  type NativeProfileMatch,
   type NativeProfileOptions,
 } from "./backend/native-profile.js";
 import {
@@ -659,6 +660,7 @@ interface NativeProgramObject {
   partitionPaths: string[];
   artifactPath: string;
   dependencies: NativeArtifactDependency[];
+  profileMatch?: NativeProfileMatch;
 }
 
 async function emitNativeProgramObject(
@@ -694,6 +696,7 @@ async function emitNativeProgramObject(
       partitionPaths: artifact.outputPaths.slice(1),
       artifactPath,
       dependencies: artifact.dependencies,
+      ...(artifact.profileMatch === undefined ? {} : { profileMatch: artifact.profileMatch }),
     };
   } catch (error) {
     await removeNativeProgramObject(linkPath, nativePartitionPaths(linkPath, partitions));
@@ -1851,6 +1854,7 @@ async function compileTracked(
   }
   await pruneBuildCache(cacheRoot);
   timing("complete");
+  const profileMatch = nativeProgramObject?.profileMatch;
   return {
     ok: true,
     artifact: {
@@ -1858,6 +1862,7 @@ async function compileTracked(
       path: opts.outPath,
       translationUnitPath: llvmPath,
       backend,
+      ...(profileMatch === undefined ? {} : { profileMatch }),
     },
     binaryPath: opts.outPath,
     llvmPath,

scripts/build-native-cli.mts

@@ -20,6 +20,12 @@ const sanitize = process.env["SCRIPTC_SAN"] === "1";
 // SCRIPTC_NATIVE_PROFILE_USE names; an empty SCRIPTC_NATIVE_PROFILE_USE
 // builds without one.
 const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined;
+// Below this share of matching functions, the build warns that the profile
+// has lost most of its gain. A profile refreshed on the same commit matched
+// 92.7% of the native compiler's profiled functions, one refreshed 1.5 days
+// and about 200 commits earlier 87.2%, and that one kept about half of the
+// fresh profile's gain on a cold tsc-ts build.
+const PROFILE_MATCH_WARNING = 0.9;
 const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz");
 const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"];
 const useProfile =
@@ -127,6 +133,25 @@ if (!result.ok)
       )
       .join("\n"),
   );
+// How well the profile still describes the compiler: a function whose
+// control flow changed since the profile was collected is optimized as
+// without one. The share falls as compiler code changes, and with it the
+// profile's gain (refresh it with scripts/refresh-cli-profile.mjs).
+const match = useProfile && "artifact" in result ? result.artifact.profileMatch : undefined;
+if (match !== undefined) {
+  const profiled = match.annotated + match.mismatched;
+  const share = profiled === 0 ? 0 : match.annotated / profiled;
+  console.log(
+    `native compiler profile: ${match.annotated} of ${profiled} profiled functions match (${(100 * share).toFixed(1)}%); ${match.missing} functions have no profile` +
+      (match.unknownPartitions > 0
+        ? `; ${match.unknownPartitions} partitions reused without counts`
+        : ""),
+  );
+  if (match.unknownPartitions === 0 && share < PROFILE_MATCH_WARNING)
+    console.warn(
+      `warning: the native compiler profile matches ${(100 * share).toFixed(1)}% of the functions it describes; run scripts/refresh-cli-profile.mjs`,
+    );
+}
 const compilerVersion = (
   JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as {
     version: string;