diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 9cae164d3..35d7ce4f0 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/IR/DiagnosticHandler.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LegacyPassManager.h" @@ -41,11 +42,13 @@ #include "llvm/Support/SourceMgr.h" #include "llvm/Support/ThreadPool.h" #include "llvm/Support/Threading.h" +#include "llvm/Support/JSON.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/SplitModule.h" +#include #include #include #include @@ -170,6 +173,108 @@ void limitProfileImports() { (void)Limited; } +// How the --profile-use profile matched the functions it annotated, summed +// over partitions: functions that received its counts, functions whose +// control flow changed since it was collected (hash mismatch), and functions +// it has no record of (new, or never run in training: the merged profile is +// sparse). A partition reused from the cache without its counts is counted +// in Unknown. +struct ProfileMatch { + std::atomic Annotated{0}; + std::atomic Mismatched{0}; + std::atomic Missing{0}; + std::atomic Unknown{0}; + + void add(const ProfileMatch &Other) { + Annotated += Other.Annotated; + Mismatched += Other.Mismatched; + Missing += Other.Missing; + } + + std::string serialize() const { + return formatv("{0} {1} {2}", Annotated.load(), Mismatched.load(), + Missing.load()) + .str(); + } + + bool deserialize(StringRef Text) { + SmallVector Parts; + Text.split(Parts, ' '); + uint64_t Values[3]; + if (Parts.size() != 3) + return false; + for (size_t I = 0; I < 3; ++I) + if (Parts[I].getAsInteger(10, Values[I])) + return false; + Annotated += Values[0]; + Mismatched += Values[1]; + Missing += Values[2]; + return true; + } +}; + +// Counts the profile annotation's hash-mismatch and missing-function +// warnings instead of printing them. Every other diagnostic takes the +// context's default handling. +class ProfileMatchHandler : public DiagnosticHandler { +public: + explicit ProfileMatchHandler(ProfileMatch &Match) : Match(Match) {} + + bool handleDiagnostics(const DiagnosticInfo &Info) override { + if (Info.getKind() != DK_PGOProfile) + return false; + std::string Message; + raw_string_ostream Stream(Message); + DiagnosticPrinterRawOStream Printer(Stream); + Info.print(Printer); + if (StringRef(Message).contains("hash mismatch")) + ++Match.Mismatched; + else if (StringRef(Message).contains_insensitive("no profile data")) + ++Match.Missing; + else + return false; + return true; + } + +private: + ProfileMatch &Match; +}; + +// The annotation warns about functions the profile does not describe only +// when asked; the match counts need those warnings. Process-wide, set once. +void reportMissingProfiles() { + static const bool Reported = [] { + if (cl::Option *Option = + cl::getRegisteredOptions().lookup("pgo-warn-missing-function")) + Option->addOccurrence(0, "pgo-warn-missing-function", "true"); + return true; + }(); + (void)Reported; +} + +// Functions of `M` that the profile annotated with counts. +uint64_t annotatedFunctions(const Module &M) { + uint64_t Count = 0; + for (const Function &F : M) { + if (F.isDeclaration()) + continue; + std::optional Entry = F.getEntryCount(); + if (Entry && !Entry->isSynthetic()) + ++Count; + } + return Count; +} + +// Prints the profile match as the successful emit's one line of JSON. +void reportProfileMatch(const ProfileMatch &Match) { + json::Object Profile{{"annotated", Match.Annotated.load()}, + {"mismatched", Match.Mismatched.load()}, + {"missing", Match.Missing.load()}, + {"unknown_partitions", Match.Unknown.load()}}; + json::Object Response{{"ok", true}, {"profile", std::move(Profile)}}; + outs() << formatv("{0}\n", json::Value(std::move(Response))); +} + // Reads the profile's header once, before any partition needs it: it must // hold IR-level counts. std::optional inspectProfile(const EmitOptions &Options) { @@ -387,9 +492,16 @@ std::optional publish(StringRef TemporaryPath, std::optional emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level, std::optional Profile, CodeGenFileType Type, - StringRef OutputPath, std::vector> &Temporaries) { + StringRef OutputPath, std::vector> &Temporaries, + ProfileMatch &Match) { + bool Annotating = Profile && Profile->Action == PGOOptions::IRUse; + if (Annotating) + M.getContext().setDiagnosticHandler( + std::make_unique(Match)); Pipeline P(Machine, Level, std::move(Profile)); P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM); + if (Annotating) + Match.Annotated += annotatedFunctions(M); if (std::optional Failure = verify(M, "post_optimization_verification_failed")) return Failure; @@ -555,20 +667,32 @@ std::optional compileOptimizedPartitions(const EmitOptions &Options, const PartitionPlan &Plan, OptimizationLevel Level, StringRef Salt, const ArtifactCache &Cache, - std::vector> &Objects) { + std::vector> &Objects, + ProfileMatch &Match) { size_t Count = Plan.Modules.size(); + bool Annotating = !Options.ProfileUse.empty(); std::vector> Summaries(Count); SharedFailure Failures; forEachPartition(Count, [&](size_t I) { std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); - if ((Summaries[I] = Cache.lookup(Key, "bc"))) + if ((Summaries[I] = Cache.lookup(Key, "bc"))) { + if (Annotating) { + std::unique_ptr Counts = Cache.lookup(Key, "match"); + if (!Counts || !Match.deserialize(Counts->getBuffer())) + ++Match.Unknown; + } return; + } std::string LookupError; std::unique_ptr Machine = createTargetMachine(Options.Target, Options.OptLevel, LookupError); if (!Machine) return Failures.set({"target_machine_failed", LookupError}); LLVMContext Context; + ProfileMatch PartitionMatch; + if (Annotating) + Context.setDiagnosticHandler( + std::make_unique(PartitionMatch)); Expected> Loaded = loadPartition(Options, Plan.Modules[I], I, Context, *Machine); if (!Loaded) @@ -598,6 +722,12 @@ compileOptimizedPartitions(const EmitOptions &Options, /*EmitModuleHash=*/true)); MPM.run(Mod, P.MAM); } + if (Annotating) { + PartitionMatch.Annotated += annotatedFunctions(Mod); + Match.add(PartitionMatch); + // Stored before the summary, so a reused summary finds its counts. + Cache.store(Key, "match", PartitionMatch.serialize()); + } StringRef Contents(Bitcode.data(), Bitcode.size()); Cache.store(Key, "bc", Contents); Summaries[I] = MemoryBuffer::getMemBufferCopy(Contents); @@ -744,7 +874,8 @@ compileOptimizedPartitions(const EmitOptions &Options, std::optional compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, - std::vector> &Objects) { + std::vector> &Objects, + ProfileMatch &Match) { // Everything besides a partition's own bytes that determines its // artifacts. The caller's cache directory already identifies the exact // helper build; source content and partition count arrive per module. @@ -778,7 +909,7 @@ compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, return compileUnoptimizedPartitions(Options, Plan, Salt, Cache, Objects); return compileOptimizedPartitions(Options, Plan, optimizationLevel(Options.OptLevel), Salt, - Cache, Objects); + Cache, Objects, Match); } // Publishes in-memory partition objects through private sibling files. @@ -819,6 +950,8 @@ int emitPartitions(const EmitOptions &Options) { std::vector> Objects; std::optional Failure; bool Compiled = false; + // A failed attempt's counts are discarded with its objects. + auto Match = std::make_unique(); { ErrorOr> Input = MemoryBuffer::getFile(Options.Input, /*IsText=*/false, @@ -830,15 +963,16 @@ int emitPartitions(const EmitOptions &Options) { Input->reset(); PartitionPlan Plan{std::move(Text->Sources), std::move(Text->PublicSymbols)}; - Compiled = !compilePartitions(Options, Plan, Objects); + Compiled = !compilePartitions(Options, Plan, Objects, *Match); } } } if (!Compiled) { + Match = std::make_unique(); PartitionPlan Plan; Failure = splitParsedModule(Options, Plan); if (!Failure) - Failure = compilePartitions(Options, Plan, Objects); + Failure = compilePartitions(Options, Plan, Objects, *Match); } std::vector> Temporaries; if (!Failure) @@ -851,6 +985,8 @@ int emitPartitions(const EmitOptions &Options) { return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); } + if (!Options.ProfileUse.empty()) + reportProfileMatch(*Match); return 0; } @@ -895,6 +1031,8 @@ int emit(const EmitOptions &Options) { return reportError("invalid_import", "runtime bitcode import requires an optimized build", Options.DiagnosticFormat); + if (!Options.ProfileUse.empty()) + reportMissingProfiles(); if (std::optional Failure = inspectProfile(Options)) return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); @@ -957,9 +1095,10 @@ int emit(const EmitOptions &Options) { ? CodeGenFileType::ObjectFile : CodeGenFileType::AssemblyFile; std::vector> Temporaries; + ProfileMatch Match; std::optional Failure = emitModule(*Mod, *Machine, Level, profileOptions(Options), Type, - Options.Outputs[0], Temporaries); + Options.Outputs[0], Temporaries, Match); if (!Failure) Failure = publish(Temporaries[0], Options.Outputs[0]); if (Failure) { @@ -968,6 +1107,8 @@ int emit(const EmitOptions &Options) { return reportError(Failure->Code, Failure->Message, Options.DiagnosticFormat); } + if (!Options.ProfileUse.empty()) + reportProfileMatch(Match); return 0; } diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 5b91c60e7..413fb7e02 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -30,6 +30,8 @@ import { compilerReleaseVersion } from "../library/sidecar.js"; import { nativeProfileHelperArgs, nativeProfileKey, + parseNativeProfileMatch, + type NativeProfileMatch, type NativeProfileOptions, } from "./native-profile.js"; import { writeTextParts } from "../text-parts.js"; @@ -69,6 +71,8 @@ export interface NativeCodegenArtifact { dependencies: NativeArtifactDependency[]; /** Every emitted file, starting with the requested output path. */ outputPaths: string[]; + /** How the build's profile matched, when it used one and the helper ran. */ + profileMatch?: NativeProfileMatch; } const resolvedHelperCache = new Map>(); @@ -458,7 +462,7 @@ export async function emitNativeArtifact( try { if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); else await writeLlvmSource(input, options.llvm, 0o600); - await invoke(helper.binaryPath, [ + const { stdout } = await invoke(helper.binaryPath, [ "emit", "--input", input, @@ -502,7 +506,9 @@ export async function emitNativeArtifact( for (let index = 0; index < stages.length; index++) await rename(stages[index]!, outputs[index]!); await pruneBuildCache(root); - return artifact; + const profileMatch = + options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); + return profileMatch === null ? artifact : { ...artifact, profileMatch }; } finally { await Promise.all([ ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)), diff --git a/packages/compiler/src/backend/native-profile.test.ts b/packages/compiler/src/backend/native-profile.test.ts index 590c39eef..b2cb0c75d 100644 --- a/packages/compiler/src/backend/native-profile.test.ts +++ b/packages/compiler/src/backend/native-profile.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "vitest"; import { executableLinkInputs } from "./link-plan-core.js"; import { + parseNativeProfileMatch, nativeProfileHelperArgs, nativeProfileKey, nativeProfileRefusal, @@ -71,3 +72,15 @@ test("only the instrumented link adds the profile runtime", () => { "-fprofile-generate", ); }); + +test("the helper's profile match is read from its output", () => { + expect( + parseNativeProfileMatch( + '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":1}}\n', + ), + ).toEqual({ annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 1 }); + // A helper that predates the report prints nothing. + expect(parseNativeProfileMatch("")).toBeNull(); + expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull(); + expect(parseNativeProfileMatch('{"ok":true,"profile":{"annotated":-1}}\n')).toBeNull(); +}); diff --git a/packages/compiler/src/backend/native-profile.ts b/packages/compiler/src/backend/native-profile.ts index 4cd766319..904994a7d 100644 --- a/packages/compiler/src/backend/native-profile.ts +++ b/packages/compiler/src/backend/native-profile.ts @@ -44,6 +44,45 @@ export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefine return []; } +/** How an optimized build's profile matched the program's functions, as the + * helper reports it: functions that took the profile's counts, functions + * whose control flow changed since the profile was collected, functions the + * profile has no record of (new, or never run while it was collected), and + * partitions reused from the cache without their counts. */ +export interface NativeProfileMatch { + annotated: number; + mismatched: number; + missing: number; + unknownPartitions: number; +} + +/** The profile match in a successful helper emit's output, or null when the + * helper reports none (a helper that predates the report). */ +export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | null { + for (const line of stdout.split("\n")) { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const profile = (parsed as { profile?: Record } | null)?.profile; + if (typeof profile !== "object" || profile === null) continue; + const count = (key: string): number | null => { + const value = profile[key]; + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : null; + }; + const annotated = count("annotated"); + const mismatched = count("mismatched"); + const missing = count("missing"); + const unknownPartitions = count("unknown_partitions"); + if (annotated === null || mismatched === null || missing === null || unknownPartitions === null) + continue; + return { annotated, mismatched, missing, unknownPartitions }; + } + return null; +} + /** Cache-key part; empty without a profile so other keys are unchanged. */ export function nativeProfileKey(profile: NativeProfileOptions | undefined): string { if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`; diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index c50931c6a..2a5cea5ea 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -2,6 +2,7 @@ import type { TargetCpu } from "./backend/target-cpu.js"; import type { WindowsSubsystem } from "./backend/targets.js"; import type { NativeLinkInfo } from "./backend/native-link-info.js"; import type { NativeOptimization } from "./backend/optimization.js"; +import type { NativeProfileMatch } from "./backend/native-profile.js"; import type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; import type { CoverageInput } from "./coverage/report.js"; import type { TierMode } from "./frontend/tier-analysis.js"; @@ -100,6 +101,8 @@ export type CompileArtifact = path: string; translationUnitPath: string; backend: "llvm"; + /** How a `profileUse` build's profile matched the program's functions. */ + profileMatch?: NativeProfileMatch; }; export type CompileFailure = { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index e2d1c97cc..607298427 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -94,6 +94,7 @@ import { createNativeLinkPlan } from "./backend/link-plan.js"; import { nativeProfileKey, nativeProfileRefusal, + type NativeProfileMatch, type NativeProfileOptions, } from "./backend/native-profile.js"; import { @@ -659,6 +660,7 @@ interface NativeProgramObject { partitionPaths: string[]; artifactPath: string; dependencies: NativeArtifactDependency[]; + profileMatch?: NativeProfileMatch; } async function emitNativeProgramObject( @@ -694,6 +696,7 @@ async function emitNativeProgramObject( partitionPaths: artifact.outputPaths.slice(1), artifactPath, dependencies: artifact.dependencies, + ...(artifact.profileMatch === undefined ? {} : { profileMatch: artifact.profileMatch }), }; } catch (error) { await removeNativeProgramObject(linkPath, nativePartitionPaths(linkPath, partitions)); @@ -1851,6 +1854,7 @@ async function compileTracked( } await pruneBuildCache(cacheRoot); timing("complete"); + const profileMatch = nativeProgramObject?.profileMatch; return { ok: true, artifact: { @@ -1858,6 +1862,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: llvmPath, backend, + ...(profileMatch === undefined ? {} : { profileMatch }), }, binaryPath: opts.outPath, llvmPath, diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index e539cea1d..30e0ae39b 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -20,6 +20,12 @@ const sanitize = process.env["SCRIPTC_SAN"] === "1"; // SCRIPTC_NATIVE_PROFILE_USE names; an empty SCRIPTC_NATIVE_PROFILE_USE // builds without one. const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined; +// Below this share of matching functions, the build warns that the profile +// has lost most of its gain. A profile refreshed on the same commit matched +// 92.7% of the native compiler's profiled functions, one refreshed 1.5 days +// and about 200 commits earlier 87.2%, and that one kept about half of the +// fresh profile's gain on a cold tsc-ts build. +const PROFILE_MATCH_WARNING = 0.9; const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz"); const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"]; const useProfile = @@ -127,6 +133,25 @@ if (!result.ok) ) .join("\n"), ); +// How well the profile still describes the compiler: a function whose +// control flow changed since the profile was collected is optimized as +// without one. The share falls as compiler code changes, and with it the +// profile's gain (refresh it with scripts/refresh-cli-profile.mjs). +const match = useProfile && "artifact" in result ? result.artifact.profileMatch : undefined; +if (match !== undefined) { + const profiled = match.annotated + match.mismatched; + const share = profiled === 0 ? 0 : match.annotated / profiled; + console.log( + `native compiler profile: ${match.annotated} of ${profiled} profiled functions match (${(100 * share).toFixed(1)}%); ${match.missing} functions have no profile` + + (match.unknownPartitions > 0 + ? `; ${match.unknownPartitions} partitions reused without counts` + : ""), + ); + if (match.unknownPartitions === 0 && share < PROFILE_MATCH_WARNING) + console.warn( + `warning: the native compiler profile matches ${(100 * share).toFixed(1)}% of the functions it describes; run scripts/refresh-cli-profile.mjs`, + ); +} const compilerVersion = ( JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string;