From 0933548b8de1e18c8a7e3da3b58e30471464fd3f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 30 Oct 2026 00:51:09 -0500 Subject: [PATCH] Simplify optimized partitions at O1 before the thin link An optimized program divided into partitions was simplified at O2 before the thin link and optimized again at O2 after importing, so the work on each partition's largest functions ran twice. The pre-link now runs the O1 simplification, which still annotates the profile and inlines and simplifies enough for the summaries the import decisions read; the backend's O2 optimization is unchanged. A program optimized as one module is unchanged. The O1 simplification has no correlated value propagation. Without it, SimplifyCFG folds a jammed loop's early exit and its trip-count exit into one branch, the loop's latch no longer exits, and the backend's unroller keeps a trip-count test in every unrolled iteration. The pre-link runs the pass once per function after simplifying it. --- native/llvm-codegen/src/emit.cpp | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index f5136c2e5..7c28298b0 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -54,6 +54,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h" #include "llvm/Transforms/Utils/SplitModule.h" #include @@ -872,6 +873,35 @@ std::optional splitParsedModule(const EmitOptions &Options, return std::nullopt; } +// The level of an optimized partition's simplification before the thin +// link. The backend optimizes every partition again at Level after +// importing, so the pre-link runs the O1 simplification: it annotates the +// profile, inlines and simplifies enough for the summaries the import +// decisions read, and leaves the full simplification to the backend. The +// O2 simplification before the link repeated most of the backend's work: on +// a partition dominated by one large function it took about half as long as +// the backend's optimization and code generation of that partition. +OptimizationLevel preLinkLevel(OptimizationLevel Level) { + if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3) + return OptimizationLevel::O1; + return Level; +} + +// Adds correlated value propagation, which the O2 simplification runs and +// the O1 one does not, once per function after the O1 pre-link simplifies +// it. On the edge where a loop's early exit is taken, it replaces the values +// the exit condition fixed with constants. Without it, SimplifyCFG folds a +// jammed loop's early exit and its trip-count exit into one branch, the +// latch no longer exits, and the backend's unroller keeps a trip-count test +// in every unrolled iteration. +void addPreLinkValuePropagation(PassBuilder &PB) { + PB.registerCGSCCOptimizerLateEPCallback( + [](CGSCCPassManager &CGPM, OptimizationLevel) { + CGPM.addPass( + createCGSCCToFunctionPassAdaptor(CorrelatedValuePropagationPass())); + }); +} + // Runs Work(I) for every partition on a pool sized to the host. Outputs never // depend on the pool size or scheduling order. void forEachPartition(size_t Count, function_ref Work) { @@ -1051,7 +1081,10 @@ compileOptimizedPartitions(const EmitOptions &Options, P.collectAnnotated(Weigher.Annotated); P.afterProfileAnnotation(PartitionMatch); } - ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); + if (preLinkLevel(Level) != Level) + addPreLinkValuePropagation(P.PB); + ModulePassManager MPM = + P.PB.buildThinLTOPreLinkDefaultPipeline(preLinkLevel(Level)); MPM.addPass(UnnameStructTypes()); // The module hash is set at the thin link instead (see Names). MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false,