From 2d2343e36fd19bd47f1247f143dd58227403b6f2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 05:03:24 -0500 Subject: [PATCH] Keep program partitions local to the code an edit changes --- native/llvm-codegen/README.md | 2 +- native/llvm-codegen/src/partition.cpp | 178 ++++++++++++------ native/llvm-codegen/src/partition.h | 14 +- .../compiler/test/native-partitions.test.ts | 109 ++++++++++- 4 files changed, 231 insertions(+), 72 deletions(-) diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index 02ac6e3ef..0af8a11d2 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -33,7 +33,7 @@ lowering, verifies before and after optimization, and publishes through a private sibling file so a failed or interrupted request cannot truncate the requested output. -Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host. +Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions that no other definition reaches are left out of every partition, and the others become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host. WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled. diff --git a/native/llvm-codegen/src/partition.cpp b/native/llvm-codegen/src/partition.cpp index 93b9af4a3..ef5230129 100644 --- a/native/llvm-codegen/src/partition.cpp +++ b/native/llvm-codegen/src/partition.cpp @@ -98,7 +98,7 @@ void placeGroups(std::vector &Symbols, ArrayRef Roots, StringMap GroupBytes; for (unsigned I = 0; I < Symbols.size(); ++I) { const Symbol &S = Symbols[I]; - if (S.Kind != SymbolKind::Function) + if (S.Kind != SymbolKind::Function || S.Partition == NoPartition) continue; Total += Bytes(S); StringRef Group = Symbols[Roots[I]].Group; @@ -119,8 +119,10 @@ void placeGroups(std::vector &Symbols, ArrayRef Roots, StringMap KeyBytes; for (unsigned I = 0; I < Symbols.size(); ++I) { const Symbol &S = Symbols[I]; + if (S.Kind != SymbolKind::Function || S.Partition == NoPartition) + continue; StringRef Group = Symbols[Roots[I]].Group; - if (S.Kind != SymbolKind::Function || Group.empty()) + if (Group.empty()) continue; uint64_t Chunks = 1; while (Chunks * Chunk < GroupBytes[Group]) @@ -534,64 +536,6 @@ std::optional partitionProgramText(StringRef Text, } } - // An internal function referenced by exactly one other function, and by - // no global, is a whole-program inlining candidate; keep it with that - // function so its partition can inline it without importing. A function - // body larger than this is past LLVM's inline cost limit even for the only - // call to a local function, so it is placed by its own name. Otherwise a - // program's module initializers, which only the entry point calls, would - // all share the entry point's partition. - constexpr size_t MaxInlinedBodyBytes = 256 * 1024; - constexpr unsigned Shared = ~1u; - std::vector Referrer(Symbols.size(), NoPartition); - for (unsigned I = 0; I < Symbols.size(); ++I) { - const Symbol &S = Symbols[I]; - if (S.Kind == SymbolKind::Declaration) - continue; - for (unsigned Ref : S.Refs) { - if (Ref == I) - continue; - if (S.Kind == SymbolKind::Variable || Referrer[Ref] != NoPartition) - Referrer[Ref] = Shared; - else - Referrer[Ref] = I; - } - } - std::vector Roots(Symbols.size()); - for (unsigned I = 0; I < Symbols.size(); ++I) { - if (Symbols[I].Kind != SymbolKind::Function) - continue; - unsigned Root = I; - for (unsigned Steps = 0; Steps < 1024; ++Steps) { - const Symbol &S = Symbols[Root]; - unsigned Parent = Referrer[Root]; - if (!S.Local || S.Body.size() > MaxInlinedBodyBytes || - Parent == NoPartition || Parent == Shared || - Symbols[Parent].Kind != SymbolKind::Function || Parent == I) - break; - Root = Parent; - } - Roots[I] = Root; - Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; - } - placeGroups(Symbols, Roots, Count); - for (const Symbol &S : Symbols) { - if (S.Kind != SymbolKind::Function) - continue; - for (unsigned Ref : S.Refs) - if (Symbols[Ref].Kind == SymbolKind::Variable && - Symbols[Ref].Partition == NoPartition) - Symbols[Ref].Partition = S.Partition; - } - for (Symbol &S : Symbols) - if (S.Kind == SymbolKind::Variable && S.Partition == NoPartition) - S.Partition = stableHash(S.Name) % Count; - - ProgramPartitions Result; - for (const Symbol &S : Symbols) - if (S.Kind != SymbolKind::Declaration && !S.Local) - Result.PublicSymbols.insert(S.Name); - // Metadata nodes are numbered in the emitter's global order, so one added // node would renumber the references in every partition. Each partition // instead numbers the nodes it uses in order of first use, which keeps an @@ -606,10 +550,122 @@ std::optional partitionProgramText(StringRef Text, for (StringRef Line : NamedMetadata) scanReferences(Line, Index, NamedSymbols, NamedRoots); + // An internal definition that no other definition reaches is left out of + // every partition: the optimizer would delete it, and the emitter writes + // variants of helpers that the program never calls. Leaving them out + // shortens every partition's parse, summary and optimization, and since + // placement ignores them too, an edit that adds one changes no partition. + std::vector Live(Symbols.size(), 0); + { + std::vector Work; + auto Reach = [&](unsigned I) { + if (!Live[I]) { + Live[I] = 1; + Work.push_back(I); + } + }; + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind != SymbolKind::Declaration && !Symbols[I].Local) + Reach(I); + for (unsigned I : NamedSymbols) + Reach(I); + for (auto &Entry : Metadata) + for (unsigned I : Entry.second.Symbols) + Reach(I); + while (!Work.empty()) { + unsigned I = Work.back(); + Work.pop_back(); + for (unsigned Ref : Symbols[I].Refs) + Reach(Ref); + } + } + + // An internal definition reached from exactly one other definition is an + // inlining candidate of it, or data only it uses; keep it with that + // definition so its partition can inline it without importing, and so a + // helper an edit adds lands in the partition of the code that uses it. A + // chain through a global continues to the function that uses the global. + // A function body larger than this is past LLVM's inline cost limit even + // for the only call to a local function, so it is placed by its own name. + // A chain stops where it would leave a function's source-module group: + // module initializers, which only the entry point calls, keep their own + // partitions instead of all sharing the entry point's. + constexpr size_t MaxInlinedBodyBytes = 256 * 1024; + constexpr unsigned Shared = ~1u; + std::vector Referrer(Symbols.size(), NoPartition); + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind == SymbolKind::Declaration || !Live[I]) + continue; + for (unsigned Ref : S.Refs) { + if (Ref == I) + continue; + Referrer[Ref] = Referrer[Ref] == NoPartition ? I : Shared; + } + } + auto Follows = [&](unsigned Child, unsigned Parent) { + const Symbol &S = Symbols[Child]; + if (!S.Local || Parent == NoPartition || Parent == Shared || + Symbols[Parent].Kind == SymbolKind::Declaration) + return false; + if (S.Kind == SymbolKind::Function && + (S.Body.size() > MaxInlinedBodyBytes || + (Symbols[Parent].Kind == SymbolKind::Function && !S.Group.empty() && + S.Group != Symbols[Parent].Group))) + return false; + return true; + }; + std::vector Roots(Symbols.size()); + for (unsigned I = 0; I < Symbols.size(); ++I) { + if (Symbols[I].Kind == SymbolKind::Declaration || !Live[I]) + continue; + unsigned Root = I; + for (unsigned Steps = 0; Steps < 1024; ++Steps) { + unsigned Parent = Referrer[Root]; + if (!Follows(Root, Parent) || Parent == I) + break; + Root = Parent; + } + // A function whose chain ends at a global that is not itself placed + // with a function (a table many functions use) is placed by its own + // name, so the functions of one table spread over the partitions. + if (Symbols[I].Kind == SymbolKind::Function && + Symbols[Root].Kind != SymbolKind::Function) + Root = I; + Roots[I] = Root; + if (Symbols[I].Kind == SymbolKind::Function) + Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; + } + placeGroups(Symbols, Roots, Count); + // A global follows the function its chain ends at, then the first + // function that references it, then its own name. + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Variable && Live[I] && + Symbols[Roots[I]].Kind == SymbolKind::Function) + Symbols[I].Partition = Symbols[Roots[I]].Partition; + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind != SymbolKind::Function || !Live[I]) + continue; + for (unsigned Ref : S.Refs) + if (Symbols[Ref].Kind == SymbolKind::Variable && + Symbols[Ref].Partition == NoPartition) + Symbols[Ref].Partition = S.Partition; + } + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Variable && Live[I] && + Symbols[I].Partition == NoPartition) + Symbols[I].Partition = stableHash(Symbols[I].Name) % Count; + + ProgramPartitions Result; + for (const Symbol &S : Symbols) + if (S.Kind != SymbolKind::Declaration && !S.Local) + Result.PublicSymbols.insert(S.Name); + Result.Sources.resize(Count); std::vector> Owned(Count); for (unsigned I = 0; I < Symbols.size(); ++I) - if (Symbols[I].Kind != SymbolKind::Declaration) + if (Symbols[I].Kind != SymbolKind::Declaration && Live[I]) Owned[Symbols[I].Partition].push_back(I); auto Assemble = [&](unsigned P) { diff --git a/native/llvm-codegen/src/partition.h b/native/llvm-codegen/src/partition.h index 35ef4c543..dc000d751 100644 --- a/native/llvm-codegen/src/partition.h +++ b/native/llvm-codegen/src/partition.h @@ -24,11 +24,15 @@ struct ProgramPartitions { // more partitions, functions under a "; scriptc-group " line are placed // with the other functions of that source-module group, which is divided // into chunks when it is larger than the mean partition or 1 MiB; other -// functions are placed by a stable hash of their name. An internal function referenced by exactly one other function stays -// with that function unless its body is too large to inline, and a global -// stays with the first function that references it. Returns nullopt for any -// construct outside the emitter's narrow top-level grammar; callers then -// split a parsed module. +// functions are placed by a stable hash of their name. An internal +// definition referenced by exactly one other definition stays with it, also +// through a global, unless it is a function whose body is too large to +// inline or that would join a function of another source-module group; a +// global otherwise stays with the first function that references it. +// Internal definitions that no non-internal definition, named metadata or +// metadata node reaches are left out of every partition. Returns nullopt +// for any construct outside the emitter's narrow top-level grammar; callers +// then split a parsed module. std::optional partitionProgramText(llvm::StringRef Text, unsigned Count); diff --git a/packages/compiler/test/native-partitions.test.ts b/packages/compiler/test/native-partitions.test.ts index fcd0132e8..113cd95ad 100644 --- a/packages/compiler/test/native-partitions.test.ts +++ b/packages/compiler/test/native-partitions.test.ts @@ -204,12 +204,111 @@ describe.runIf(helper !== null)("LLVM program partitions", () => { expect([...after].filter((entry) => !sources.has(entry))).toHaveLength(1); expect(await exitCode(dir, edited, "edited")).toBe(expectedExit(7, steps)); - // With fewer partitions, functions keep their placement by name. - const contents = (paths: string[]) => Promise.all(paths.map((path) => readFile(path))); + // With fewer partitions, functions keep their placement by name, and a + // group line only stops a single-caller chain: step0 joins main without + // groups, and keeps its own placement in group 0. const few = { outputs: 4, optLevel: "0" } as const; - expect( - await contents(await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" })), - ).toEqual(await contents(await emit(dir, program(), { ...few, name: "few-plain" }))); + const placed = async (objects: string[]) => { + const where = new Map(); + for (const [index, object] of objects.entries()) { + const { stdout } = await execFileAsync("nm", [object]); + for (const line of stdout.split("\n")) { + const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step\d+|main)$/.exec(line.trim()); + if (defined !== null) where.set(defined[1]!, index); + } + } + return where; + }; + const fewGrouped = await placed( + await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" }), + ); + const fewPlain = await placed(await emit(dir, program(), { ...few, name: "few-plain" })); + expect(fewPlain.get("step0")).toBe(fewPlain.get("main")); + for (let step = 1; step < 24; step++) + expect(fewGrouped.get(`step${step}`)).toBe(fewPlain.get(`step${step}`)); + expect(fewGrouped.get("main")).toBe(fewPlain.get("main")); + }); + + test("internal definitions nothing reaches are left out of every partition", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-unreachable-")); + dirs.push(dir); + // An unreferenced internal function that uses live code and its own + // state, which only it references. + const unused = (name: string) => + [ + `@${name}State = internal global i64 7`, + `define internal void @${name}(i64 %value) {`, + ` %state = load i64, ptr @${name}State`, + " call void @bump(i64 %state)", + " ret void", + "}", + ].join("\n"); + const cache = join(dir, "cache"); + const options = { outputs: 4, optLevel: "0", cache } as const; + const objects = await emit(dir, program(0, unused("unusedOne")), { ...options, name: "first" }); + expect(await exitCode(dir, objects, "first")).toBe(expectedExit()); + for (const object of objects) { + const { stdout } = await execFileAsync("nm", [object]); + expect(stdout).not.toMatch(/unusedOne/); + } + // Adding another one changes no partition. + const before = await cacheEntries(cache, ".o"); + const more = await emit(dir, program(0, `${unused("unusedOne")}\n${unused("unusedTwo")}`), { + ...options, + name: "more", + }); + expect(await cacheEntries(cache, ".o")).toEqual(before); + expect(await exitCode(dir, more, "more")).toBe(expectedExit()); + }); + + test("a helper that one function uses goes to that function's partition", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-helpers-")); + dirs.push(dir); + // step7 calls @helper directly, and step9 calls @viaTable through an + // internal global only step9 reads. + const helpers = (withHelpers: boolean) => + program() + .split("\n") + .flatMap((line) => { + if (!withHelpers) return [line]; + if (line === "define internal void @step7(i64 %value) noinline {") + return [ + "define internal void @helper(i64 %value) noinline {", + " call void @bump(i64 %value)", + " ret void", + "}", + "@table = internal constant ptr @viaTable", + "define internal void @viaTable(i64 %value) noinline {", + " call void @bump(i64 %value)", + " ret void", + "}", + line, + " call void @helper(i64 0)", + ]; + if (line === "define internal void @step9(i64 %value) noinline {") + return [line, " %fn = load ptr, ptr @table", " call void %fn(i64 0)"]; + return [line]; + }) + .join("\n"); + const cache = join(dir, "cache"); + const options = { outputs: 4, optLevel: "0", cache } as const; + await emit(dir, helpers(false), { ...options, name: "before" }); + const before = await cacheEntries(cache, ".o"); + const objects = await emit(dir, helpers(true), { ...options, name: "after" }); + expect(await exitCode(dir, objects, "after")).toBe(expectedExit()); + const where = new Map(); + for (const [index, object] of objects.entries()) { + const { stdout } = await execFileAsync("nm", [object]); + for (const line of stdout.split("\n")) { + const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step7|step9|helper|viaTable)$/.exec(line.trim()); + if (defined !== null) where.set(defined[1]!, index); + } + } + expect(where.get("helper")).toBe(where.get("step7")); + expect(where.get("viaTable")).toBe(where.get("step9")); + // Only the partitions of step7 and step9 change. + const changed = [...(await cacheEntries(cache, ".o"))].filter((entry) => !before.has(entry)); + expect(changed).toHaveLength(new Set([where.get("step7"), where.get("step9")]).size); }); test("a table written one element per line keeps the text partitioner and its reuse", async () => {