From 3ea231d70dbf6eae42780a7c9e98d011623cdba0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 9 Oct 2026 19:51:47 -0500 Subject: [PATCH] Optimize and generate native code in parallel partitions Large executables were simplified and inlined as one module before the optimizer split them, so most of a release build ran on a single core, and any edit recompiled the whole program. The LLVM helper now divides the program before optimization. It splits the emitter's assembly text directly, so partitions also parse concurrently, and falls back to LLVM's module splitter for any input outside the emitter's grammar. Internal functions referenced by a single function stay with that function, and globals stay with their first user. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link imports small functions across partitions for inlining and returns unexported hidden definitions to internal linkage, then each partition is optimized and compiled on its own thread. Unoptimized partitions compile independently, so dev builds of large programs are parallel too. A private per-helper cache directory stores simplified partitions under the hash of their source and objects under the hash of the module after importing, so a rebuild recompiles only the partitions an edit can affect. To keep unchanged partitions identical across edits, each partition numbers its metadata in first-use order, lists only the debug compile units it uses, and orders declarations by name, and interned string constants are named by their content rather than first-use order. Partition counts are powers of two that depend only on the module size, up to 64, so output stays the same on every host and functions keep their partition across ordinary edits. LLVM text that arrives in parts is written in large batches and handed to the helper in place instead of being written a second time. --- docs/content/docs/how-it-works.mdx | 2 +- native/llvm-codegen/CMakeLists.txt | 5 +- native/llvm-codegen/README.md | 4 + native/llvm-codegen/src/cache.cpp | 108 ++++ native/llvm-codegen/src/cache.h | 38 ++ native/llvm-codegen/src/emit.cpp | 499 ++++++++++++--- native/llvm-codegen/src/emit.h | 2 + native/llvm-codegen/src/main.cpp | 2 +- native/llvm-codegen/src/partition.cpp | 583 ++++++++++++++++++ native/llvm-codegen/src/partition.h | 31 + native/llvm-codegen/src/target.cpp | 32 +- native/llvm-codegen/src/target.h | 9 +- packages/compiler/src/backend/llvm/emitter.ts | 28 +- .../src/backend/native-codegen-core.ts | 27 +- .../src/backend/native-codegen.test.ts | 47 +- .../compiler/src/backend/native-codegen.ts | 62 +- packages/compiler/src/backend/native-tools.ts | 5 + packages/compiler/src/index.ts | 19 +- packages/compiler/src/native/cache.test.ts | 32 +- packages/compiler/src/native/cache.ts | 36 ++ packages/compiler/src/native/compiler.ts | 16 +- .../compiler/test/native-partitions.test.ts | 234 +++++++ 22 files changed, 1682 insertions(+), 139 deletions(-) create mode 100644 native/llvm-codegen/src/cache.cpp create mode 100644 native/llvm-codegen/src/cache.h create mode 100644 native/llvm-codegen/src/partition.cpp create mode 100644 native/llvm-codegen/src/partition.h create mode 100644 packages/compiler/test/native-partitions.test.ts diff --git a/docs/content/docs/how-it-works.mdx b/docs/content/docs/how-it-works.mdx index 0d59767de..a3d37fff1 100644 --- a/docs/content/docs/how-it-works.mdx +++ b/docs/content/docs/how-it-works.mdx @@ -27,7 +27,7 @@ Runtime pack and linker -> executable 1. **Frontend:** TypeScript parses and checks the program against `es2025`, scriptc's ambient declarations, and available project declarations. `tsconfig.json` controls checker strictness. The compiler uses the checked types and narrowing information to lower supported operations; unsupported operations produce diagnostics. 2. **Typed IR:** A validated, serializable intermediate representation connects the frontend and backend. Types are concrete: generic functions are specialized, unions use tags, and closures have explicit captures. `--emit=ir` writes this representation as JSON. -3. **LLVM backend:** `--emit=llvm` writes textual LLVM IR. The matching LLVM 22 helper generates assembly or objects on supported macOS, Linux, and Windows hosts, including output targeting WASI. Executable and library builds use this backend. Large optimized native executables are inlined as a whole program, then finish optimization and code generation in parallel partitions; the partition count depends only on the program, so output is the same on every host. +3. **LLVM backend:** `--emit=llvm` writes textual LLVM IR. The matching LLVM 22 helper generates assembly or objects on supported macOS, Linux, and Windows hosts, including output targeting WASI. Executable and library builds use this backend. Large native executables are divided into partitions that are optimized and compiled in parallel; optimized builds summarize every partition first so small functions can still be inlined across partitions. The partition count depends only on the program, so output is the same on every host, and a rebuild reuses the compiled partitions that an edit does not affect. 4. **Linking:** Runtime packs contain precompiled C objects and vendor archives. A hashed manifest selects artifacts for the program's features, optimization mode, and library state model. The target linker and SDK or sysroot produce the executable. Sanitizer builds use an instrumented C runtime. ## Inspect compiler output diff --git a/native/llvm-codegen/CMakeLists.txt b/native/llvm-codegen/CMakeLists.txt index 4e57ccb3e..a0d11d574 100644 --- a/native/llvm-codegen/CMakeLists.txt +++ b/native/llvm-codegen/CMakeLists.txt @@ -75,9 +75,11 @@ if(TARGET zstd::libzstd_shared AND TARGET zstd::libzstd_static) endif() add_executable(scriptc-llvm-codegen + src/cache.cpp src/diagnostics.cpp src/emit.cpp src/main.cpp + src/partition.cpp src/runtime-import.cpp src/runtime-unit.cpp src/target.cpp @@ -178,7 +180,7 @@ if(SCRIPTC_STATIC_LINUX_BUILD) endif() endfunction() - set(SCRIPTC_LLVM_LINK_ROOTS ${SCRIPTC_LLVM_TARGET_LIBS} BitReader BitWriter IRReader Linker Passes TransformUtils) + set(SCRIPTC_LLVM_LINK_ROOTS ${SCRIPTC_LLVM_TARGET_LIBS} BitReader BitWriter IRReader LTO Linker Passes TransformUtils) foreach(component IN LISTS SCRIPTC_LLVM_LINK_ROOTS) if(TARGET "LLVM${component}") scriptc_collect_llvm_target("LLVM${component}") @@ -204,6 +206,7 @@ else() Core Coroutines IRReader + LTO Linker Passes Support diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index 377d3a3a8..e651fd188 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -33,6 +33,10 @@ lowering, verifies before and after optimization, and publishes through a private sibling file so a failed or interrupted request cannot truncate the requested output. +Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host. + +`--cache-dir ` names a private directory the caller owns for one helper build. Simplified partitions are stored under the hash of their exact source and objects under the hash of the exact module after importing, each with a trailing digest and published by rename, so a rebuild recompiles only the partitions an edit can affect. + `--optimization=speed` program builds pass `--import-bitcode ` once per runtime-pack unit the program links. Small runtime functions reachable from the program's runtime calls are imported as `available_externally` so LLVM can inline them; the runtime objects remain the only definitions. The helper never runs a sanitizer pass, so with import it also drops the emitter's inert `sanitize_address` attributes, which would otherwise block that inlining. Without `--import-bitcode` the module is optimized exactly as emitted. Runtime-pack builds emit each unit of the executable `speed` flavor through the helper: diff --git a/native/llvm-codegen/src/cache.cpp b/native/llvm-codegen/src/cache.cpp new file mode 100644 index 000000000..464ef2821 --- /dev/null +++ b/native/llvm-codegen/src/cache.cpp @@ -0,0 +1,108 @@ +#include "cache.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Chrono.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/SHA256.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include +#include + +using namespace llvm; + +namespace scriptc { + +namespace { + +constexpr size_t DigestBytes = 32; + +std::array digest(StringRef Contents) { + SHA256 Hash; + Hash.update(Contents); + return Hash.final(); +} + +} // namespace + +std::string hashParts(ArrayRef Parts) { + SHA256 Hash; + for (StringRef Part : Parts) { + uint64_t Size = Part.size(); + uint8_t Length[8]; + for (unsigned I = 0; I < 8; ++I) + Length[I] = static_cast(Size >> (8 * I)); + Hash.update(ArrayRef(Length, 8)); + Hash.update(Part); + } + return toHex(Hash.final(), /*LowerCase=*/true); +} + +std::string ArtifactCache::path(StringRef Key, StringRef Kind) const { + SmallString<256> Path(Directory); + sys::path::append(Path, Key.take_front(2), (Key + "." + Kind).str()); + return std::string(Path); +} + +std::unique_ptr ArtifactCache::lookup(StringRef Key, + StringRef Kind) const { + if (!enabled()) + return nullptr; + std::string Path = path(Key, Kind); + int FD = -1; + if (sys::fs::openFileForRead(Path, FD)) + return nullptr; + ErrorOr> Buffer = + MemoryBuffer::getOpenFile(sys::fs::convertFDToNativeFile(FD), Path, + /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + if (Buffer && (*Buffer)->getBufferSize() > DigestBytes) { + StringRef Data = (*Buffer)->getBuffer(); + StringRef Contents = Data.drop_back(DigestBytes); + std::array Expected = digest(Contents); + if (Data.take_back(DigestBytes) == + StringRef(reinterpret_cast(Expected.data()), + DigestBytes)) { + (void)sys::fs::setLastAccessAndModificationTime( + FD, std::chrono::system_clock::now()); + sys::Process::SafelyCloseFileDescriptor(FD); + return MemoryBuffer::getMemBufferCopy(Contents, Path); + } + } + sys::Process::SafelyCloseFileDescriptor(FD); + return nullptr; +} + +void ArtifactCache::store(StringRef Key, StringRef Kind, + StringRef Contents) const { + if (!enabled()) + return; + std::string Path = path(Key, Kind); + if (sys::fs::create_directories(sys::path::parent_path(Path), + /*IgnoreExisting=*/true, + sys::fs::perms::owner_all)) + return; + SmallString<256> Temporary(sys::path::parent_path(Path)); + sys::path::append(Temporary, ".tmp-%%%%%%%%"); + int FD = -1; + if (sys::fs::createUniqueFile(Temporary, FD, Temporary)) + return; + bool Written; + { + raw_fd_ostream Output(FD, /*shouldClose=*/true); + std::array Digest = digest(Contents); + Output << Contents; + Output.write(reinterpret_cast(Digest.data()), DigestBytes); + Output.close(); + Written = !Output.has_error(); + Output.clear_error(); + } + if (!Written || sys::fs::rename(Temporary, Path)) + sys::fs::remove(Temporary); +} + +} // namespace scriptc diff --git a/native/llvm-codegen/src/cache.h b/native/llvm-codegen/src/cache.h new file mode 100644 index 000000000..a2739300e --- /dev/null +++ b/native/llvm-codegen/src/cache.h @@ -0,0 +1,38 @@ +#pragma once + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" + +#include +#include + +namespace scriptc { + +// Hex SHA-256 of length-prefixed parts, so distinct part lists never collide +// by concatenation. +std::string hashParts(llvm::ArrayRef Parts); + +// A content-addressed store of partition artifacts in a private directory +// owned by the caller. Each entry carries a trailing SHA-256 of its contents +// and is published by rename, so a reader observes either a complete, +// verified entry or a miss. Lookups refresh the entry's modification time so +// the caller's least-recently-used pruning keeps artifacts still in use. +class ArtifactCache { +public: + explicit ArtifactCache(std::string Directory) + : Directory(std::move(Directory)) {} + + bool enabled() const { return !Directory.empty(); } + std::unique_ptr lookup(llvm::StringRef Key, + llvm::StringRef Kind) const; + // Best effort: a full or read-only cache never fails the build. + void store(llvm::StringRef Key, llvm::StringRef Kind, + llvm::StringRef Contents) const; + +private: + std::string path(llvm::StringRef Key, llvm::StringRef Kind) const; + std::string Directory; +}; + +} // namespace scriptc diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index ab12f6238..b39d2c1bf 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -1,30 +1,42 @@ #include "emit.h" +#include "cache.h" #include "diagnostics.h" +#include "partition.h" #include "runtime-import.h" #include "target.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" #include "llvm/IRReader/IRReader.h" +#include "llvm/LTO/LTO.h" #include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/Caching.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SHA256.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/SplitModule.h" +#include #include #include #include -#include #include using namespace llvm; @@ -59,6 +71,8 @@ std::optional parseEmitOptions(int Argc, char **Argv) { Options.SourcePath = Value.str(); else if (Arg == "--import-bitcode") Options.ImportBitcode.push_back(Value.str()); + else if (Arg == "--cache-dir") + Options.CacheDirectory = Value.str(); else return std::nullopt; } @@ -174,85 +188,403 @@ emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level, return generateCode(M, Machine, Type, OutputPath, Temporaries); } -// Large programs keep whole-program simplification, including every inlining -// decision, then split into independent partitions whose remaining -// optimization and code generation run concurrently. Each partition is -// reloaded from bitcode in its own context because LLVM contexts are not -// shared across threads. The split is a deterministic function of the module -// and partition count, so output never depends on host parallelism. +// Bytes of each partition's module, as LLVM assembly or bitcode, plus the +// names of definitions that were externally visible before partitioning. +struct PartitionPlan { + std::vector Modules; + StringSet<> PublicSymbols; +}; + +// Fallback when the input is outside the text partitioner's grammar: parse +// the whole module once and split it with LLVM, which makes every local +// definition hidden so partitions can reference each other. +std::optional splitParsedModule(const EmitOptions &Options, + PartitionPlan &Plan) { + SMDiagnostic ParseDiagnostic; + LLVMContext Context; + std::unique_ptr Mod = + parseIRFile(Options.Input, ParseDiagnostic, Context); + if (!Mod) { + std::string Detail; + raw_string_ostream Stream(Detail); + ParseDiagnostic.print("scriptc-llvm-codegen", Stream); + return EmitFailure{"invalid_ir", Stream.str()}; + } + if (std::optional Failure = verify(*Mod, "verification_failed")) + return Failure; + for (const GlobalValue &GV : Mod->global_values()) + if (!GV.isDeclaration() && !GV.hasLocalLinkage()) + Plan.PublicSymbols.insert(GV.getName()); + SplitModule(*Mod, static_cast(Options.Outputs.size()), + [&](std::unique_ptr Partition) { + Plan.Modules.emplace_back(); + raw_string_ostream Stream(Plan.Modules.back()); + WriteBitcodeToFile(*Partition, Stream); + }); + if (Plan.Modules.size() != Options.Outputs.size()) + return EmitFailure{"partition_failed", "LLVM produced an unexpected number " + "of program partitions"}; + return std::nullopt; +} + +// Runs Work(I) for every partition on a pool sized to the host. Outputs never +// depend on the pool size or scheduling order. +void forEachPartition(size_t Count, function_ref Work) { + DefaultThreadPool Pool(hardware_concurrency()); + for (size_t I = 0; I < Count; ++I) + Pool.async([&Work, I] { Work(I); }); + Pool.wait(); +} + +class SharedFailure { +public: + void set(EmitFailure Failure) { + std::lock_guard Lock(Mutex); + if (!First) + First = std::move(Failure); + } + std::optional take() { return std::move(First); } + +private: + std::mutex Mutex; + std::optional First; +}; + +// Parses one partition into its own context with the requested target. +Expected> loadPartition(const EmitOptions &Options, + StringRef Bytes, size_t Index, + LLVMContext &Context, + TargetMachine &Machine) { + SMDiagnostic Diagnostic; + std::string Name = "partition-" + std::to_string(Index); + std::unique_ptr Mod = + parseIR(MemoryBufferRef(Bytes, Name), Diagnostic, Context); + if (!Mod) { + std::string Detail; + raw_string_ostream Stream(Detail); + Diagnostic.print("scriptc-llvm-codegen", Stream); + return createStringError(inconvertibleErrorCode(), Stream.str()); + } + if (!Options.SourcePath.empty()) + Mod->setSourceFileName(Options.SourcePath); + Mod->setTargetTriple(Triple(Options.Target)); + Mod->setDataLayout(Machine.createDataLayout()); + if (std::optional Failure = verify(*Mod, "verification_failed")) + return createStringError(inconvertibleErrorCode(), Failure->Message); + return std::move(Mod); +} + +// A raw_ostream that only hashes what is written to it. +class HashingStream : public raw_ostream { +public: + explicit HashingStream(SHA256 &Hash) : Hash(Hash) { SetUnbuffered(); } + +private: + void write_impl(const char *Data, size_t Size) override { + Hash.update(StringRef(Data, Size)); + Position += Size; + } + uint64_t current_pos() const override { return Position; } + SHA256 &Hash; + uint64_t Position = 0; +}; + +// Collects one native object per partition in memory. +class ObjectStream : public CachedFileStream { +public: + explicit ObjectStream(SmallVector &Buffer) + : CachedFileStream(std::make_unique(Buffer)) {} +}; + +// Unoptimized partitions are independent: each compiles, or comes from the +// cache keyed by its exact source, on its own thread. std::optional -emitPartitions(std::unique_ptr Mod, const EmitOptions &Options, - OptimizationLevel Level, CodeGenFileType Type, - std::vector> &Temporaries) { - { +compileUnoptimizedPartitions(const EmitOptions &Options, + const PartitionPlan &Plan, StringRef Salt, + const ArtifactCache &Cache, + std::vector> &Objects) { + SharedFailure Failures; + forEachPartition(Plan.Modules.size(), [&](size_t I) { + std::string Key = hashParts({Salt, "object", Plan.Modules[I]}); + if (std::unique_ptr Hit = Cache.lookup(Key, "o")) { + Objects[I].assign(Hit->getBufferStart(), Hit->getBufferEnd()); + return; + } std::string LookupError; std::unique_ptr Machine = createTargetMachine(Options.Target, Options.OptLevel, LookupError); if (!Machine) - return EmitFailure{"target_machine_failed", LookupError}; - Pipeline P(*Machine, Level); - P.PB.buildModuleSimplificationPipeline(Level, ThinOrFullLTOPhase::None) - .run(*Mod, P.MAM); - } - size_t Count = Options.Outputs.size(); - std::vector> Partitions; - SplitModule(*Mod, static_cast(Count), - [&](std::unique_ptr Partition) { - Partitions.emplace_back(); - raw_svector_ostream Stream(Partitions.back()); - WriteBitcodeToFile(*Partition, Stream); - }); - Mod.reset(); - if (Partitions.size() != Count) - return EmitFailure{"partition_failed", "LLVM produced an unexpected number " - "of program partitions"}; + return Failures.set({"target_machine_failed", LookupError}); + LLVMContext Context; + Expected> Mod = + loadPartition(Options, Plan.Modules[I], I, Context, *Machine); + if (!Mod) + return Failures.set({"partition_failed", toString(Mod.takeError())}); + // The O0 pipeline still lowers coroutines and always-inline calls. + Pipeline P(*Machine, OptimizationLevel::O0); + P.PB.buildPerModuleDefaultPipeline(OptimizationLevel::O0) + .run(**Mod, P.MAM); + raw_svector_ostream Output(Objects[I]); + legacy::PassManager CodeGeneration; + if (Machine->addPassesToEmitFile(CodeGeneration, Output, nullptr, + CodeGenFileType::ObjectFile)) + return Failures.set({"emission_not_supported", + "target does not support object emission"}); + CodeGeneration.run(**Mod); + Cache.store(Key, "o", StringRef(Objects[I].data(), Objects[I].size())); + }); + return Failures.take(); +} - std::vector> Failures(Count); - std::vector>> Written(Count); - std::vector Workers; - for (size_t I = 0; I < Count; ++I) { - Workers.emplace_back([&, I] { - LLVMContext Context; - Expected> Partition = parseBitcodeFile( - MemoryBufferRef(StringRef(Partitions[I].data(), Partitions[I].size()), - Options.Outputs[I]), - Context); - if (!Partition) { - Failures[I] = - EmitFailure{"partition_failed", toString(Partition.takeError())}; - return; - } - std::string LookupError; - std::unique_ptr Machine = - createTargetMachine(Options.Target, Options.OptLevel, LookupError); - if (!Machine) { - Failures[I] = EmitFailure{"target_machine_failed", LookupError}; - return; - } +// Optimized partitions follow the ThinLTO model. Each partition is simplified +// on its own thread and summarized; a thin link over the summaries decides +// which small functions every partition imports from the others for +// inlining and which promoted symbols return to internal linkage; then each +// partition is optimized and compiled on its own thread. Simplified bitcode +// is cached by the partition's exact source, and objects by the exact module +// after importing, so an edit recompiles only the partitions it can affect. +std::optional +compileOptimizedPartitions(const EmitOptions &Options, + const PartitionPlan &Plan, OptimizationLevel Level, + StringRef Salt, const ArtifactCache &Cache, + std::vector> &Objects) { + size_t Count = Plan.Modules.size(); + std::vector> Summaries(Count); + SharedFailure Failures; + forEachPartition(Count, [&](size_t I) { + std::string Key = hashParts({Salt, "summary", Plan.Modules[I]}); + if ((Summaries[I] = Cache.lookup(Key, "bc"))) + return; + std::string LookupError; + std::unique_ptr Machine = + createTargetMachine(Options.Target, Options.OptLevel, LookupError); + if (!Machine) + return Failures.set({"target_machine_failed", LookupError}); + LLVMContext Context; + Expected> Loaded = + loadPartition(Options, Plan.Modules[I], I, Context, *Machine); + if (!Loaded) + return Failures.set({"partition_failed", toString(Loaded.takeError())}); + Module &Mod = **Loaded; + if (!Options.ImportBitcode.empty()) { + // See emit(): the attribute is inert without a sanitizer pass and + // would block inlining of the imported runtime bodies. + for (Function &F : Mod) + F.removeFnAttr(Attribute::SanitizeAddress); + if (std::optional Error = + importRuntimeBitcode(Mod, Options.ImportBitcode)) + return Failures.set({"runtime_import_failed", *Error}); + if (std::optional Failure = + verify(Mod, "runtime_import_verification_failed")) + return Failures.set(std::move(*Failure)); + } + SmallVector Bitcode; + { + raw_svector_ostream Stream(Bitcode); Pipeline P(*Machine, Level); - P.PB.buildModuleOptimizationPipeline(Level, ThinOrFullLTOPhase::None) - .run(**Partition, P.MAM); - Failures[I] = - verify(**Partition, "post_optimization_verification_failed"); - if (!Failures[I]) - Failures[I] = generateCode(**Partition, *Machine, Type, - Options.Outputs[I], Written[I]); - }); - } - for (std::thread &Worker : Workers) - Worker.join(); + ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level); + MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false, + /*EmitSummaryIndex=*/true, + /*EmitModuleHash=*/true)); + MPM.run(Mod, P.MAM); + } + StringRef Contents(Bitcode.data(), Bitcode.size()); + Cache.store(Key, "bc", Contents); + Summaries[I] = MemoryBuffer::getMemBufferCopy(Contents); + }); + if (std::optional Failure = Failures.take()) + return Failure; + + lto::Config Config; + Triple TargetTriple(Options.Target); + Config.CPU = "generic"; + Config.Options = targetOptions(TargetTriple); + Config.RelocModel = Reloc::PIC_; + Config.CodeModel = CodeModel::Small; + Config.CGOptLevel = codeGenLevel(Options.OptLevel); + Config.OptLevel = Level == OptimizationLevel::O1 ? 1 + : Level == OptimizationLevel::O3 ? 3 + : 2; + Config.PTO = tuningFor(Level); + Config.CGFileType = CodeGenFileType::ObjectFile; + Config.DefaultTriple = Options.Target; + Config.DiagHandler = [&](const DiagnosticInfo &Info) { + if (Info.getSeverity() != DS_Error) + return; + std::string Message; + raw_string_ostream Stream(Message); + DiagnosticPrinterRawOStream Printer(Stream); + Info.print(Printer); + Failures.set({"partition_failed", Stream.str()}); + }; + std::vector ObjectKeys(Count); + Config.PostImportModuleHook = [&](unsigned Task, const Module &Mod) { + size_t I = Task - 1; + if (I >= Count) + return true; + SHA256 Hash; + { + HashingStream Stream(Hash); + WriteBitcodeToFile(Mod, Stream); + } + std::string Key = + hashParts({Salt, "object", toHex(Hash.final(), /*LowerCase=*/true)}); + if (std::unique_ptr Hit = Cache.lookup(Key, "o")) { + Objects[I].assign(Hit->getBufferStart(), Hit->getBufferEnd()); + return false; + } + ObjectKeys[I] = std::move(Key); + return true; + }; + + // The link refers to module identifiers for its whole lifetime. + std::vector Names(Count); + for (size_t I = 0; I < Count; ++I) + Names[I] = "partition-" + std::to_string(I); + lto::LTO Link(std::move(Config), + lto::createInProcessThinBackend(hardware_concurrency())); for (size_t I = 0; I < Count; ++I) { - if (Written[I].empty()) - Temporaries.emplace_back(); - else - Temporaries.push_back(Written[I].front()); + Expected> Input = lto::InputFile::create( + MemoryBufferRef(Summaries[I]->getBuffer(), Names[I])); + if (!Input) + return EmitFailure{"partition_failed", toString(Input.takeError())}; + std::vector Resolutions; + for (const lto::InputFile::Symbol &Symbol : (*Input)->symbols()) { + lto::SymbolResolution Resolution; + if (!Symbol.isUndefined()) { + // Partitions define disjoint symbols. Only definitions that were + // external before partitioning are visible to the runtime objects + // linked beside them; the rest may become internal again. + bool Public = Plan.PublicSymbols.contains(Symbol.getIRName()); + Resolution.Prevailing = true; + Resolution.VisibleToRegularObj = Public; + Resolution.FinalDefinitionInLinkageUnit = !Public; + } + Resolutions.push_back(Resolution); + } + if (Error Err = Link.add(std::move(*Input), Resolutions)) + return EmitFailure{"partition_failed", toString(std::move(Err))}; } - for (std::optional &Failure : Failures) - if (Failure) - return Failure; + Error Err = Link.run([&](unsigned Task, const Twine &) + -> Expected> { + size_t I = Task - 1; + if (I >= Count) + return createStringError(inconvertibleErrorCode(), + "unexpected regular LTO output"); + Objects[I].clear(); + return std::make_unique(Objects[I]); + }); + if (Err) + return EmitFailure{"partition_failed", toString(std::move(Err))}; + if (std::optional Failure = Failures.take()) + return Failure; + for (size_t I = 0; I < Count; ++I) + if (!ObjectKeys[I].empty()) + Cache.store(ObjectKeys[I], "o", + StringRef(Objects[I].data(), Objects[I].size())); return std::nullopt; } +std::optional +compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan, + std::vector> &Objects) { + // Everything besides a partition's own bytes that determines its + // artifacts. The caller's cache directory already identifies the exact + // helper build; source content and partition count arrive per module. + std::string Salt = hashParts({"scriptc-partition-v1", LLVM_VERSION_STRING, + Options.Target, Options.OptLevel, + Options.RelocationModel, Options.SourcePath}); + for (const std::string &Path : Options.ImportBitcode) { + ErrorOr> Bitcode = + MemoryBuffer::getFile(Path, /*IsText=*/false, + /*RequiresNullTerminator=*/false); + if (!Bitcode) + return EmitFailure{"runtime_import_failed", + Path + ": " + Bitcode.getError().message()}; + Salt = hashParts({Salt, (*Bitcode)->getBuffer()}); + } + ArtifactCache Cache(Options.CacheDirectory); + Objects.assign(Plan.Modules.size(), {}); + if (Options.OptLevel == "0") + return compileUnoptimizedPartitions(Options, Plan, Salt, Cache, Objects); + return compileOptimizedPartitions(Options, Plan, + optimizationLevel(Options.OptLevel), Salt, + Cache, Objects); +} + +// Publishes in-memory partition objects through private sibling files. +std::optional +writePartitionObjects(const EmitOptions &Options, + const std::vector> &Objects, + std::vector> &Temporaries) { + for (size_t I = 0; I < Objects.size(); ++I) { + SmallString<256> TemporaryPath(Options.Outputs[I]); + TemporaryPath.append(".tmp-%%%%%%"); + int TemporaryFd = -1; + if (std::error_code EC = sys::fs::createUniqueFile( + TemporaryPath, TemporaryFd, TemporaryPath)) + return EmitFailure{"output_open_failed", EC.message()}; + Temporaries.push_back(TemporaryPath); + raw_fd_ostream Output(TemporaryFd, true); + Output.write(Objects[I].data(), Objects[I].size()); + Output.close(); + if (Output.has_error()) { + std::string Message = Output.error().message(); + Output.clear_error(); + return EmitFailure{"output_write_failed", Message}; + } + } + return std::nullopt; +} + +// Several outputs: divide the program into that many partitions, then +// compile them concurrently. The division is a deterministic function of the +// module and the partition count, so output never depends on host +// parallelism. Splitting the assembly text lets partitions parse +// concurrently; any input outside the text partitioner's grammar, or a +// partition it produced that does not load, falls back to LLVM's splitter. +int emitPartitions(const EmitOptions &Options) { + // Every partition may come from the cache, so the optimizer's target + // lookups cannot rely on an earlier target machine having registered it. + initializeTargets(); + std::vector> Objects; + std::optional Failure; + bool Compiled = false; + { + ErrorOr> Input = + MemoryBuffer::getFile(Options.Input, /*IsText=*/false, + /*RequiresNullTerminator=*/false); + if (Input) { + std::optional Text = partitionProgramText( + (*Input)->getBuffer(), static_cast(Options.Outputs.size())); + if (Text) { + Input->reset(); + PartitionPlan Plan{std::move(Text->Sources), + std::move(Text->PublicSymbols)}; + Compiled = !compilePartitions(Options, Plan, Objects); + } + } + } + if (!Compiled) { + PartitionPlan Plan; + Failure = splitParsedModule(Options, Plan); + if (!Failure) + Failure = compilePartitions(Options, Plan, Objects); + } + std::vector> Temporaries; + if (!Failure) + Failure = writePartitionObjects(Options, Objects, Temporaries); + for (size_t I = 0; !Failure && I < Temporaries.size(); ++I) + Failure = publish(Temporaries[I], Options.Outputs[I]); + if (Failure) { + for (const SmallString<256> &Temporary : Temporaries) + sys::fs::remove(Temporary); + return reportError(Failure->Code, Failure->Message, + Options.DiagnosticFormat); + } + return 0; +} + } // namespace int emit(const EmitOptions &Options) { @@ -276,14 +608,26 @@ int emit(const EmitOptions &Options) { " outputs are supported", Options.DiagnosticFormat); if (Options.Outputs.size() > 1 && - (Options.FileType != "obj" || Options.OptLevel == "0")) + (Options.FileType != "obj" || Options.OptLevel == "s" || + Options.OptLevel == "z")) return reportError("invalid_partitions", - "several outputs require optimized object emission", + "several outputs require object emission at opt-level " + "0, 1, 2, or 3", + Options.DiagnosticFormat); + if (!Options.CacheDirectory.empty() && Options.Outputs.size() == 1) + return reportError("invalid_cache_directory", + "a cache directory requires several outputs", Options.DiagnosticFormat); if (Options.RelocationModel != "pic") return reportError("invalid_relocation_model", "only the pic relocation model is supported", Options.DiagnosticFormat); + if (!Options.ImportBitcode.empty() && Options.OptLevel == "0") + return reportError("invalid_import", + "runtime bitcode import requires an optimized build", + Options.DiagnosticFormat); + if (Options.Outputs.size() > 1) + return emitPartitions(Options); SMDiagnostic ParseDiagnostic; LLVMContext Context; @@ -317,10 +661,6 @@ int emit(const EmitOptions &Options) { // Runtime bitcode import is the `speed` posture's opt-in; without it the // module reaches the pipeline exactly as emitted. if (!Options.ImportBitcode.empty()) { - if (Options.OptLevel == "0") - return reportError("invalid_import", - "runtime bitcode import requires an optimized build", - Options.DiagnosticFormat); // The emitter marks every function sanitize_address so the sanitized // lane's clang link can instrument it. This helper never runs a // sanitizer pass, so the attribute is inert here except that it @@ -345,13 +685,10 @@ int emit(const EmitOptions &Options) { ? CodeGenFileType::ObjectFile : CodeGenFileType::AssemblyFile; std::vector> Temporaries; - std::optional Failure = - Options.Outputs.size() == 1 - ? emitModule(*Mod, *Machine, Level, Type, Options.Outputs[0], - Temporaries) - : emitPartitions(std::move(Mod), Options, Level, Type, Temporaries); - for (size_t I = 0; !Failure && I < Temporaries.size(); ++I) - Failure = publish(Temporaries[I], Options.Outputs[I]); + std::optional Failure = emitModule( + *Mod, *Machine, Level, Type, Options.Outputs[0], Temporaries); + if (!Failure) + Failure = publish(Temporaries[0], Options.Outputs[0]); if (Failure) { for (const SmallString<256> &Temporary : Temporaries) sys::fs::remove(Temporary); diff --git a/native/llvm-codegen/src/emit.h b/native/llvm-codegen/src/emit.h index e80afcdaf..444716a86 100644 --- a/native/llvm-codegen/src/emit.h +++ b/native/llvm-codegen/src/emit.h @@ -18,6 +18,8 @@ struct EmitOptions { std::string SourcePath; // Runtime bitcode whose small functions are imported available_externally. std::vector ImportBitcode; + // Private directory for reusable partition artifacts; empty disables reuse. + std::string CacheDirectory; }; std::optional parseEmitOptions(int Argc, char **Argv); diff --git a/native/llvm-codegen/src/main.cpp b/native/llvm-codegen/src/main.cpp index 2f831449e..21881659d 100644 --- a/native/llvm-codegen/src/main.cpp +++ b/native/llvm-codegen/src/main.cpp @@ -84,7 +84,7 @@ int main(int Argc, char **Argv) { "emit requires --input --output and accepts " "--filetype --target --opt-level <0|1|2|3|s|z> " "--relocation-model pic --diagnostic-format json " - "--source-path "); + "--source-path --import-bitcode --cache-dir "); return scriptc::emit(*Options); } return scriptc::reportError("usage", diff --git a/native/llvm-codegen/src/partition.cpp b/native/llvm-codegen/src/partition.cpp new file mode 100644 index 000000000..066ba614c --- /dev/null +++ b/native/llvm-codegen/src/partition.cpp @@ -0,0 +1,583 @@ +#include "partition.h" + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/Threading.h" + +#include +#include + +using namespace llvm; + +namespace scriptc { + +namespace { + +constexpr unsigned NoPartition = ~0u; + +enum class SymbolKind : uint8_t { Function, Variable, Declaration }; + +struct Symbol { + SymbolKind Kind = SymbolKind::Declaration; + bool Local = false; + StringRef Name; + // Definition header line, or the complete declaration line. + StringRef Header; + // Function lines after the header, through the closing brace. + StringRef Body; + // Declaration other partitions use for this definition. + std::string Declaration; + // Local definitions are rewritten with hidden visibility. + std::string PromotedHeader; + std::vector Refs; + std::vector Metadata; + unsigned Partition = NoPartition; +}; + +struct MetadataNode { + StringRef Line; + std::vector Refs; + std::vector Symbols; +}; + +bool isNameChar(char C) { + return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z') || + (C >= '0' && C <= '9') || C == '-' || C == '$' || C == '.' || + C == '_'; +} + +bool isDigit(char C) { return C >= '0' && C <= '9'; } + +// Length of the unquoted symbol name starting at Text[Start], or 0. +size_t nameLength(StringRef Text, size_t Start) { + size_t End = Start; + while (End < Text.size() && isNameChar(Text[End])) + ++End; + return End - Start; +} + +uint32_t stableHash(StringRef Name) { + uint32_t Hash = 0x811c9dc5u; + for (char C : Name) { + Hash ^= static_cast(C); + Hash *= 0x01000193u; + } + return Hash; +} + +bool startsWithLinkage(StringRef Rest) { + for (StringRef Linkage : + {"linkonce", "weak", "common", "appending", "extern_weak", + "available_externally", "external"}) + if (Rest.starts_with(Linkage)) + return true; + return false; +} + +bool hasVisibilityToken(StringRef Prefix) { + SmallVector Tokens; + Prefix.split(Tokens, ' ', -1, false); + for (StringRef Token : Tokens) + if (Token == "dso_local" || Token == "dso_preemptable" || + Token == "hidden" || Token == "protected" || Token == "default" || + Token == "dllimport" || Token == "dllexport") + return true; + return false; +} + +// Position of the first unquoted occurrence of Wanted, or npos. +size_t findUnquoted(StringRef Text, char Wanted) { + bool Quoted = false; + for (size_t I = 0; I < Text.size(); ++I) { + if (Text[I] == '"') + Quoted = !Quoted; + else if (!Quoted && Text[I] == Wanted) + return I; + } + return StringRef::npos; +} + +bool parseFunctionHeader(StringRef Line, Symbol &S) { + StringRef Rest = Line.drop_front(sizeof("define ") - 1); + if (Rest.consume_front("internal ") || Rest.consume_front("private ")) + S.Local = true; + else if (startsWithLinkage(Rest)) + return false; + size_t At = Rest.find('@'); + if (At == StringRef::npos) + return false; + StringRef Prefix = Rest.take_front(At); + if (S.Local && hasVisibilityToken(Prefix)) + return false; + size_t Length = nameLength(Rest, At + 1); + size_t Open = At + 1 + Length; + if (Length == 0 || Open >= Rest.size() || Rest[Open] != '(') + return false; + S.Name = Rest.substr(At + 1, Length); + size_t Depth = 0, Close = StringRef::npos; + bool Quoted = false; + for (size_t I = Open; I < Rest.size(); ++I) { + char C = Rest[I]; + if (C == '"') + Quoted = !Quoted; + if (Quoted) + continue; + if (C == '(') + ++Depth; + else if (C == ')' && --Depth == 0) { + Close = I; + break; + } + } + if (Close == StringRef::npos) + return false; + StringRef Tail = Rest.drop_front(Close + 1); + size_t Brace = findUnquoted(Tail, '{'); + if (Brace == StringRef::npos) + return false; + StringRef After = Tail.drop_front(Brace + 1).ltrim(); + if (!After.empty() && After.front() != ';') + return false; + S.Declaration = "declare "; + if (S.Local) + S.Declaration += "hidden "; + S.Declaration += Prefix; + S.Declaration += '@'; + S.Declaration += S.Name; + S.Declaration += Rest.slice(Open, Close + 1); + // Attribute groups and address significance carry over. Personality, + // sections, comdats and debug attachments belong only to the definition. + SmallVector Tokens; + Tail.take_front(Brace).split(Tokens, ' ', -1, false); + for (StringRef Token : Tokens) { + bool Group = Token.size() > 1 && Token.front() == '#' && + all_of(Token.drop_front(), isDigit); + if (Group || Token == "unnamed_addr" || Token == "local_unnamed_addr") { + S.Declaration += ' '; + S.Declaration += Token; + } + } + if (S.Local) + S.PromotedHeader = (Twine("define hidden ") + Rest).str(); + return true; +} + +// Parses `@name = ...`. Returns false for unsupported forms. +bool parseGlobal(StringRef Line, Symbol &S) { + size_t Length = nameLength(Line, 1); + if (Length == 0) + return false; + S.Name = Line.substr(1, Length); + StringRef Rest = Line.drop_front(1 + Length); + if (!Rest.consume_front(" = ")) + return false; + if (Rest.starts_with("external ") || Rest.starts_with("extern_weak ")) { + S.Kind = SymbolKind::Declaration; + return true; + } + S.Kind = SymbolKind::Variable; + if (Rest.consume_front("internal ") || Rest.consume_front("private ")) + S.Local = true; + else if (startsWithLinkage(Rest)) + return false; + StringRef Cursor = Rest; + std::string Qualifiers; + StringRef Keyword; + while (true) { + if (Cursor.consume_front("global ")) { + Keyword = "global"; + break; + } + if (Cursor.consume_front("constant ")) { + Keyword = "constant"; + break; + } + size_t Space = Cursor.find(' '); + if (Space == StringRef::npos) + return false; + StringRef Token = Cursor.take_front(Space); + if (Token == "dso_local" || Token == "hidden" || Token == "protected" || + Token == "default") { + if (S.Local) + return false; + Qualifiers += Token; + Qualifiers += ' '; + } else if (Token.starts_with("thread_local") || + Token.starts_with("addrspace(")) { + Qualifiers += Token; + Qualifiers += ' '; + } else if (Token != "unnamed_addr" && Token != "local_unnamed_addr" && + Token != "externally_initialized") { + return false; + } + Cursor = Cursor.drop_front(Space + 1); + } + // The value type ends at the first space outside brackets. + int Depth = 0; + size_t TypeEnd = StringRef::npos; + for (size_t I = 0; I < Cursor.size(); ++I) { + char C = Cursor[I]; + if (C == '{' || C == '[' || C == '<' || C == '(') + ++Depth; + else if (C == '}' || C == ']' || C == '>' || C == ')') + --Depth; + else if (C == ' ' && Depth == 0) { + TypeEnd = I; + break; + } + } + if (TypeEnd == StringRef::npos) + return false; + S.Declaration = (Twine("@") + S.Name + " = external " + (S.Local ? "hidden " : "") + + Qualifiers + Keyword + " " + Cursor.take_front(TypeEnd)) + .str(); + if (S.Local) + S.PromotedHeader = (Twine("@") + S.Name + " = hidden " + Rest).str(); + return true; +} + +// Collects the symbols (sorted) and metadata nodes (in order of first +// appearance) that Text references outside quoted strings. +void scanReferences(StringRef Text, const StringMap &Index, + std::vector &Refs, + std::vector &Metadata) { + DenseSet Seen(Metadata.begin(), Metadata.end()); + bool Quoted = false; + for (size_t I = 0; I < Text.size(); ++I) { + char C = Text[I]; + if (C == '\n') { + Quoted = false; + } else if (C == '"') { + Quoted = !Quoted; + } else if (Quoted) { + continue; + } else if (C == '@') { + size_t Length = nameLength(Text, I + 1); + if (Length != 0) { + auto It = Index.find(Text.substr(I + 1, Length)); + if (It != Index.end()) + Refs.push_back(It->second); + I += Length; + } + } else if (C == '!' && I + 1 < Text.size() && isDigit(Text[I + 1])) { + unsigned Number = 0; + size_t J = I + 1; + for (; J < Text.size() && isDigit(Text[J]); ++J) + Number = Number * 10 + static_cast(Text[J] - '0'); + if (Seen.insert(Number).second) + Metadata.push_back(Number); + I = J - 1; + } + } + llvm::sort(Refs); + Refs.erase(std::unique(Refs.begin(), Refs.end()), Refs.end()); +} + +// Appends Text with every unquoted metadata reference renumbered by Map. +void appendRenumbered(StringRef Text, const DenseMap &Map, + std::string &Out) { + bool Quoted = false; + size_t Copied = 0; + for (size_t I = 0; I < Text.size(); ++I) { + char C = Text[I]; + if (C == '\n') { + Quoted = false; + } else if (C == '"') { + Quoted = !Quoted; + } else if (!Quoted && C == '!' && I + 1 < Text.size() && + isDigit(Text[I + 1])) { + unsigned Number = 0; + size_t J = I + 1; + for (; J < Text.size() && isDigit(Text[J]); ++J) + Number = Number * 10 + static_cast(Text[J] - '0'); + auto It = Map.find(Number); + if (It != Map.end()) { + Out.append(Text.data() + Copied, I + 1 - Copied); + Out += std::to_string(It->second); + Copied = J; + } + I = J - 1; + } + } + Out.append(Text.data() + Copied, Text.size() - Copied); +} + +} // namespace + +std::optional partitionProgramText(StringRef Text, + unsigned Count) { + if (Count < 2) + return std::nullopt; + std::vector Header, Types, Attributes, NamedMetadata; + std::vector CompileUnits; + DenseMap Metadata; + std::vector Symbols; + StringMap Index; + + auto Add = [&](Symbol S) { + if (!Index.try_emplace(S.Name, Symbols.size()).second) + return false; + Symbols.push_back(std::move(S)); + return true; + }; + + size_t Pos = 0; + while (Pos < Text.size()) { + size_t End = Text.find('\n', Pos); + if (End == StringRef::npos) + End = Text.size(); + StringRef Line = Text.slice(Pos, End); + size_t Next = End + 1; + if (Line.empty() || Line.trim().empty()) { + // Blank separator. + } else if (Line.starts_with("define ")) { + Symbol S; + S.Kind = SymbolKind::Function; + S.Header = Line; + if (!parseFunctionHeader(Line, S)) + return std::nullopt; + // The emitter indents every body line, so the first line that is + // exactly "}" closes the function. + size_t Close = End; + while (true) { + Close = Text.find("\n}", Close); + if (Close == StringRef::npos) + return std::nullopt; + if (Close + 2 == Text.size() || Text[Close + 2] == '\n') + break; + Close += 2; + } + S.Body = Text.slice(std::min(End + 1, Text.size()), Close + 2); + Next = Close + 3; + if (!Add(std::move(S))) + return std::nullopt; + } else if (Line.front() == '@') { + Symbol S; + S.Header = Line; + if (!parseGlobal(Line, S) || !Add(std::move(S))) + return std::nullopt; + } else if (Line.starts_with("declare ")) { + Symbol S; + S.Header = Line; + size_t At = Line.find('@'); + size_t Length = At == StringRef::npos ? 0 : nameLength(Line, At + 1); + if (Length == 0) + return std::nullopt; + S.Name = Line.substr(At + 1, Length); + if (!Add(std::move(S))) + return std::nullopt; + } else if (Line.front() == '%') { + if (Line.find(" = type ") == StringRef::npos) + return std::nullopt; + Types.push_back(Line); + } else if (Line.starts_with("attributes #")) { + Attributes.push_back(Line); + } else if (Line.front() == '!') { + if (Line.size() > 1 && isDigit(Line[1])) { + unsigned Number = 0; + size_t I = 1; + for (; I < Line.size() && isDigit(Line[I]); ++I) + Number = Number * 10 + static_cast(Line[I] - '0'); + if (!Line.drop_front(I).starts_with(" = ")) + return std::nullopt; + if (!Metadata.try_emplace(Number, MetadataNode{Line, {}, {}}).second) + return std::nullopt; + } else if (Line.starts_with("!llvm.dbg.cu = !{")) { + // Each partition lists only the compile units its functions use. + std::vector Unused; + scanReferences(Line, Index, Unused, CompileUnits); + } else { + NamedMetadata.push_back(Line); + } + } else if (Line.front() == ';' || Line.starts_with("source_filename ") || + Line.starts_with("target ")) { + Header.push_back(Line); + } else { + return std::nullopt; + } + Pos = Next; + } + + for (Symbol &S : Symbols) { + if (S.Kind == SymbolKind::Declaration) + continue; + scanReferences(S.Header, Index, S.Refs, S.Metadata); + if (!S.Body.empty()) { + std::vector Refs; + scanReferences(S.Body, Index, Refs, S.Metadata); + S.Refs.insert(S.Refs.end(), Refs.begin(), Refs.end()); + llvm::sort(S.Refs); + S.Refs.erase(std::unique(S.Refs.begin(), S.Refs.end()), S.Refs.end()); + } + } + + // An internal function referenced by exactly one other function, and by + // no global, is a whole-program inlining candidate; keep it with that + // function so its partition can inline it without importing. + constexpr unsigned Shared = ~1u; + std::vector Referrer(Symbols.size(), NoPartition); + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind == SymbolKind::Declaration) + continue; + for (unsigned Ref : S.Refs) { + if (Ref == I) + continue; + if (S.Kind == SymbolKind::Variable || Referrer[Ref] != NoPartition) + Referrer[Ref] = Shared; + else + Referrer[Ref] = I; + } + } + for (unsigned I = 0; I < Symbols.size(); ++I) { + if (Symbols[I].Kind != SymbolKind::Function) + continue; + unsigned Root = I; + for (unsigned Steps = 0; Steps < 1024; ++Steps) { + const Symbol &S = Symbols[Root]; + unsigned Parent = Referrer[Root]; + if (!S.Local || Parent == NoPartition || Parent == Shared || + Symbols[Parent].Kind != SymbolKind::Function || Parent == I) + break; + Root = Parent; + } + Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; + } + for (const Symbol &S : Symbols) { + if (S.Kind != SymbolKind::Function) + continue; + for (unsigned Ref : S.Refs) + if (Symbols[Ref].Kind == SymbolKind::Variable && + Symbols[Ref].Partition == NoPartition) + Symbols[Ref].Partition = S.Partition; + } + for (Symbol &S : Symbols) + if (S.Kind == SymbolKind::Variable && S.Partition == NoPartition) + S.Partition = stableHash(S.Name) % Count; + + ProgramPartitions Result; + for (const Symbol &S : Symbols) + if (S.Kind != SymbolKind::Declaration && !S.Local) + Result.PublicSymbols.insert(S.Name); + + // Metadata nodes are numbered in the emitter's global order, so one added + // node would renumber the references in every partition. Each partition + // instead numbers the nodes it uses in order of first use, which keeps an + // unchanged partition's text, and therefore its cached artifacts, intact. + for (auto &Entry : Metadata) { + MetadataNode &Node = Entry.second; + size_t Equals = Node.Line.find(" = "); + scanReferences(Node.Line.drop_front(Equals + 3), Index, Node.Symbols, + Node.Refs); + } + std::vector NamedRoots, NamedSymbols; + for (StringRef Line : NamedMetadata) + scanReferences(Line, Index, NamedSymbols, NamedRoots); + + Result.Sources.resize(Count); + std::vector> Owned(Count); + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind != SymbolKind::Declaration) + Owned[Symbols[I].Partition].push_back(I); + + auto Assemble = [&](unsigned P) { + std::vector Needed(Symbols.size(), 0); + DenseMap Renumber; + std::vector Nodes; + auto Use = [&](ArrayRef Numbers) { + for (unsigned Number : Numbers) + if (Metadata.count(Number) && + Renumber.try_emplace(Number, Nodes.size()).second) + Nodes.push_back(Number); + }; + Use(NamedRoots); + for (unsigned Ref : NamedSymbols) + Needed[Ref] = 1; + size_t Bytes = 0; + for (SymbolKind Kind : {SymbolKind::Variable, SymbolKind::Function}) + for (unsigned I : Owned[P]) + if (Symbols[I].Kind == Kind) { + const Symbol &S = Symbols[I]; + for (unsigned Ref : S.Refs) + Needed[Ref] = 1; + Use(S.Metadata); + Bytes += S.Header.size() + S.Body.size() + 2; + } + for (size_t K = 0; K < Nodes.size(); ++K) { + const MetadataNode &Node = Metadata.find(Nodes[K])->second; + Use(Node.Refs); + for (unsigned Ref : Node.Symbols) + Needed[Ref] = 1; + } + + std::string &Out = Result.Sources[P]; + Out.reserve(Bytes + (Bytes >> 3)); + auto Line = [&](StringRef Text) { + if (Renumber.empty()) + Out += Text; + else + appendRenumbered(Text, Renumber, Out); + Out += '\n'; + }; + for (StringRef Text : Header) + Line(Text); + for (StringRef Text : Types) + Line(Text); + // Declarations follow name order rather than the emitter's first-use + // order, which an edit elsewhere in the program can change. + std::vector Declared; + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Needed[I] && (Symbols[I].Kind == SymbolKind::Declaration || + Symbols[I].Partition != P)) + Declared.push_back(I); + llvm::sort(Declared, [&](unsigned A, unsigned B) { + return Symbols[A].Name < Symbols[B].Name; + }); + for (unsigned I : Declared) { + const Symbol &S = Symbols[I]; + Line(S.Kind == SymbolKind::Declaration ? S.Header + : StringRef(S.Declaration)); + } + for (SymbolKind Kind : {SymbolKind::Variable, SymbolKind::Function}) + for (unsigned I : Owned[P]) { + const Symbol &S = Symbols[I]; + if (S.Kind != Kind) + continue; + Line(S.Local ? StringRef(S.PromotedHeader) : S.Header); + if (!S.Body.empty()) + Line(S.Body); + } + for (StringRef Text : Attributes) + Line(Text); + for (StringRef Text : NamedMetadata) + Line(Text); + std::string Units; + for (unsigned Unit : CompileUnits) { + auto It = Renumber.find(Unit); + if (It == Renumber.end()) + continue; + Units += Units.empty() ? "!" : ", !"; + Units += std::to_string(It->second); + } + if (!Units.empty()) + Out += "!llvm.dbg.cu = !{" + Units + "}\n"; + for (size_t K = 0; K < Nodes.size(); ++K) { + StringRef Text = Metadata.find(Nodes[K])->second.Line; + Out += '!'; + Out += std::to_string(K); + Line(Text.drop_front(Text.find(" = "))); + } + }; + DefaultThreadPool Pool(hardware_concurrency()); + for (unsigned P = 0; P < Count; ++P) + Pool.async([&Assemble, P] { Assemble(P); }); + Pool.wait(); + return Result; +} + +} // namespace scriptc diff --git a/native/llvm-codegen/src/partition.h b/native/llvm-codegen/src/partition.h new file mode 100644 index 000000000..9d48548fb --- /dev/null +++ b/native/llvm-codegen/src/partition.h @@ -0,0 +1,31 @@ +#pragma once + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" + +#include +#include +#include + +namespace scriptc { + +// A program module divided into independently compilable textual modules. +struct ProgramPartitions { + // One complete LLVM assembly module per partition. + std::vector Sources; + // Definitions that were not internal in the original module. Every other + // definition was internal and is promoted to hidden visibility only so the + // partitions can reference each other. + llvm::StringSet<> PublicSymbols; +}; + +// Divides the LLVM assembly scriptc emits into Count modules without parsing +// it, so the partitions can be parsed concurrently. Functions are placed by a +// stable hash of their name; an internal function referenced by exactly one +// other function stays with that function, and a global stays with the first +// function that references it. Returns nullopt for any construct outside the +// emitter's narrow top-level grammar; callers then split a parsed module. +std::optional partitionProgramText(llvm::StringRef Text, + unsigned Count); + +} // namespace scriptc diff --git a/native/llvm-codegen/src/target.cpp b/native/llvm-codegen/src/target.cpp index 97d070c91..fd2866761 100644 --- a/native/llvm-codegen/src/target.cpp +++ b/native/llvm-codegen/src/target.cpp @@ -38,10 +38,7 @@ void LLVMInitializeWebAssemblyAsmParser(); namespace scriptc { -void initializeTargets() { - static bool Initialized = false; - if (Initialized) - return; +static bool registerTargets() { #if defined(SCRIPTC_ENABLE_AARCH64) LLVMInitializeAArch64TargetInfo(); LLVMInitializeAArch64Target(); @@ -66,7 +63,14 @@ void initializeTargets() { // Runtime units may contain inline assembly (runtime-unit emission). LLVMInitializeWebAssemblyAsmParser(); #endif - Initialized = true; + return true; +} + +// Partitions create target machines on several threads; the registry must be +// populated exactly once before any lookup. +void initializeTargets() { + static const bool Registered = registerTargets(); + (void)Registered; } bool supportsTarget(StringRef TripleName) { @@ -104,7 +108,7 @@ bool supportsTarget(StringRef TripleName) { } } -static CodeGenOptLevel codeGenLevel(StringRef Level) { +CodeGenOptLevel codeGenLevel(StringRef Level) { if (Level == "0") return CodeGenOptLevel::None; if (Level == "1") @@ -114,6 +118,14 @@ static CodeGenOptLevel codeGenLevel(StringRef Level) { return CodeGenOptLevel::Default; } +TargetOptions targetOptions(const Triple &TargetTriple) { + TargetOptions Options; + // The Android API 26 runtime pack uses the NDK's emulated TLS ABI. + // Program globals and runtime thread instances must use the same ABI. + Options.EmulatedTLS = TargetTriple.isAndroid(); + return Options; +} + std::unique_ptr createTargetMachine(StringRef TripleName, StringRef OptLevel, std::string &Error, @@ -127,13 +139,9 @@ std::unique_ptr createTargetMachine(StringRef TripleName, const Target *Definition = TargetRegistry::lookupTarget(TargetTriple, Error); if (Definition == nullptr) return nullptr; - TargetOptions Options; - // The Android API 26 runtime pack uses the NDK's emulated TLS ABI. - // Program globals and runtime thread instances must use the same ABI. - Options.EmulatedTLS = TargetTriple.isAndroid(); return std::unique_ptr(Definition->createTargetMachine( - TargetTriple, "generic", "", Options, Reloc::PIC_, CodeModel::Small, - codeGenLevel(OptLevel))); + TargetTriple, "generic", "", targetOptions(TargetTriple), Reloc::PIC_, + CodeModel::Small, codeGenLevel(OptLevel))); } } // namespace scriptc diff --git a/native/llvm-codegen/src/target.h b/native/llvm-codegen/src/target.h index b16407cb4..5049d34f5 100644 --- a/native/llvm-codegen/src/target.h +++ b/native/llvm-codegen/src/target.h @@ -1,13 +1,16 @@ #pragma once #include "llvm/ADT/StringRef.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Target/TargetOptions.h" #include #include namespace llvm { class TargetMachine; -} +class Triple; +} // namespace llvm namespace scriptc { @@ -34,6 +37,10 @@ inline constexpr llvm::StringLiteral AllowedTargets = SCRIPTC_ALLOWED_TARGETS; void initializeTargets(); bool supportsTarget(llvm::StringRef Triple); +// The target options and code generation level every helper target machine +// uses, shared with the partitioned optimizer's configuration. +llvm::TargetOptions targetOptions(const llvm::Triple &TargetTriple); +llvm::CodeGenOptLevel codeGenLevel(llvm::StringRef OptLevel); std::unique_ptr createTargetMachine(llvm::StringRef Triple, llvm::StringRef OptLevel, std::string &Error, bool RequireAllowed = true); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index ba71d6bad..4cff15af0 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -19,6 +19,7 @@ import { InternalCompilerError } from "../../errors.js"; * Dynamic values and islands share the runtime ABI; embedded npm modules use * compressed source tables that the runtime inflates on demand. */ +import { createHash } from "node:crypto"; import { deflateRawSync } from "node:zlib"; import { endsWithJump, @@ -516,6 +517,8 @@ export class LlEmitter { /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ private readonly literals = new Map(); + /** Symbols of interned constant data, named by content (contentSymbol). */ + private readonly contentSymbols = new Set(); /** Interned unit-armed union instances: "unionId:tag" → symbol — one * immortal (rc == SIZE_MAX) static per (union, unit tag). * RC entry points and the collector skip immortals. */ @@ -3383,10 +3386,31 @@ export class LlEmitter { // ── plumbing (frame and scope ownership) ────── + /** A symbol derived from the constant's text rather than its first-use + * index, so an edit that interns one more string leaves every other + * constant's symbol, and the code referencing it, unchanged. This lets + * native builds reuse the compiled partitions an edit does not touch. A + * collision retries with a counter, still in first-use order. */ + private contentSymbol(prefix: string, text: string): string { + for (let attempt = 0; ; attempt++) { + const digest = createHash("sha256") + .update(attempt === 0 ? text : `${attempt}\0${text}`) + .digest("hex"); + const symbol = `${prefix}${parseInt(digest.slice(0, 13), 16)}`; + if (!this.contentSymbols.has(symbol)) { + this.contentSymbols.add(symbol); + return symbol; + } + } + } + internLiteral(text: string): string { let lit = this.literals.get(text); if (!lit) { - lit = { sym: `sc_lit_${this.literals.size}`, len: Buffer.byteLength(text, "utf8") }; + lit = { + sym: this.contentSymbol("sc_lit_", text), + len: Buffer.byteLength(text, "utf8"), + }; this.literals.set(text, lit); } const name = `@${lit.sym}`; @@ -3399,7 +3423,7 @@ export class LlEmitter { cstr(text: string): string { let c = this.cstrs.get(text); if (!c) { - c = { sym: `sc_cs_${this.cstrs.size}`, len: Buffer.byteLength(text, "utf8") }; + c = { sym: this.contentSymbol("sc_cs_", text), len: Buffer.byteLength(text, "utf8") }; this.cstrs.set(text, c); } return `@${c.sym}`; diff --git a/packages/compiler/src/backend/native-codegen-core.ts b/packages/compiler/src/backend/native-codegen-core.ts index 252541641..e9b11ff7e 100644 --- a/packages/compiler/src/backend/native-codegen-core.ts +++ b/packages/compiler/src/backend/native-codegen-core.ts @@ -1,5 +1,4 @@ /** Native codegen helper protocol shared by compiler hosts. */ -import type { NativeOptimization } from "./optimization.js"; import type { NativeHelperSpec, NativeTargetSpec } from "./targets.js"; export const NATIVE_CODEGEN_PROTOCOL_VERSION = "1"; @@ -8,20 +7,20 @@ export const NATIVE_CODEGEN_LLVM_VERSION = "22.1.8"; export type NativeCodegenOutputKind = "asm" | "obj"; /** LLVM text per partition, and the most partitions one executable uses. */ -const PROGRAM_PARTITION_BYTES = 1024 * 1024; -const MAX_PROGRAM_PARTITIONS = 8; +const PROGRAM_PARTITION_BYTES = 4 * 1024 * 1024; +const MAX_PROGRAM_PARTITIONS = 64; -/** Optimized executable objects this large keep whole-program simplification - * and inlining, then optimize and generate code in concurrent partitions. The - * count depends only on the module, so artifacts never vary with the host. */ -export function nativeProgramPartitions( - target: NativeTargetSpec, - optimization: NativeOptimization, - llvmBytes: number, -): number { - if (optimization === "dev" || target.platform === "wasi") return 1; - const count = Math.floor(llvmBytes / PROGRAM_PARTITION_BYTES); - return Math.max(1, Math.min(MAX_PROGRAM_PARTITIONS, count)); +/** Executable objects this large are divided into partitions that the helper + * simplifies, optimizes and compiles concurrently, importing small functions + * across partitions for inlining, and reuses individually across builds. The + * count is a power of two that depends only on the module, so artifacts never + * vary with the host and ordinary edits keep every function in its + * partition. */ +export function nativeProgramPartitions(target: NativeTargetSpec, llvmBytes: number): number { + if (target.platform === "wasi") return 1; + const required = Math.ceil(llvmBytes / PROGRAM_PARTITION_BYTES); + if (required < 2) return 1; + return Math.min(MAX_PROGRAM_PARTITIONS, 2 ** Math.ceil(Math.log2(required))); } /** Output paths for each partition; the first is the requested path. */ diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index 9a13434f1..ab362dbe1 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; import { emitNativeArtifact, + writeLlvmSource, NativeCodegenError, validateNativeCodegenVersion, } from "./native-codegen.js"; @@ -345,13 +346,16 @@ test("sanitized native artifacts fail before helper resolution", async () => { ).rejects.toMatchObject({ diagnosticCode: "SC3002", detailCode: "sanitize_unsupported" }); }); -test("program partitions depend only on optimized module size and target", () => { +test("program partitions depend only on module size and target", () => { const megabyte = 1024 * 1024; - expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, "release", megabyte - 1)).toBe(1); - expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, "release", 3.5 * megabyte)).toBe(3); - expect(nativeProgramPartitions(MACOS_ARM64_TARGET, "release", 100 * megabyte)).toBe(8); - expect(nativeProgramPartitions(MACOS_ARM64_TARGET, "dev", 100 * megabyte)).toBe(1); - expect(nativeProgramPartitions(WASM32_WASI_TARGET, "release", 100 * megabyte)).toBe(1); + expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, 4 * megabyte)).toBe(1); + expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, 4 * megabyte + 1)).toBe(2); + // Powers of two keep every function in its partition across small edits. + expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, 13 * megabyte)).toBe(4); + expect(nativeProgramPartitions(LINUX_X64_GNU_TARGET, 16 * megabyte)).toBe(4); + expect(nativeProgramPartitions(MACOS_ARM64_TARGET, 100 * megabyte)).toBe(32); + expect(nativeProgramPartitions(MACOS_ARM64_TARGET, 1024 * megabyte)).toBe(64); + expect(nativeProgramPartitions(WASM32_WASI_TARGET, 100 * megabyte)).toBe(1); expect(nativePartitionPaths("/build.d/program.o", 3)).toEqual([ "/build.d/program.o", "/build.d/program.part1.o", @@ -375,7 +379,12 @@ test("partitioned objects are emitted, cached and restored together", async () = expect(await readFile(path, "utf8")).toBe("define i32 @answer() { ret i32 42 }\n"); const calls = (await readFile(pkg.log, "utf8")).trim().split("\n"); expect(calls).toHaveLength(1); - expect(calls[0]!.split(" ").filter((arg) => arg === "--output")).toHaveLength(3); + const args = calls[0]!.split(" "); + expect(args.filter((arg) => arg === "--output")).toHaveLength(3); + // Partition artifacts are reused from a private directory per helper build. + const partitionCache = args[args.indexOf("--cache-dir") + 1]!; + expect(partitionCache).toMatch(/[/\\]cache[/\\]native-partitions-v1[/\\][0-9a-f]{32}$/); + expect((await stat(partitionCache)).mode & 0o777).toBe(0o700); const second = join(pkg.root, "second", "program.o"); const restored = await emitNativeArtifact({ @@ -388,5 +397,27 @@ test("partitioned objects are emitted, cached and restored together", async () = // A single object for the same module is a distinct artifact. await emitNativeArtifact(request(pkg.root, pkg.packageJson, join(pkg.root, "whole.o"))); - expect((await readFile(pkg.log, "utf8")).trim().split("\n")).toHaveLength(2); + const all = (await readFile(pkg.log, "utf8")).trim().split("\n"); + expect(all).toHaveLength(2); + expect(all[1]).not.toContain("--cache-dir"); +}); + +test("LLVM text parts are written as one file and existing inputs are read in place", async () => { + const pkg = await fakePackage(); + const parts = Array.from({ length: 5000 }, (_, index) => `; part ${index} \u00e9\n`); + const written = join(pkg.root, "parts.ll"); + await writeLlvmSource(written, parts); + expect(await readFile(written, "utf8")).toBe(parts.join("")); + + const output = join(pkg.root, "in-place", "program.o"); + await emitNativeArtifact({ + ...request(pkg.root, pkg.packageJson, output), + llvm: parts, + inputPath: written, + cacheRoot: null, + }); + const call = (await readFile(pkg.log, "utf8")).trim().split(" "); + expect(call[call.indexOf("--input") + 1]).toBe(written); + expect(await readFile(output, "utf8")).toBe(parts.join("")); + expect(await readFile(written, "utf8")).toBe(parts.join("")); }); diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 4e7ede5d1..edd0461ee 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -2,7 +2,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { constants } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { access, chmod, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -85,6 +85,43 @@ export interface NativeCodegenOptions { partitions?: number; /** Runtime bitcode whose small functions the helper imports for inlining. */ importBitcode?: { paths: readonly string[]; digests: readonly string[] }; + /** An existing file that holds exactly `llvm`; the helper reads it in place. */ + inputPath?: string; +} + +/** Writes LLVM text in large batches. Programs past the engine's string + * limit arrive as many parts, and writing each part separately is several + * times slower than writing the same bytes in a few large chunks. */ +export async function writeLlvmSource( + path: string, + llvm: string | readonly string[], + mode = 0o666, +): Promise { + const file = await open(path, "w", mode); + try { + const write = async (text: string): Promise => { + const bytes = Buffer.from(text, "utf8"); + for (let offset = 0; offset < bytes.length;) { + const { bytesWritten } = await file.write(bytes, offset, bytes.length - offset); + offset += bytesWritten; + } + }; + if (typeof llvm === "string") return await write(llvm); + let batch: string[] = []; + let length = 0; + for (const part of llvm) { + batch.push(part); + length += part.length; + if (length >= 16 * 1024 * 1024) { + await write(batch.join("")); + batch = []; + length = 0; + } + } + if (batch.length > 0) await write(batch.join("")); + } finally { + await file.close(); + } } function parseJsonObject(text: string): Record | null { @@ -358,13 +395,29 @@ export async function emitNativeArtifact( outputPaths: outputs, } satisfies NativeCodegenArtifact; if (await installCachedPartitions(cached, outputs)) return artifact; + // Partitioned programs also reuse individual partition artifacts, so an + // edit recompiles only the partitions it affects. The directory names the + // exact helper build and target; the helper keys entries by content. + const partitionCache = + root === null || outputs.length < 2 + ? null + : join( + root, + "native-partitions-v1", + createHash("sha256") + .update(JSON.stringify({ helper: helper.identity, target })) + .digest("hex") + .slice(0, 32), + ); + if (partitionCache !== null) await mkdir(partitionCache, { recursive: true, mode: 0o700 }); const stages = outputs.map((output) => privateSiblingPath(output, `native-${options.outputKind}`), ); - const input = privateSiblingPath(options.outputPath, "native-input"); + const ownsInput = options.inputPath === undefined; + const input = options.inputPath ?? privateSiblingPath(options.outputPath, "native-input"); try { - await writeFile(input, options.llvm, { mode: 0o600 }); + if (ownsInput) await writeLlvmSource(input, options.llvm, 0o600); await invoke(helper.binaryPath, [ "emit", "--input", @@ -383,6 +436,7 @@ export async function emitNativeArtifact( "--source-path", options.sourcePath, ...(options.importBitcode?.paths ?? []).flatMap((path) => ["--import-bitcode", path]), + ...(partitionCache === null ? [] : ["--cache-dir", partitionCache]), ]); for (const stage of stages) { const emitted = await stat(stage).catch(() => null); @@ -412,7 +466,7 @@ export async function emitNativeArtifact( } finally { await Promise.all([ ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)), - rm(input, { force: true }).catch(() => undefined), + ...(ownsInput ? [rm(input, { force: true }).catch(() => undefined)] : []), ]); } } diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index eaacd8353..d9d47ba9f 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -43,6 +43,8 @@ export function emitNativeObject(options: { outputKind?: NativeCodegenOutputKind; /** Program partitions, written to nativePartitionPaths(outputPath). */ partitions?: number; + /** Private directory where the helper reuses partition artifacts. */ + partitionCache?: string; }): string[] { verifyNativeHelper(options); const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1); @@ -63,6 +65,9 @@ export function emitNativeObject(options: { "json", "--source-path", options.sourcePath, + ...(options.partitionCache === undefined || outputs.length < 2 + ? [] + : ["--cache-dir", options.partitionCache]), ]); for (const output of outputs) requireNativeArtifact(output); return outputs; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 39dac8a4c..418cdfd95 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,8 +1,4 @@ -import { - optimizationClass, - optimizationField, - type NativeOptimization, -} from "./backend/optimization.js"; +import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; import { compilationTiming, type CompilationTiming } from "./timing.js"; import { prepareExecutableModule } from "./executable/prepare.js"; import { @@ -66,7 +62,11 @@ import { resolveCc, } from "./backend/external-c.js"; import { emitLlvmModuleSource, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; -import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen.js"; +import { + emitNativeArtifact, + NativeCodegenError, + writeLlvmSource, +} from "./backend/native-codegen.js"; import { nativePartitionPaths, nativeProgramPartitions } from "./backend/native-codegen-core.js"; import { privateSiblingPath } from "./backend/build-cache.js"; import { nativeCodegenTarget, nativeCodegenTargetRefusal } from "./backend/targets.js"; @@ -551,6 +551,7 @@ async function emitNativeProgramObject( opts: CompileRequestOptions, llvm: string | readonly string[], features: EarlyExecutableNativeFeatures, + llvmPath: string, ): Promise { const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const artifactPath = join(opts.outDir, `${stem}.helper.o`); @@ -565,7 +566,6 @@ async function emitNativeProgramObject( ? 1 : nativeProgramPartitions( target, - optimizationClass(opts.optimization), (typeof llvm === "string" ? [llvm] : llvm).reduce( (bytes, part) => bytes + Buffer.byteLength(part), 0, @@ -582,6 +582,7 @@ async function emitNativeProgramObject( const artifact = await emitNativeArtifact({ outputPath: linkPath, llvm, + inputPath: llvmPath, outputKind: "obj", sourcePath: entryPath, optimization: opts.optimization === "dev" ? "0" : "2", @@ -762,7 +763,7 @@ async function prepareExecutableInput( return { ok: false, diagnostics: [llvmRefusalDiag(err, entryPath)], sourceTexts }; } timing("llvm-emit"); - await writeFile(llvmPath, llvmSource); + await writeLlvmSource(llvmPath, llvmSource); timing("llvm-write"); let irPath: string | undefined; if (opts.emitIr) { @@ -1082,6 +1083,7 @@ async function compileTracked( opts, await readFile(earlyHit.llvmPath, "utf8"), earlyHit.native, + earlyHit.llvmPath, ); nativeInputPath = nativeProgramObject.linkPath; } catch (err) { @@ -1194,6 +1196,7 @@ async function compileTracked( opts, llvmSource, nativeFeatures, + llvmPath, ); timing("native-object"); } catch (err) { diff --git a/packages/compiler/src/native/cache.test.ts b/packages/compiler/src/native/cache.test.ts index bfe9513ad..d5ac6bc6a 100644 --- a/packages/compiler/src/native/cache.test.ts +++ b/packages/compiler/src/native/cache.test.ts @@ -1,4 +1,13 @@ -import { mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test, vi } from "vitest"; @@ -53,6 +62,27 @@ test("eviction removes oldest complete entries and leaves unrelated paths alone" expect(readFileSync(unrelated, "utf8")).toBe("keep"); }); +test("eviction also ages helper partition entries", () => { + const cache = new NativeCache(directory()); + const partitions = cache.partitionDirectory("helper identity")!; + expect(partitions).toBe(cache.partitionDirectory("helper identity")); + expect(statSync(partitions).mode & 0o777).toBe(0o700); + const bucket = join(partitions, "ab"); + mkdirSync(bucket); + const oldEntry = join(bucket, `${contentDigest("old")}.o`); + const newEntry = join(bucket, `${contentDigest("new")}.bc`); + const pending = join(bucket, ".tmp-pending"); + writeFileSync(oldEntry, "a".repeat(700)); + writeFileSync(newEntry, "b".repeat(700)); + writeFileSync(pending, "c".repeat(700)); + utimesSync(oldEntry, new Date(0), new Date(0)); + vi.stubEnv("SCRIPTC_CACHE_MAX_MB", String(1000 / (1024 * 1024))); + cache.prune(); + expect(existsSync(oldEntry)).toBe(false); + expect(readFileSync(newEntry, "utf8")).toHaveLength(700); + expect(readFileSync(pending, "utf8")).toHaveLength(700); +}); + test("private-directory admission and cache disable options fail closed", () => { vi.stubEnv("SCRIPTC_NO_CACHE", "0"); vi.stubEnv("SCRIPTC_CACHE_DIR", directory()); diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index b89605dbe..22741a0c4 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -40,6 +40,19 @@ export class NativeCache { } } + /** Directory where the LLVM helper reuses partition artifacts for one + * helper build. The helper names and verifies the entries itself. */ + partitionDirectory(helperIdentity: string): string | null { + const directory = join(this.root, "partitions", contentDigest(helperIdentity).slice(0, 32)); + try { + mkdirSync(directory, { recursive: true, mode: 0o700 }); + return directory; + } catch { + /* An unavailable cache only prevents reuse. */ + return null; + } + } + write(family: string, key: string, bytes: string | Uint8Array): void { let stage: string | null = null; try { @@ -96,6 +109,29 @@ export class NativeCache { total += info.size; } } + // Helper partition entries: partitions///.. + const partitions = join(this.root, "partitions"); + const list = (directory: string) => { + try { + return readdirSync(directory, { withFileTypes: true }); + } catch { + return []; + } + }; + for (const helper of list(partitions)) { + if (!helper.isDirectory()) continue; + for (const bucket of list(join(partitions, helper.name))) { + if (!bucket.isDirectory()) continue; + const directory = join(partitions, helper.name, bucket.name); + for (const entry of list(directory)) { + if (!entry.isFile() || !/^[0-9a-f]{64}\.(?:o|bc)$/.test(entry.name)) continue; + const path = join(directory, entry.name); + const info = statSync(path); + files.push({ path, size: info.size, time: info.mtimeMs }); + total += info.size; + } + } + } if (total <= limit) return; files.sort((a, b) => a.time - b.time); for (const file of files) { diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index aaf2d87ff..984eb4339 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -56,7 +56,7 @@ import { archiveNativeLibrary, localizeNativeLibrary, linkNativeWasmLibrary } fr import type { NativeToolchain } from "./toolchain.js"; import { evaluateNativeComptime } from "./comptime.js"; import { contentDigest, type NativeCache } from "./cache.js"; -import { splitLlvmProgram, splitLlvmLibraryProgram } from "../backend/llvm/split.js"; +import { splitLlvmLibraryProgram } from "../backend/llvm/split.js"; import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; import { openNativeExecutableCache } from "./executable-cache.js"; @@ -189,6 +189,10 @@ export class NativeCompiler { key = null; } } + const partitionCache = + this.cache === null || identity === null || partitions < 2 + ? null + : this.cache.partitionDirectory(identity); emitNativeObject({ executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, @@ -201,6 +205,7 @@ export class NativeCompiler { optimization, outputKind, partitions, + ...(partitionCache === null ? {} : { partitionCache }), }); if (this.cache !== null && key !== null) { try { @@ -223,16 +228,17 @@ export class NativeCompiler { stage: string, library: boolean, ): string[] { + // Library archives merge dev shards relocatably; executables link the + // helper's program partitions directly. const llvm = - optimization === "dev" && this.toolchain.target.platform !== "wasi" + library && optimization === "dev" && this.toolchain.target.platform !== "wasi" ? readFileSync(input, "utf8") : null; - const split = - llvm === null ? null : library ? splitLlvmLibraryProgram(llvm) : splitLlvmProgram(llvm); + const split = llvm === null ? null : splitLlvmLibraryProgram(llvm); if (split === null) { const partitions = library ? 1 - : nativeProgramPartitions(this.toolchain.target, optimization, statSync(input).size); + : nativeProgramPartitions(this.toolchain.target, statSync(input).size); return this.emitObject(input, output, source, optimization, "obj", partitions); } const objects: string[] = []; diff --git a/packages/compiler/test/native-partitions.test.ts b/packages/compiler/test/native-partitions.test.ts new file mode 100644 index 000000000..bb07ee5fc --- /dev/null +++ b/packages/compiler/test/native-partitions.test.ts @@ -0,0 +1,234 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, test } from "vitest"; +import { nativeCodegenTarget, nativeHelperForTarget } from "../src/backend/targets.js"; + +const execFileAsync = promisify(execFile); +const target = nativeCodegenTarget(); +const helperSpec = target === null ? null : nativeHelperForTarget(target); +const helper = (() => { + if (helperSpec === null || process.platform === "win32") return null; + try { + const packageJson = createRequire(import.meta.url).resolve( + `${helperSpec.packageName}/package.json`, + ); + return join(dirname(packageJson), "bin", "scriptc-llvm-codegen"); + } catch { + return null; + } +})(); +const dirs: string[] = []; + +afterEach(async () => { + await Promise.all(dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +/** A program whose internal functions, constants and state must stay + * reachable across partitions. `edit` changes one function's addend. */ +function program(edit = 0, extra = ""): string { + const count = 24; + const lines = [ + "; partitioned program", + "%Pair = type { i64, i64 }", + "@total = internal global i64 0", + "@pair = internal global %Pair { i64 3, i64 4 }", + '@label = private unnamed_addr constant [6 x i8] c"total\\00"', + "declare i64 @strlen(ptr)", + extra, + "define internal void @bump(i64 %value) {", + " %old = load i64, ptr @total", + " %new = add i64 %old, %value", + " store i64 %new, ptr @total", + " ret void", + "}", + ]; + for (let index = 0; index < count; index++) { + const addend = index === 5 ? index + edit : index; + lines.push( + `define internal void @step${index}(i64 %value) noinline {`, + ` %scaled = mul i64 %value, ${addend}`, + ` call void @bump(i64 %scaled)`, + ...(index + 1 < count ? [` call void @step${index + 1}(i64 %value)`] : []), + " ret void", + "}", + ); + } + lines.push( + "define i32 @main() {", + " %second = load i64, ptr getelementptr (%Pair, ptr @pair, i32 0, i32 1)", + " call void @step0(i64 %second)", + // A second caller for every step keeps the steps in separate partitions. + ...Array.from({ length: count - 1 }, (_, index) => ` call void @step${index + 1}(i64 1)`), + " %length = call i64 @strlen(ptr @label)", + " call void @bump(i64 %length)", + " %total = load i64, ptr @total", + " %low = urem i64 %total, 251", + " %code = trunc i64 %low to i32", + " ret i32 %code", + "}", + ); + return lines.join("\n") + "\n"; +} + +function expectedExit(edit = 0): number { + const addends = Array.from({ length: 24 }, (_, index) => (index === 5 ? index + edit : index)); + let total = 5; // strlen("total") + for (let start = 0; start < 24; start++) + for (let index = start; index < 24; index++) total += (start === 0 ? 4 : 1) * addends[index]!; + return total % 251; +} + +async function emit( + dir: string, + source: string, + options: { outputs: number; optLevel: "0" | "2"; cache?: string; name: string }, +): Promise { + const input = join(dir, `${options.name}.ll`); + await writeFile(input, source); + const outputs = Array.from({ length: options.outputs }, (_, index) => + join(dir, `${options.name}.${index}.o`), + ); + await execFileAsync(helper!, [ + "emit", + "--input", + input, + ...outputs.flatMap((output) => ["--output", output]), + "--filetype", + "obj", + "--target", + target!.llvmTriple, + "--opt-level", + options.optLevel, + "--relocation-model", + "pic", + "--diagnostic-format", + "json", + "--source-path", + "/src/program.ts", + ...(options.cache === undefined ? [] : ["--cache-dir", options.cache]), + ]); + return outputs; +} + +async function exitCode(dir: string, objects: string[], name: string): Promise { + const executable = join(dir, name); + await execFileAsync(process.env["CC"] ?? "cc", [...objects, "-o", executable]); + try { + await execFileAsync(executable, []); + return 0; + } catch (error) { + return (error as { code: number }).code; + } +} + +async function cacheEntries(cache: string, suffix: string): Promise> { + const entries = new Set(); + for (const bucket of await readdir(cache).catch(() => [])) { + for (const name of await readdir(join(cache, bucket)).catch(() => [])) + if (name.endsWith(suffix)) entries.add(name); + } + return entries; +} + +describe.runIf(helper !== null)("LLVM program partitions", () => { + for (const optLevel of ["2", "0"] as const) { + test(`opt-level ${optLevel} partitions link deterministically and reuse unchanged partitions`, async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-")); + dirs.push(dir); + const cache = join(dir, "cache"); + const first = await emit(dir, program(), { outputs: 4, optLevel, cache, name: "first" }); + const whole = await emit(dir, program(), { outputs: 1, optLevel, name: "whole" }); + expect(await exitCode(dir, first, "first")).toBe(expectedExit()); + expect(await exitCode(dir, whole, "whole")).toBe(expectedExit()); + + // The same module always yields the same objects, with or without reuse. + const fresh = await emit(dir, program(), { outputs: 4, optLevel, name: "fresh" }); + const reused = await emit(dir, program(), { outputs: 4, optLevel, cache, name: "reused" }); + const contents = (paths: string[]) => Promise.all(paths.map((path) => readFile(path))); + expect(await contents(fresh)).toEqual(await contents(first)); + expect(await contents(reused)).toEqual(await contents(first)); + + // An edit to one function changes one partition's source; the rest + // are reused. + const sources = await cacheEntries(cache, optLevel === "0" ? ".o" : ".bc"); + const edited = await emit(dir, program(7), { outputs: 4, optLevel, cache, name: "edited" }); + const after = await cacheEntries(cache, optLevel === "0" ? ".o" : ".bc"); + expect([...after].filter((entry) => !sources.has(entry))).toHaveLength(1); + expect(await exitCode(dir, edited, "edited")).toBe(expectedExit(7)); + }); + } + + test("inputs outside the text partitioner's grammar are still partitioned", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-fallback-")); + dirs.push(dir); + const source = program(0, '@"quoted name" = internal global i64 1').replace( + " %old = load i64, ptr @total", + ' %old = load i64, ptr @total\n %unused = load i64, ptr @"quoted name"', + ); + for (const optLevel of ["2", "0"] as const) { + const objects = await emit(dir, source, { + outputs: 3, + optLevel, + name: `fallback${optLevel}`, + }); + expect(await exitCode(dir, objects, `fallback${optLevel}`)).toBe(expectedExit()); + } + }); + + test("debug metadata follows the functions that reference it", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-debug-")); + dirs.push(dir); + const debug = program() + .split("\n") + .map((line) => { + const definition = /^define internal void @step(\d+)\(i64 %value\) noinline \{$/.exec(line); + if (definition !== null) + return line.replace(" {", ` !dbg !${10 + Number(definition[1])} {`); + const call = /^ {2}call void @step(\d+)\(i64 %value\)$/.exec(line); + return call === null ? line : `${line}, !dbg !${40 + Number(call[1]) - 1}`; + }); + const source = [ + ...debug, + "!llvm.dbg.cu = !{!0}", + "!llvm.module.flags = !{!2, !3}", + '!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "test", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)', + '!1 = !DIFile(filename: "program.ts", directory: "/src")', + '!2 = !{i32 2, !"Debug Info Version", i32 3}', + '!3 = !{i32 7, !"Dwarf Version", i32 5}', + "!4 = !DISubroutineType(types: !{})", + ...Array.from( + { length: 24 }, + (_, index) => + `!${10 + index} = distinct !DISubprogram(name: "step${index}", scope: !1, file: !1, line: ${index + 1}, type: !4, spFlags: DISPFlagDefinition, unit: !0)`, + ), + ...Array.from( + { length: 23 }, + (_, index) => + `!${40 + index} = !DILocation(line: ${index + 1}, column: 3, scope: !${10 + index})`, + ), + "", + ].join("\n"); + const cache = join(dir, "cache"); + const objects = await emit(dir, source, { outputs: 4, optLevel: "0", cache, name: "debug" }); + const bytes = await Promise.all(objects.map((path) => readFile(path))); + expect(bytes.filter((object) => object.includes(".debug_info")).length).toBeGreaterThan(1); + expect(await exitCode(dir, objects, "debug")).toBe(expectedExit()); + + // Partitions number their metadata locally, so a module whose node + // numbers all shift still reuses every partition. + const before = await cacheEntries(cache, ".o"); + const shifted = source.replace(/!(\d+)/g, (_, number: string) => `!${Number(number) + 100}`); + const renumbered = await emit(dir, shifted, { + outputs: 4, + optLevel: "0", + cache, + name: "renumbered", + }); + expect(await cacheEntries(cache, ".o")).toEqual(before); + expect(await Promise.all(renumbered.map((path) => readFile(path)))).toEqual(bytes); + }); +});