#include "runtime-import.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalAlias.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/Linker/Linker.h" #include "llvm/Support/Error.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" #include #include #include using namespace llvm; namespace scriptc { namespace { // Upper bound on the instruction count of an imported function. The // inliner never takes bodies much larger than this at O2, and bounding the // import keeps the extra optimization work proportional to what can pay off. // // Like ThinLTO's function import, the limit applies to functions the program // references directly and decays with every call-graph hop below them. Deep // slow paths (cycle collection, destruction queues, allocation growth) then // stay out of line instead of being folded bottom-up into small entry points // until those entry points are too large to inline at program call sites. constexpr unsigned MaxInstructions = 100; constexpr unsigned DecayPercent = 30; struct References { // Every global referenced by the body, including through constant // expressions. SmallPtrSet Globals; // Local functions referenced other than as the callee of a direct call. // Copying such a function would give it a second address. bool LocalFunctionAddressTaken = false; // First symbol that prevents import, for diagnostics. std::string Blocker; }; void collectConstant(const Constant *C, SmallPtrSetImpl &Out, SmallPtrSetImpl &Seen) { if (const auto *GV = dyn_cast(C)) { Out.insert(GV); return; } if (!Seen.insert(C).second) return; for (const Use &Operand : C->operands()) if (const auto *Inner = dyn_cast(Operand.get())) collectConstant(Inner, Out, Seen); } bool candidate(const Function &F, unsigned Limit) { if (F.isDeclaration() || F.isVarArg() || F.hasFnAttribute(Attribute::NoInline) || F.hasFnAttribute(Attribute::OptimizeNone) || F.hasFnAttribute(Attribute::Naked) || F.hasPersonalityFn() || F.hasPrefixData() || F.hasPrologueData()) return false; // Only strong external or local definitions. A weak or linkonce // definition may be replaced at link time, so its body is not the one // that will run. if (!F.hasExternalLinkage() && !F.hasLocalLinkage()) return false; // The program calls the runtime ABI marker so that linking against a // mismatched runtime object fails; that reference must survive. if (F.getName().starts_with("scr_runtime_abi_")) return false; if (F.hasLocalLinkage() && F.hasAddressTaken()) return false; unsigned Count = 0; for (const Instruction &I : instructions(F)) { if (++Count > Limit) return false; if (const auto *Call = dyn_cast(&I)) { if (Call->hasFnAttr(Attribute::ReturnsTwice)) return false; if (const Function *Callee = Call->getCalledFunction()) if (Callee->hasFnAttribute(Attribute::ReturnsTwice)) return false; } for (const Use &Operand : I.operands()) if (isa(Operand.get())) return false; } return true; } References referencesOf(const Function &F) { References Refs; SmallPtrSet Seen; for (const Instruction &I : instructions(F)) { const auto *Call = dyn_cast(&I); for (const Use &Operand : I.operands()) { const Value *V = Operand.get(); if (const auto *Local = dyn_cast(V)) { Refs.Globals.insert(Local); if (Local->hasLocalLinkage() && !(Call && Call->isCallee(&Operand))) { Refs.LocalFunctionAddressTaken = true; if (Refs.Blocker.empty()) Refs.Blocker = ("&" + Local->getName()).str(); } continue; } if (const auto *C = dyn_cast(V)) { SmallPtrSet Inner; collectConstant(C, Inner, Seen); for (const GlobalValue *GV : Inner) { Refs.Globals.insert(GV); if (isa(GV) && GV->hasLocalLinkage()) { Refs.LocalFunctionAddressTaken = true; if (Refs.Blocker.empty()) Refs.Blocker = ("&" + GV->getName()).str(); } } } } } return Refs; } // A local constant may be copied when its address is insignificant and its // initializer only names external symbols or other copyable constants. bool copyableConstant(const GlobalVariable &V, DenseMap &Memo) { auto Found = Memo.find(&V); if (Found != Memo.end()) return Found->second; Memo[&V] = false; if (!V.isConstant() || V.isThreadLocal() || !V.hasAtLeastLocalUnnamedAddr() || !V.hasInitializer()) return false; SmallPtrSet Inner; SmallPtrSet Seen; collectConstant(V.getInitializer(), Inner, Seen); for (const GlobalValue *GV : Inner) { if (!GV->hasLocalLinkage()) { if (isa(GV) || isa(GV)) return false; continue; } const auto *Nested = dyn_cast(GV); if (Nested == nullptr || !copyableConstant(*Nested, Memo)) return false; } Memo[&V] = true; return true; } void stripTargetSpecificAttributes(Function &F) { // The program module compiles for the helper's target machine; keep the // imported bodies on the same subtarget so they are inline-compatible, // and do not spread the runtime's stack protector into program frames. F.removeFnAttr("target-cpu"); F.removeFnAttr("target-features"); F.removeFnAttr("tune-cpu"); F.removeFnAttr("stack-protector-buffer-size"); // Apple clang requests __chkstk_darwin probes, which upstream LLVM does not // lower; program frames are not probed either. F.removeFnAttr("probe-stack"); F.removeFnAttr("stack-probe-size"); F.removeFnAttr(Attribute::StackProtect); F.removeFnAttr(Attribute::StackProtectStrong); F.removeFnAttr(Attribute::StackProtectReq); } // Rewrites a merged runtime module so that linking it with LinkOnlyNeeded // imports only safe, available_externally definitions. void prepareImport(Module &Runtime, const std::vector &Roots) { unsigned Limit = MaxInstructions; std::vector Candidates; DenseMap Refs; DenseSet Importable; DenseMap Blockers; for (Function &F : Runtime) if (candidate(F, Limit)) { References R = referencesOf(F); if (R.LocalFunctionAddressTaken) { Blockers[&F] = R.Blocker; continue; } Candidates.push_back(&F); Importable.insert(&F); Refs[&F] = std::move(R); } DenseMap ConstantMemo; auto Restrict = [&]() { bool Changed = true; while (Changed) { Changed = false; for (Function *F : Candidates) { if (!Importable.contains(F)) continue; bool Ok = true; for (const GlobalValue *GV : Refs[F].Globals) { if (isa(GV) || isa(GV)) { Ok = false; break; } if (!GV->hasLocalLinkage()) continue; if (const auto *Callee = dyn_cast(GV)) { if (!Importable.contains(Callee)) { Ok = false; break; } } else if (!copyableConstant(*cast(GV), ConstantMemo)) { Ok = false; break; } } if (!Ok) { if (Blockers.find(F) == Blockers.end()) { for (const GlobalValue *GV : Refs[F].Globals) if (GV->hasLocalLinkage() || isa(GV)) { const auto *Callee = dyn_cast(GV); const auto *Var = dyn_cast(GV); bool Blocks = Callee ? !Importable.contains(Callee) : Var ? !copyableConstant(*Var, ConstantMemo) : true; if (Blocks) { Blockers[F] = GV->getName().str(); break; } } } Importable.erase(F); Changed = true; } } } }; Restrict(); // Select by call-graph distance from the program's references. { DenseSet Selected; DenseSet Queued; std::vector> Queue; for (const std::string &Name : Roots) if (Function *F = Runtime.getFunction(Name)) if (Importable.contains(F) && Queued.insert(F).second) Queue.push_back({F, 0}); for (size_t Next = 0; Next < Queue.size(); ++Next) { auto [F, Depth] = Queue[Next]; double Allowed = Limit; for (unsigned I = 0; I < Depth; ++I) Allowed = Allowed * DecayPercent / 100.0; if (F->getInstructionCount() > Allowed) continue; Selected.insert(F); for (const GlobalValue *GV : Refs[F].Globals) if (const auto *Callee = dyn_cast(GV)) { auto *Mutable = const_cast(Callee); if (Importable.contains(Callee) && Queued.insert(Callee).second) Queue.push_back({Mutable, Depth + 1}); } } Importable = std::move(Selected); Restrict(); } if (std::getenv("SCRIPTC_RUNTIME_IMPORT_DEBUG") != nullptr) { for (Function &F : Runtime) { if (F.isDeclaration()) continue; const char *State = Importable.contains(&F) ? "import" : Refs.count(&F) ? "reject-refs" : "reject"; auto Blocker = Blockers.find(&F); errs() << State << " " << F.getName() << " " << F.getInstructionCount() << " " << (Blocker == Blockers.end() ? "" : Blocker->second) << "\n"; } } for (Function &F : Runtime) { if (F.isDeclaration()) continue; if (Importable.contains(&F)) { stripTargetSpecificAttributes(F); if (!F.hasLocalLinkage()) { F.setLinkage(GlobalValue::AvailableExternallyLinkage); F.setComdat(nullptr); } } else if (!F.hasLocalLinkage()) { F.deleteBody(); F.setComdat(nullptr); } } std::vector Appending; for (GlobalVariable &GV : Runtime.globals()) { if (GV.hasAppendingLinkage()) { Appending.push_back(&GV); continue; } if (!GV.isDeclaration() && !GV.hasLocalLinkage()) { GV.setInitializer(nullptr); GV.setLinkage(GlobalValue::ExternalLinkage); GV.setComdat(nullptr); } } for (GlobalVariable *GV : Appending) GV->eraseFromParent(); std::vector Aliases; for (GlobalAlias &A : Runtime.aliases()) Aliases.push_back(&A); for (GlobalAlias *A : Aliases) { if (A->hasLocalLinkage()) continue; GlobalValue *Declaration; if (auto *Type = dyn_cast(A->getValueType())) Declaration = Function::Create(Type, GlobalValue::ExternalLinkage, A->getAddressSpace(), "", &Runtime); else Declaration = new GlobalVariable(Runtime, A->getValueType(), false, GlobalValue::ExternalLinkage, nullptr, ""); Declaration->takeName(A); A->replaceAllUsesWith(Declaration); A->eraseFromParent(); } // Module-level assembly defines symbols in the runtime object (musl's // getcontext/setcontext); the linker would otherwise copy it verbatim into // the program and define those symbols twice. Runtime.setModuleInlineAsm(""); // Module flags, identification, and autolink metadata describe the runtime // object, which is linked separately; none of it applies to program code. std::vector Named; for (NamedMDNode &Node : Runtime.named_metadata()) Named.push_back(&Node); for (NamedMDNode *Node : Named) Runtime.eraseNamedMetadata(Node); } } // namespace std::optional importRuntimeBitcode(Module &Program, const std::vector &BitcodePaths) { if (BitcodePaths.empty()) return std::nullopt; LLVMContext &Context = Program.getContext(); std::unique_ptr Runtime; std::unique_ptr Merger; for (const std::string &Path : BitcodePaths) { ErrorOr> Buffer = MemoryBuffer::getFile(Path); if (!Buffer) return (Twine("could not read runtime bitcode ") + Path + ": " + Buffer.getError().message()) .str(); Expected> Unit = parseBitcodeFile((*Buffer)->getMemBufferRef(), Context); if (!Unit) return (Twine("invalid runtime bitcode ") + Path + ": " + toString(Unit.takeError())) .str(); if ((*Unit)->getDataLayout() != Program.getDataLayout()) return (Twine("runtime bitcode ") + Path + " has a different data layout than the program") .str(); (*Unit)->setTargetTriple(Program.getTargetTriple()); if (!Runtime) { Runtime = std::move(*Unit); Merger = std::make_unique(*Runtime); } else if (Merger->linkInModule(std::move(*Unit))) { return (Twine("could not merge runtime bitcode ") + Path).str(); } } Merger.reset(); std::vector Roots; for (const Function &F : Program) if (F.isDeclaration() && !F.isIntrinsic()) Roots.push_back(F.getName().str()); prepareImport(*Runtime, Roots); if (Linker::linkModules(Program, std::move(Runtime), Linker::Flags::LinkOnlyNeeded)) return std::string("could not link runtime bitcode into the program"); return std::nullopt; } } // namespace scriptc