feat(compiler): add native LLVM object emission (#242)

* feat(compiler): add native LLVM object emission

- ship the pinned macOS arm64 LLVM helper and platform package
- add assembly/object CLI outputs, caching, diagnostics, and runtime ABI checks
- validate helper artifacts through CI, packaging, docs, and differential coverage

* fix(compiler): harden native LLVM emission

* fix(compiler): harden LLVM helper packaging

* fix(compiler): stabilize LLVM helper package size

* fix(compiler): harden native helper validation

* fix(compiler): harden LLVM helper verification

* fix(compiler): harden native object emission

native/llvm-codegen/CMakeLists.txt

@@ -0,0 +1,79 @@
+cmake_minimum_required(VERSION 3.24)
+
+# These must be cache variables before project() enables the Apple toolchain.
+# A target property named OSX_DEPLOYMENT_TARGET is not recognized by CMake and
+# silently leaves the executable at the build host/SDK's deployment version.
+if(CMAKE_HOST_APPLE)
+  set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING
+      "Architectures for the macOS helper" FORCE)
+  # Homebrew's pinned LLVM 22 bottle is built for macOS 15. The helper must
+  # tell the truth about that host requirement; the objects it emits retain
+  # their separate arm64-apple-macosx14.0.0 deployment target.
+  set(CMAKE_OSX_DEPLOYMENT_TARGET "15.0" CACHE STRING
+      "Minimum macOS version for the packaged helper" FORCE)
+endif()
+
+project(scriptc_llvm_codegen LANGUAGES C CXX)
+
+find_package(LLVM 22.1.8 EXACT REQUIRED CONFIG)
+
+set(SCRIPTC_PACKAGE_VERSION "0.0.0-dev" CACHE STRING
+    "scriptc npm package version reported by the helper protocol")
+
+# LLVM's exported Support target names zstd's shared target even when the
+# selected LLVM components are static. The npm sidecar must not retain a
+# Homebrew zstd dylib dependency.
+if(TARGET zstd::libzstd_shared AND TARGET zstd::libzstd_static)
+  get_target_property(SCRIPTC_ZSTD_STATIC zstd::libzstd_static IMPORTED_LOCATION_RELEASE)
+  if(NOT SCRIPTC_ZSTD_STATIC)
+    get_target_property(SCRIPTC_ZSTD_STATIC zstd::libzstd_static IMPORTED_LOCATION)
+  endif()
+  if(SCRIPTC_ZSTD_STATIC)
+    set_target_properties(zstd::libzstd_shared PROPERTIES
+      IMPORTED_LOCATION "${SCRIPTC_ZSTD_STATIC}"
+      IMPORTED_LOCATION_RELEASE "${SCRIPTC_ZSTD_STATIC}"
+    )
+  endif()
+endif()
+
+add_executable(scriptc-llvm-codegen
+  src/diagnostics.cpp
+  src/emit.cpp
+  src/main.cpp
+  src/target.cpp
+)
+
+target_compile_features(scriptc-llvm-codegen PRIVATE cxx_std_17)
+target_include_directories(scriptc-llvm-codegen PRIVATE ${LLVM_INCLUDE_DIRS})
+target_compile_definitions(scriptc-llvm-codegen PRIVATE
+  ${LLVM_DEFINITIONS}
+  SCRIPTC_PACKAGE_VERSION="${SCRIPTC_PACKAGE_VERSION}"
+)
+
+llvm_map_components_to_libnames(SCRIPTC_LLVM_LIBS
+  AArch64CodeGen
+  CodeGen
+  Core
+  Coroutines
+  IRReader
+  Passes
+  Support
+  Target
+)
+target_link_libraries(scriptc-llvm-codegen PRIVATE ${SCRIPTC_LLVM_LIBS})
+
+if(APPLE)
+  # Homebrew's static LLVM archives contain broad component object files.
+  # Export only the process entry point so archive-global LLVM symbols do not
+  # become roots, then discard functions/data the small helper protocol cannot
+  # reach. This also keeps the result stable across LLVM bottle layouts whose
+  # static archives expose different sets of global symbols.
+  target_link_options(scriptc-llvm-codegen PRIVATE
+    "LINKER:-exported_symbol,_main"
+    "LINKER:-dead_strip"
+    # Homebrew's unversioned LLVM 22 bottle adds Z3 as an unconditional
+    # transitive dylib even though this helper never reaches the solver. Drop
+    # unused dylibs so the npm sidecar remains runnable without Homebrew.
+    "LINKER:-dead_strip_dylibs"
+  )
+endif()

native/llvm-codegen/README.md

@@ -0,0 +1,34 @@
+# scriptc LLVM code-generation helper
+
+This out-of-process helper owns LLVM assembly and object emission for
+scriptc. It is built against exactly LLVM 22.1.8 and currently contains only
+the AArch64 backend. The shipping `@scriptc/llvm-darwin-arm64` package builds
+and carries the executable; the compiler resolves that package directly and
+never searches `PATH` for this program.
+
+The ordinary workspace `pnpm -r build` does not rebuild this release artifact.
+On macOS arm64, install CMake, Ninja, and Homebrew `llvm@22`, then build it
+explicitly when working on native emission or preparing a package:
+
+```console
+$ brew install cmake ninja llvm@22
+$ pnpm --filter @scriptc/llvm-darwin-arm64 build:native
+```
+
+The protocol is intentionally small and versioned:
+
+The packaged helper itself requires macOS 15 or newer because that is the
+minimum version of the pinned LLVM bottle it statically links. Its emitted
+assembly and objects separately target macOS 14 via the triple below.
+
+```console
+scriptc-llvm-codegen version --format=json
+scriptc-llvm-codegen emit --input app.ll --output app.o --filetype obj \
+  --target arm64-apple-macosx14.0.0 --opt-level 2 \
+  --relocation-model pic --diagnostic-format json --source-path app.ts
+```
+
+Emission uses LLVM 22's default per-module O2 pipeline, including coroutine
+lowering, verifies before and after optimization, and publishes through a
+private sibling file so a failed or interrupted request cannot truncate the
+requested output.

native/llvm-codegen/src/diagnostics.cpp

@@ -0,0 +1,44 @@
+#include "diagnostics.h"
+
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cstdlib>
+
+using namespace llvm;
+
+namespace scriptc {
+
+int reportError(StringRef Code, const Twine &Message,
+                StringRef DiagnosticFormat) {
+  if (DiagnosticFormat == "json") {
+    json::Object Diagnostic{
+        {"ok", false},
+        {"code", Code},
+        {"message", Message.str()},
+    };
+    errs() << formatv("{0}\n", json::Value(std::move(Diagnostic)));
+  } else {
+    errs() << "scriptc-llvm-codegen: " << Message << '\n';
+  }
+  return 1;
+}
+
+static void fatalDiagnostic(void *, const char *Reason, bool) {
+  json::Object Diagnostic{
+      {"ok", false},
+      {"code", "llvm_fatal"},
+      {"message", Reason == nullptr ? "LLVM reported a fatal error" : Reason},
+  };
+  errs() << formatv("{0}\n", json::Value(std::move(Diagnostic)));
+  errs().flush();
+  std::_Exit(70);
+}
+
+void installFatalDiagnosticHandler() {
+  install_fatal_error_handler(fatalDiagnostic, nullptr);
+}
+
+} // namespace scriptc

native/llvm-codegen/src/diagnostics.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+
+namespace scriptc {
+
+int reportError(llvm::StringRef Code, const llvm::Twine &Message,
+                llvm::StringRef DiagnosticFormat = "json");
+void installFatalDiagnosticHandler();
+
+} // namespace scriptc

native/llvm-codegen/src/emit.cpp

@@ -0,0 +1,197 @@
+#include "emit.h"
+
+#include "diagnostics.h"
+#include "target.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/TargetParser/Triple.h"
+
+#include <optional>
+#include <string>
+#include <system_error>
+
+using namespace llvm;
+
+namespace scriptc {
+
+std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv) {
+  EmitOptions Options;
+  Options.Target = DefaultTarget.str();
+  for (int I = 2; I < Argc; ++I) {
+    StringRef Arg(Argv[I]);
+    if (!Arg.starts_with("--") || I + 1 >= Argc)
+      return std::nullopt;
+    StringRef Value(Argv[++I]);
+    if (Arg == "--input")
+      Options.Input = Value.str();
+    else if (Arg == "--output")
+      Options.Output = Value.str();
+    else if (Arg == "--filetype")
+      Options.FileType = Value.str();
+    else if (Arg == "--target")
+      Options.Target = Value.str();
+    else if (Arg == "--opt-level")
+      Options.OptLevel = Value.str();
+    else if (Arg == "--relocation-model")
+      Options.RelocationModel = Value.str();
+    else if (Arg == "--diagnostic-format")
+      Options.DiagnosticFormat = Value.str();
+    else if (Arg == "--source-path")
+      Options.SourcePath = Value.str();
+    else
+      return std::nullopt;
+  }
+  if (Options.Input.empty() || Options.Output.empty())
+    return std::nullopt;
+  return Options;
+}
+
+static OptimizationLevel optimizationLevel(StringRef Level) {
+  if (Level == "0")
+    return OptimizationLevel::O0;
+  if (Level == "1")
+    return OptimizationLevel::O1;
+  if (Level == "3")
+    return OptimizationLevel::O3;
+  if (Level == "s")
+    return OptimizationLevel::Os;
+  if (Level == "z")
+    return OptimizationLevel::Oz;
+  return OptimizationLevel::O2;
+}
+
+int emit(const EmitOptions &Options) {
+  if (Options.Target != DefaultTarget)
+    return reportError("unsupported_target",
+                       Twine("unsupported target '") + Options.Target +
+                           "' (supported: " + DefaultTarget + ")",
+                       Options.DiagnosticFormat);
+  if (Options.FileType != "obj" && Options.FileType != "asm")
+    return reportError("invalid_filetype", "filetype must be obj or asm",
+                       Options.DiagnosticFormat);
+  if (Options.OptLevel != "0" && Options.OptLevel != "1" &&
+      Options.OptLevel != "2" && Options.OptLevel != "3" &&
+      Options.OptLevel != "s" && Options.OptLevel != "z")
+    return reportError("invalid_opt_level",
+                       "opt-level must be 0, 1, 2, 3, s, or z",
+                       Options.DiagnosticFormat);
+  if (Options.RelocationModel != "pic")
+    return reportError("invalid_relocation_model",
+                       "only the pic relocation model is supported",
+                       Options.DiagnosticFormat);
+
+  SMDiagnostic ParseDiagnostic;
+  LLVMContext Context;
+  std::unique_ptr<Module> Mod =
+      parseIRFile(Options.Input, ParseDiagnostic, Context);
+  if (!Mod) {
+    std::string Detail;
+    raw_string_ostream Stream(Detail);
+    ParseDiagnostic.print("scriptc-llvm-codegen", Stream);
+    return reportError("invalid_ir", Stream.str(), Options.DiagnosticFormat);
+  }
+  if (!Options.SourcePath.empty())
+    Mod->setSourceFileName(Options.SourcePath);
+
+  std::string LookupError;
+  std::unique_ptr<TargetMachine> Machine =
+      createTargetMachine(Options.Target, Options.OptLevel, LookupError);
+  if (!Machine)
+    return reportError("target_machine_failed", LookupError,
+                       Options.DiagnosticFormat);
+
+  Triple TargetTriple(Options.Target);
+  Mod->setTargetTriple(TargetTriple);
+  Mod->setDataLayout(Machine->createDataLayout());
+  if (Mod->getDataLayoutStr() != DefaultDataLayout)
+    return reportError("data_layout_mismatch",
+                       Twine("LLVM produced unexpected data layout '") +
+                           Mod->getDataLayoutStr() + "'",
+                       Options.DiagnosticFormat);
+
+  std::string VerificationError;
+  raw_string_ostream VerificationStream(VerificationError);
+  if (verifyModule(*Mod, &VerificationStream))
+    return reportError("verification_failed", VerificationStream.str(),
+                       Options.DiagnosticFormat);
+
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(Machine.get());
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+  ModulePassManager Optimizations =
+      PB.buildPerModuleDefaultPipeline(optimizationLevel(Options.OptLevel));
+  Optimizations.run(*Mod, MAM);
+
+  VerificationError.clear();
+  if (verifyModule(*Mod, &VerificationStream))
+    return reportError("post_optimization_verification_failed",
+                       VerificationStream.str(), Options.DiagnosticFormat);
+
+  SmallString<256> OutputPath(Options.Output);
+  SmallString<256> TemporaryPath(OutputPath);
+  TemporaryPath.append(".tmp-%%%%%%");
+  int TemporaryFd = -1;
+  if (std::error_code EC =
+          sys::fs::createUniqueFile(TemporaryPath, TemporaryFd, TemporaryPath))
+    return reportError("output_open_failed", EC.message(),
+                       Options.DiagnosticFormat);
+
+  {
+    raw_fd_ostream Output(TemporaryFd, true);
+    legacy::PassManager CodeGeneration;
+    CodeGenFileType Type = Options.FileType == "obj"
+                               ? CodeGenFileType::ObjectFile
+                               : CodeGenFileType::AssemblyFile;
+    if (Machine->addPassesToEmitFile(CodeGeneration, Output, nullptr, Type)) {
+      sys::fs::remove(TemporaryPath);
+      return reportError("emission_not_supported",
+                         "target does not support the requested file type",
+                         Options.DiagnosticFormat);
+    }
+    CodeGeneration.run(*Mod);
+    Output.flush();
+    if (Output.has_error()) {
+      std::error_code EC = Output.error();
+      sys::fs::remove(TemporaryPath);
+      return reportError("output_write_failed", EC.message(),
+                         Options.DiagnosticFormat);
+    }
+  }
+
+  uint64_t Size = 0;
+  if (std::error_code EC = sys::fs::file_size(TemporaryPath, Size)) {
+    sys::fs::remove(TemporaryPath);
+    return reportError("output_verify_failed", EC.message(),
+                       Options.DiagnosticFormat);
+  }
+  if (Size == 0) {
+    sys::fs::remove(TemporaryPath);
+    return reportError("output_verify_failed", "LLVM emitted an empty file",
+                       Options.DiagnosticFormat);
+  }
+  if (std::error_code EC = sys::fs::rename(TemporaryPath, OutputPath)) {
+    sys::fs::remove(TemporaryPath);
+    return reportError("output_publish_failed", EC.message(),
+                       Options.DiagnosticFormat);
+  }
+  return 0;
+}
+
+} // namespace scriptc

native/llvm-codegen/src/emit.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <optional>
+#include <string>
+
+namespace scriptc {
+
+struct EmitOptions {
+  std::string Input;
+  std::string Output;
+  std::string FileType = "obj";
+  std::string Target;
+  std::string OptLevel = "2";
+  std::string RelocationModel = "pic";
+  std::string DiagnosticFormat = "json";
+  std::string SourcePath;
+};
+
+std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv);
+int emit(const EmitOptions &Options);
+
+} // namespace scriptc

native/llvm-codegen/src/main.cpp

@@ -0,0 +1,74 @@
+#include "diagnostics.h"
+#include "emit.h"
+#include "target.h"
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/TargetParser/Host.h"
+
+#include <optional>
+#include <cstdlib>
+#include <string>
+
+using namespace llvm;
+
+#ifndef SCRIPTC_PACKAGE_VERSION
+#define SCRIPTC_PACKAGE_VERSION "0.0.0-dev"
+#endif
+
+namespace {
+
+int version(int Argc, char **Argv) {
+  if (Argc != 3 || StringRef(Argv[2]) != "--format=json")
+    return scriptc::reportError("usage", "version requires --format=json");
+  std::string Error;
+  std::unique_ptr<TargetMachine> Machine = scriptc::createTargetMachine(
+      scriptc::DefaultTarget, "2", Error);
+  if (!Machine)
+    return scriptc::reportError("target_machine_failed", Error);
+
+  json::Object Response{
+      {"ok", true},
+      {"protocol_version", scriptc::ProtocolVersion},
+      {"scriptc_package_version", SCRIPTC_PACKAGE_VERSION},
+      {"llvm_version", LLVM_VERSION_STRING},
+      {"host_triple", sys::getDefaultTargetTriple()},
+      {"targets", json::Array{"AArch64"}},
+      {"default_target", scriptc::DefaultTarget},
+      {"data_layout", Machine->createDataLayout().getStringRepresentation()},
+  };
+  outs() << formatv("{0}\n", json::Value(std::move(Response)));
+  return 0;
+}
+
+} // namespace
+
+int main(int Argc, char **Argv) {
+  scriptc::installFatalDiagnosticHandler();
+  // Process-isolated test seam for the fatal handler itself. It is inert in
+  // every ordinary invocation and ensures a future LLVM fatal never becomes
+  // an unstructured abort/stack trace in the Node caller.
+  if (std::getenv("SCRIPTC_LLVM_TEST_FATAL") != nullptr)
+    report_fatal_error("scriptc LLVM fatal diagnostic self-test");
+  if (Argc >= 2 && StringRef(Argv[1]) == "version")
+    return version(Argc, Argv);
+  if (Argc >= 2 && StringRef(Argv[1]) == "emit") {
+    std::optional<scriptc::EmitOptions> Options =
+        scriptc::parseEmitOptions(Argc, Argv);
+    if (!Options)
+      return scriptc::reportError(
+          "usage",
+          "emit requires --input <file> --output <file> and accepts "
+          "--filetype <obj|asm> --target <triple> --opt-level <0|1|2|3|s|z> "
+          "--relocation-model pic --diagnostic-format json "
+          "--source-path <path>");
+    return scriptc::emit(*Options);
+  }
+  return scriptc::reportError("usage",
+                              "expected 'version --format=json' or 'emit'");
+}

native/llvm-codegen/src/target.cpp

@@ -0,0 +1,55 @@
+#include "target.h"
+
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CodeGen.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+#include "llvm/TargetParser/Triple.h"
+
+using namespace llvm;
+
+extern "C" {
+void LLVMInitializeAArch64TargetInfo();
+void LLVMInitializeAArch64Target();
+void LLVMInitializeAArch64TargetMC();
+void LLVMInitializeAArch64AsmPrinter();
+}
+
+namespace scriptc {
+
+void initializeTargets() {
+  static bool Initialized = false;
+  if (Initialized)
+    return;
+  LLVMInitializeAArch64TargetInfo();
+  LLVMInitializeAArch64Target();
+  LLVMInitializeAArch64TargetMC();
+  LLVMInitializeAArch64AsmPrinter();
+  Initialized = true;
+}
+
+static CodeGenOptLevel codeGenLevel(StringRef Level) {
+  if (Level == "0")
+    return CodeGenOptLevel::None;
+  if (Level == "1")
+    return CodeGenOptLevel::Less;
+  if (Level == "3")
+    return CodeGenOptLevel::Aggressive;
+  return CodeGenOptLevel::Default;
+}
+
+std::unique_ptr<TargetMachine> createTargetMachine(StringRef TripleName,
+                                                   StringRef OptLevel,
+                                                   std::string &Error) {
+  initializeTargets();
+  Triple TargetTriple(TripleName);
+  const Target *Definition = TargetRegistry::lookupTarget(TargetTriple, Error);
+  if (Definition == nullptr)
+    return nullptr;
+  TargetOptions Options;
+  return std::unique_ptr<TargetMachine>(Definition->createTargetMachine(
+      TargetTriple, "generic", "", Options, Reloc::PIC_, CodeModel::Small,
+      codeGenLevel(OptLevel)));
+}
+
+} // namespace scriptc

native/llvm-codegen/src/target.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "llvm/ADT/StringRef.h"
+
+#include <memory>
+#include <string>
+
+namespace llvm {
+class TargetMachine;
+}
+
+namespace scriptc {
+
+inline constexpr llvm::StringLiteral ProtocolVersion = "1";
+inline constexpr llvm::StringLiteral LlvmVersion = "22.1.8";
+inline constexpr llvm::StringLiteral DefaultTarget =
+    "arm64-apple-macosx14.0.0";
+inline constexpr llvm::StringLiteral DefaultDataLayout =
+    "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32";
+
+void initializeTargets();
+std::unique_ptr<llvm::TargetMachine>
+createTargetMachine(llvm::StringRef Triple, llvm::StringRef OptLevel,
+                    std::string &Error);
+
+} // namespace scriptc