Link Linux executables with lld when the driver finds it

Linux executables linked through the clang driver with its default linker,
GNU ld. lld links the same objects in about half the time.

- Native and Node-hosted Linux exe links pass -fuse-ld=lld when ld.lld is
  beside the driver's executable (where clang looks first) or on PATH, and
  the default driver is in use (no SCRIPTC_LINKER, no sanitizer link).
- A link with lld that fails runs again with the driver's default linker,
  whose error is the one reported when that fails too.

packages/compiler/src/backend/linker.ts

@@ -21,6 +21,7 @@ import {
 import { type NativeArtifactDependency } from "./native/contracts.js";
 import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js";
 import type { NativeLinkPlan } from "./link-plan.js";
+import { lldLinkerArgs } from "./lld.js";
 import {
   createDarwinDebugSymbols,
   installDarwinDebugSymbols,
@@ -240,8 +241,11 @@ export async function linkNativeExecutable(
   try {
     const staged = await stageRuntimePackArtifacts(plan.runtimePack);
     stagedRoot = staged.root;
+    // Linux links with lld when the default driver finds it.
+    const lld = explicitLinker === undefined ? lldLinkerArgs(plan.target.platform, linker) : [];
     const args = [
       ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []),
+      ...lld,
       ...plan.driverFlags,
       ...plan.inputs.map((input) => staged.replacements.get(input) ?? input),
       ...plan.systemLibraries.map((name) => `-l${name}`),
@@ -276,7 +280,15 @@ export async function linkNativeExecutable(
               ...(await snapshotDependencies([...toolchain, ...additionalDependencyPaths])),
             ])
             .catch(() => null);
-    await execFileAsync(linker, args);
+    await execFileAsync(linker, args).catch(async (error: unknown) => {
+      // A failed link with lld runs again with the driver's default linker,
+      // whose error (if any) is the one reported.
+      if (lld.length === 0) throw error;
+      await execFileAsync(
+        linker,
+        args.filter((arg) => !lld.includes(arg)),
+      );
+    });
     const output = await stat(privateOut);
     if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable");
     if (plan.darwinDebugSymbols) {

packages/compiler/src/backend/lld.test.ts

@@ -0,0 +1,61 @@
+import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+import { driverFindsLld, lldLinkerArgs } from "./lld.js";
+
+const directories: string[] = [];
+afterEach(() => {
+  for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true });
+});
+
+function tool(path: string): string {
+  writeFileSync(path, "#!/bin/sh\n");
+  chmodSync(path, 0o755);
+  return path;
+}
+
+function fixture() {
+  const root = mkdtempSync(join(tmpdir(), "scriptc-lld-"));
+  directories.push(root);
+  const llvm = join(root, "llvm", "bin");
+  const usr = join(root, "usr", "bin");
+  const other = join(root, "other");
+  for (const directory of [llvm, usr, other]) mkdirSync(directory, { recursive: true });
+  return { root, llvm, usr, other };
+}
+
+test.skipIf(process.platform === "win32")(
+  "a driver finds ld.lld beside itself, beside its real executable, or on PATH",
+  () => {
+    const f = fixture();
+    const clang = tool(join(f.llvm, "clang"));
+    // Without ld.lld anywhere.
+    expect(driverFindsLld(clang, { PATH: f.other })).toBe(false);
+    // Beside the driver.
+    tool(join(f.llvm, "ld.lld"));
+    expect(driverFindsLld(clang, { PATH: f.other })).toBe(true);
+    // A PATH entry that links to the driver: clang looks beside its real executable.
+    symlinkSync(clang, join(f.usr, "clang"));
+    expect(driverFindsLld("clang", { PATH: f.usr })).toBe(true);
+    // A driver elsewhere finds ld.lld on PATH, and not otherwise.
+    const gcc = tool(join(f.other, "gcc"));
+    expect(driverFindsLld(gcc, { PATH: f.other })).toBe(false);
+    expect(driverFindsLld(gcc, { PATH: [f.other, f.llvm].join(":") })).toBe(true);
+    expect(driverFindsLld("missing-driver", { PATH: f.llvm })).toBe(false);
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "Linux links with the default driver use lld when the driver finds it",
+  () => {
+    const f = fixture();
+    const clang = tool(join(f.llvm, "clang"));
+    tool(join(f.llvm, "ld.lld"));
+    const env = { PATH: f.other };
+    expect(lldLinkerArgs("linux", clang, env)).toEqual(["-fuse-ld=lld"]);
+    expect(lldLinkerArgs("darwin", clang, env)).toEqual([]);
+    expect(lldLinkerArgs("windows", clang, env)).toEqual([]);
+    expect(lldLinkerArgs("linux", clang, { ...env, SCRIPTC_LINKER: clang })).toEqual([]);
+  },
+);

packages/compiler/src/backend/lld.ts

@@ -0,0 +1,49 @@
+import { realpathSync, statSync } from "node:fs";
+import { delimiter, dirname, isAbsolute, join, resolve } from "node:path";
+
+function isFile(path: string): boolean {
+  try {
+    return statSync(path).isFile();
+  } catch {
+    return false;
+  }
+}
+
+/** The executable `command` names: a path, or the first match on PATH. */
+function commandFile(command: string, env: NodeJS.ProcessEnv): string | null {
+  if (command.includes("/")) return isFile(command) ? resolve(command) : null;
+  for (const directory of (env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) {
+    const candidate = join(directory || ".", command);
+    if (isFile(candidate)) return isAbsolute(candidate) ? candidate : resolve(candidate);
+  }
+  return null;
+}
+
+/**
+ * Whether the compiler driver `driver` finds ld.lld for `-fuse-ld=lld`: in
+ * the directory of the driver's executable (where clang looks first) or on
+ * PATH. Linking still falls back to the driver's default linker when a link
+ * with lld fails.
+ */
+export function driverFindsLld(driver: string, env: NodeJS.ProcessEnv = process.env): boolean {
+  const file = commandFile(driver, env);
+  if (file === null) return false;
+  try {
+    for (const directory of new Set([dirname(file), dirname(realpathSync(file))]))
+      if (isFile(join(directory, "ld.lld"))) return true;
+  } catch {
+    return false;
+  }
+  return commandFile("ld.lld", env) !== null;
+}
+
+/** `-fuse-ld=lld` for Linux links whose driver finds ld.lld. */
+export function lldLinkerArgs(
+  platform: string,
+  driver: string,
+  env: NodeJS.ProcessEnv = process.env,
+): string[] {
+  return platform === "linux" && env["SCRIPTC_LINKER"] === undefined && driverFindsLld(driver, env)
+    ? ["-fuse-ld=lld"]
+    : [];
+}

packages/compiler/src/native/compiler.test.ts

@@ -1,5 +1,13 @@
 import { execFileSync, spawnSync } from "node:child_process";
-import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import {
+  chmodSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import { afterEach, expect, test, vi } from "vitest";
@@ -7,7 +15,7 @@ import { NativeCompiler } from "./compiler.js";
 import type { NativeToolchain } from "./toolchain.js";
 import { runNativeFrontend } from "../frontend/pipeline-native.js";
 import { emitNativeObject } from "../backend/native-tools.js";
-import { MACOS_ARM64_TARGET } from "../backend/targets.js";
+import { LINUX_X64_GNU_TARGET, MACOS_ARM64_TARGET } from "../backend/targets.js";
 import { VOID, type IrModule } from "../ir/ir.js";
 import { contentDigest, NativeCache } from "./cache.js";
 import { NativeExecutableCache, openNativeExecutableCache } from "./executable-cache.js";
@@ -283,3 +291,90 @@ test.each([false, true])(
     expect(readdirSync(directory).sort()).toEqual(["output.o", "output.o.ll"]);
   },
 );
+
+test.skipIf(process.platform === "win32").each([false, true])(
+  "Linux links use lld when the driver finds it, and the default linker when lld fails (%s)",
+  (lldFails) => {
+    const directory = mkdtempSync(join("/tmp", "scriptc-native-lld-"));
+    directories.push(directory);
+    const bin = join(directory, "bin");
+    mkdirSync(bin);
+    const clang = join(bin, "clang");
+    writeFileSync(clang, "#!/bin/sh\n");
+    chmodSync(clang, 0o755);
+    writeFileSync(join(bin, "ld.lld"), "#!/bin/sh\n");
+    chmodSync(join(bin, "ld.lld"), 0o755);
+    const entry = join(directory, "entry.ts");
+    const output = join(directory, "program");
+    const module: IrModule = {
+      irVersion: 17,
+      sourceFile: entry,
+      entry: "%main",
+      functions: [
+        {
+          name: "%main",
+          params: [],
+          locals: [],
+          body: [],
+          returnType: VOID,
+          loc: { file: entry, start: 0, end: 0 },
+        },
+      ],
+    };
+    vi.mocked(runNativeFrontend).mockReturnValue({
+      preflight: [],
+      entryText: () => "",
+      entryExports: () => new Map(),
+      entryContract: () => {
+        throw new Error("unexpected contract request");
+      },
+      sourceTexts: () => new Map(),
+      npmStatic: [],
+      npmImportSites: new Map(),
+      dispose: () => {},
+      lower: () => ({
+        module,
+        diagnostics: [],
+        runtimeFences: [],
+        stats: { statementsTotal: 0, statementsFailed: 0, functionsSkipped: 0 },
+      }),
+    });
+    vi.mocked(emitNativeObject).mockImplementation((options) => {
+      writeFileSync(options.outputPath, "object");
+      return [options.outputPath];
+    });
+    vi.mocked(openNativeExecutableCache).mockReturnValue(null);
+    const links: string[][] = [];
+    vi.mocked(execFileSync).mockImplementation((_command, args) => {
+      const list = args as string[];
+      links.push(list);
+      if (lldFails && list.includes("-fuse-ld=lld")) throw new Error("lld failed");
+      writeFileSync(list[list.indexOf("-o") + 1]!, "executable");
+      return "";
+    });
+    const target = LINUX_X64_GNU_TARGET;
+    const compiler = new NativeCompiler({
+      compilerVersion: "0.0.0",
+      target,
+      helper: target.helper,
+      ts7Executable: "unused",
+      helperExecutable: "unused",
+      helperPackageRoot: "unused",
+      runtimePackRoot: "unused",
+      linker: clang,
+      linkerArgs: [],
+      dsymutil: "unused",
+    });
+    const result = compiler.compile(entry, {
+      outDir: directory,
+      outPath: output,
+      backend: "llvm",
+      optimization: "release",
+      strip: true,
+    });
+    expect(result).toMatchObject({ ok: true, binaryPath: output });
+    expect(links[0]).toContain("-fuse-ld=lld");
+    expect(links.length).toBe(lldFails ? 2 : 1);
+    if (lldFails) expect(links[1]).not.toContain("-fuse-ld=lld");
+  },
+);

packages/compiler/src/native/compiler.ts

@@ -65,6 +65,7 @@ import {
 import { splitLlvmLibraryProgram } from "../backend/llvm/split.js";
 import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js";
 import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js";
+import { lldLinkerArgs } from "../backend/lld.js";
 import { openNativeExecutableCache } from "./executable-cache.js";
 import { nativeFileIdentity } from "./file-identity.js";
 
@@ -494,8 +495,11 @@ export class NativeCompiler {
             : { windowsSubsystem: options.windowsSubsystem }),
           ...(profile?.generate === undefined ? {} : { profileGenerate: true }),
         });
+        // Linux links with lld when the driver finds it.
+        const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker);
         const linkArgs = [
           ...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs),
+          ...lld,
           ...plan.driverFlags,
           ...plan.inputs,
           ...plan.systemLibraries.map((name) => `-l${name}`),
@@ -506,8 +510,20 @@ export class NativeCompiler {
         timing("link-inputs");
         // A successful dependency trace already produced this executable.
         // Keep its verified output instead of asking the linker to repeat it.
-        if (!cacheable)
-          runNativeTool(options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker, linkArgs);
+        if (!cacheable) {
+          const driver = options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker;
+          try {
+            runNativeTool(driver, linkArgs);
+          } catch (error) {
+            // A failed link with lld runs again with the driver's default
+            // linker, whose error (if any) is the one reported.
+            if (lld.length === 0) throw error;
+            runNativeTool(
+              driver,
+              linkArgs.filter((arg) => !lld.includes(arg)),
+            );
+          }
+        }
         timing("native-link");
         requireNativeArtifact(stagedOutput);
         if (target.platform === "darwin" && optimization === "dev" && !options.strip) {