Run the linker job directly for host Linux links with lld (#2253)

packages/compiler/src/native/cache.ts

@@ -168,6 +168,7 @@ export class NativeCache {
         "sanitizer",
         "executable",
         "kept",
+        "linker-job",
         "runtime-pack",
         "binary",
         "dsym",

packages/compiler/src/native/compiler.ts

@@ -70,6 +70,7 @@ import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js";
 import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js";
 import { lldLinkerArgs } from "../backend/lld.js";
 import { openNativeExecutableCache } from "./executable-cache.js";
+import { LinkerJobs } from "./linker-job.js";
 import { nativeFileIdentity } from "./file-identity.js";
 
 /** The first partition keeps the whole-object key used before partitioning. */
@@ -530,11 +531,14 @@ export class NativeCompiler {
           "-o",
           stagedOutput,
         ];
-        const cacheable = executableCache?.trace(linkArgs, stage) ?? false;
+        // Host Linux links with lld run the linker job the driver would.
+        const jobs = LinkerJobs.open(this.cache, toolchain, lld, options.sanitize ?? false);
+        const job = jobs?.link([...plan.inputs, stagedOutput], stage);
+        const cacheable = executableCache?.trace(linkArgs, stage, job) ?? false;
         timing("link-inputs");
         // A successful dependency trace already produced this executable.
         // Keep its verified output instead of asking the linker to repeat it.
-        if (!cacheable) {
+        if (!cacheable && (job?.run(linkArgs, true) ?? null) === null) {
           const driver = options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker;
           try {
             runNativeTool(driver, linkArgs);

packages/compiler/src/native/executable-cache.test.ts

@@ -15,7 +15,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, expect, test, vi } from "vitest";
 import { contentDigest, NativeCache } from "./cache.js";
-import { NativeExecutableCache } from "./executable-cache.js";
+import { identity, NativeExecutableCache } from "./executable-cache.js";
 
 vi.mock("node:child_process", async (original) => ({
   ...(await original<typeof import("node:child_process")>()),
@@ -331,3 +331,43 @@ test("a Linux build that keeps its output does not read the deferred inputs", ()
   expect(open().restore(f.destination)).toBe(true);
   expect(deferred).toHaveBeenCalledTimes(3);
 });
+
+test("a Linux link that runs as its linker job is traced from the job's run, and otherwise records it", () => {
+  const f = fixture();
+  const lib = join(f.root, "lib");
+  mkdirSync(lib);
+  const crt = join(lib, "Scrt1.o");
+  writeFileSync(crt, "startup object");
+  const program = join(f.stage, "program.o");
+  writeFileSync(program, "program");
+  const job = {
+    run: vi.fn((_args: string[], _list: boolean) => ({
+      output: [crt, program].join("\n") + "\n",
+      driverInputs: [identity(crt), identity(lib)],
+    })),
+    record: vi.fn(),
+  };
+  const open = () => new NativeExecutableCache(f.cache, f.key, "clang", false, [f.runtime], true);
+  const entry = open();
+  expect(entry.trace(["-o", f.output], f.stage, job)).toBe(true);
+  // The job runs the traced link, and the driver does not.
+  expect(job.run.mock.calls[0]).toEqual([["-o", f.output, "-v", "-Wl,-t"], false]);
+  expect(spawnSync).not.toHaveBeenCalled();
+  expect(job.record).not.toHaveBeenCalled();
+  entry.recordOutput(f.output);
+  const recorded = (
+    JSON.parse(f.cache.read("kept", f.key)!.toString()) as { inputs: { path: string }[] }
+  ).inputs.map((saved) => saved.path);
+  expect(recorded).toEqual(expect.arrayContaining([crt, lib]));
+  expect(recorded).not.toContain(program);
+  expect(open().keep(f.output)).toBe(true);
+  // A link without a job runs through the driver, and records the job from
+  // the driver's output.
+  const record = vi.fn();
+  expect(open().trace(["-o", f.output], f.stage, { run: () => null, record })).toBe(true);
+  expect(spawnSync).toHaveBeenCalledTimes(1);
+  expect(record).toHaveBeenCalledWith(["-o", f.output, "-v", "-Wl,-t"], f.library + "\n\n", [
+    identity(f.sdk),
+    identity(f.library),
+  ]);
+});

packages/compiler/src/native/executable-cache.ts

@@ -14,8 +14,9 @@ import type { NativeOptimization } from "../backend/optimization.js";
 import type { NativeRuntimeSelection } from "../backend/runtime-pack-native.js";
 import type { NativeToolchain } from "./toolchain.js";
 import { contentDigest, NativeCache } from "./cache.js";
+import type { LinkerJobLink } from "./linker-job.js";
 
-interface InputIdentity {
+export interface InputIdentity {
   path: string;
   canonical: string;
   kind: "file" | "directory";
@@ -33,7 +34,7 @@ interface ExecutableEntry {
   symbols: string | null;
 }
 
-function identity(path: string): InputIdentity {
+export function identity(path: string): InputIdentity {
   path = resolve(path);
   const info = statSync(path);
   if (!info.isFile() && !info.isDirectory())
@@ -63,7 +64,7 @@ function sameIdentity(a: InputIdentity, b: InputIdentity): boolean {
   );
 }
 
-function stillMatches(inputs: InputIdentity[]): boolean {
+export function stillMatches(inputs: InputIdentity[]): boolean {
   try {
     return inputs.every((input) => sameIdentity(identity(input.path), input));
   } catch {
@@ -71,7 +72,7 @@ function stillMatches(inputs: InputIdentity[]): boolean {
   }
 }
 
-function commandPath(command: string): string {
+export function commandPath(command: string): string {
   if (command.includes("/") || command.includes("\\")) return resolve(command);
   for (const directory of (process.env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) {
     const candidate = join(directory || process.cwd(), command);
@@ -92,7 +93,7 @@ function toolOutput(executable: string, args: string[]): string {
   return result.stdout + "\n" + result.stderr;
 }
 
-function tracePaths(output: string, driver: boolean, roots: string[]): string[] {
+export function tracePaths(output: string, driver: boolean, roots: string[]): string[] {
   const paths = new Set<string>();
   for (const line of output.split(/\r?\n/)) {
     for (const candidate of driver ? driverTraceCandidates(line) : linkTraceCandidate(line)) {
@@ -134,7 +135,7 @@ interface KeptEntry {
   inputs: InputIdentity[];
 }
 
-function validIdentity(input: InputIdentity): boolean {
+export function validIdentity(input: InputIdentity): boolean {
   return (
     input !== null &&
     typeof input === "object" &&
@@ -290,8 +291,10 @@ export class NativeExecutableCache {
    * Private build inputs already belong to the LLVM/runtime cache identity.
    * A true result means the requested output was linked successfully and its
    * inputs remained stable. The caller can publish that output directly. On
-   * false, the caller links the output itself. */
-  trace(args: string[], stage: string): boolean {
+   * false, the caller links the output itself. On Linux, `job` runs the
+   * traced link as its recorded linker job when it can, and otherwise records
+   * the job the driver ran (see LinkerJobs). */
+  trace(args: string[], stage: string, job?: LinkerJobLink): boolean {
     this.traced = false;
     try {
       this.addDeferredInputs();
@@ -300,8 +303,13 @@ export class NativeExecutableCache {
         ? []
         : tracePaths(toolOutput(this.linker, [...args, "-###"]), true, roots).map(identity);
       // On Linux, -v prints the jobs that -### would, while the link runs.
-      const linked = toolOutput(this.linker, [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"]);
-      if (this.linux) driverInputs = tracePaths(linked, true, roots).map(identity);
+      const traced = [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"];
+      const direct = this.linux ? (job?.run(traced, false) ?? null) : null;
+      const linked = direct?.output ?? toolOutput(this.linker, traced);
+      if (this.linux) {
+        driverInputs = direct?.driverInputs ?? tracePaths(linked, true, roots).map(identity);
+        if (direct === null) job?.record(traced, linked, driverInputs);
+      }
       const paths = tracePaths(linked, false, roots);
       if (paths.length === 0 || !stillMatches(this.inputs) || !stillMatches(driverInputs))
         return false;

packages/compiler/src/native/linker-job.test.ts

@@ -0,0 +1,408 @@
+import { spawnSync } from "node:child_process";
+import {
+  chmodSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  utimesSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, expect, test } from "vitest";
+import {
+  LINUX_ARM64_GNU_TARGET,
+  LINUX_X64_GNU_TARGET,
+  MACOS_ARM64_TARGET,
+  type NativeTargetSpec,
+} from "../backend/targets.js";
+import { NativeCache } from "./cache.js";
+import { LinkerJobs } from "./linker-job.js";
+import type { NativeToolchain } from "./toolchain.js";
+
+const directories: string[] = [];
+const saved = new Map<string, string | undefined>();
+const variables = [
+  "SCRIPTC_LINKER",
+  "COMPILER_PATH",
+  "LOG",
+  "LD",
+  "CRT",
+  "LIB",
+  "FAIL",
+  "BROKEN",
+  "DOUBLE",
+];
+beforeEach(() => {
+  for (const name of variables) {
+    saved.set(name, process.env[name]);
+    delete process.env[name];
+  }
+});
+afterEach(() => {
+  for (const [name, value] of saved) {
+    if (value === undefined) delete process.env[name];
+    else process.env[name] = value;
+  }
+  saved.clear();
+  for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true });
+});
+
+function script(path: string, text: string): string {
+  writeFileSync(path, text);
+  chmodSync(path, 0o755);
+  return path;
+}
+
+/** A driver named `name` and an ld.lld that log their arguments to $LOG. The
+ * driver's job runs $LD with `-pie`, the startup object $CRT, `-L$LIB` and
+ * its own arguments (without `-fuse-ld=lld` and `-v`, `-Wl,` removed). With
+ * -### it prints that job as clang does, every argument quoted; with -v it
+ * prints the job it runs as clang does, only the program quoted. With
+ * $BROKEN, -### prints the job twice; with $DOUBLE, it names the path $DOUBLE
+ * once more. The linker writes its inputs to its output, prints them with
+ * `-t`, and fails with $FAIL. */
+function fixture(name = "clang") {
+  const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-"));
+  directories.push(root);
+  const bin = join(root, "bin");
+  const lib = join(root, "lib");
+  // The recorded inputs include the parents of the toolchain's paths (here
+  // `root`); stages are elsewhere, as a build's are.
+  const stages = join(root, "stages");
+  for (const directory of [bin, lib, stages]) mkdirSync(directory);
+  const log = join(root, "log");
+  writeFileSync(log, "");
+  const driver = script(
+    join(bin, name),
+    `#!/bin/sh
+echo "driver $*" >> "$LOG"
+job=" \\"$LD\\" \\"-pie\\" \\"$CRT\\" \\"-L$LIB\\""
+raw=" \\"$LD\\" -pie $CRT -L$LIB"
+listing=0; verbose=0
+for a; do
+  case "$a" in
+    -'###') listing=1 ;;
+    -v) verbose=1 ;;
+    -fuse-ld=lld) ;;
+    -Wl,*) job="$job \\"\${a#-Wl,}\\""; raw="$raw \${a#-Wl,}" ;;
+    *) job="$job \\"$a\\""; raw="$raw $a" ;;
+  esac
+done
+if [ $listing = 1 ]; then
+  echo "InstalledDir: $(dirname "$0")" >&2
+  [ -z "$DOUBLE" ] && echo "$job" >&2
+  [ -n "$BROKEN" ] && echo "$job" >&2
+  [ -n "$DOUBLE" ] && echo "$job \\"$DOUBLE\\"" >&2 && exit 0
+  exit 0
+fi
+[ $verbose = 1 ] && echo "$raw" >&2
+eval "exec $job"
+`,
+  );
+  const ld = script(
+    join(bin, "ld.lld"),
+    `#!/bin/sh
+echo "ld.lld $*" >> "$LOG"
+[ -n "$FAIL" ] && exit 1
+out=""; prev=""
+for a; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done
+printf '%s\\n' "$@" > "$out"
+for a; do [ "$a" = "-t" ] && printf '%s\\n' "$@" | grep '^/'; done
+exit 0
+`,
+  );
+  const crt = join(lib, "Scrt1.o");
+  writeFileSync(crt, "startup object");
+  Object.assign(process.env, { LOG: log, LD: ld, CRT: crt, LIB: lib });
+  // A directory's identity is its stat: an old time makes a later change visible.
+  utimesSync(lib, 0, 0);
+  const cache = new NativeCache(join(root, "cache"));
+  const toolchain = (target: NativeTargetSpec = LINUX_X64_GNU_TARGET, helper = target) =>
+    ({
+      target,
+      helper: helper.helper,
+      linker: driver,
+    }) as unknown as NativeToolchain;
+  let count = 0;
+  /** A stage with a program object, the link's paths and its driver arguments. */
+  const stage = () => {
+    const directory = join(stages, `stage-${++count}`);
+    mkdirSync(directory);
+    const program = join(directory, "program.o");
+    const output = join(directory, "out");
+    writeFileSync(program, "program");
+    return {
+      directory,
+      output,
+      paths: [program, output],
+      args: ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output],
+    };
+  };
+  const calls = () =>
+    readFileSync(log, "utf8")
+      .split("\n")
+      .filter((line) => line !== "");
+  return { root, bin, lib, crt, ld, driver, cache, toolchain, stage, calls };
+}
+
+test.skipIf(process.platform === "win32")(
+  "a link records the driver's job once, and later links run ld.lld with their own paths",
+  () => {
+    const f = fixture();
+    const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+    expect(jobs).not.toBeNull();
+    const first = f.stage();
+    const run = jobs.run(first.args, first.paths, first.directory, true);
+    expect(run).not.toBeNull();
+    // The driver only listed the job; ld.lld linked.
+    expect(f.calls()).toEqual([
+      `driver ${first.args.join(" ")} -v -###`,
+      `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${first.paths[0]} -lm -o ${first.output}`,
+    ]);
+    expect(readFileSync(first.output, "utf8")).toContain(first.paths[0]);
+    const recorded = run!.driverInputs.map((input) => input.path);
+    expect(recorded).toEqual(expect.arrayContaining([f.ld, f.bin, f.crt, f.lib]));
+    expect(recorded.some((path) => path.startsWith(first.directory))).toBe(false);
+    // Another build, with other paths, runs the recorded job without the driver.
+    const second = f.stage();
+    const again = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+    expect(again.run(second.args, second.paths, second.directory, true)).not.toBeNull();
+    expect(f.calls().slice(2)).toEqual([
+      `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output}`,
+    ]);
+    // Other arguments are another job.
+    const third = f.stage();
+    expect(
+      again.run([...third.args, "-Wl,-t"], third.paths, third.directory, true)?.output,
+    ).toContain(third.paths[0]);
+    expect(f.calls().slice(3)[0]).toBe(`driver ${third.args.join(" ")} -Wl,-t -v -###`);
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "a driver link records its job from its -v output, and later links run it",
+  () => {
+    const f = fixture();
+    const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+    const first = f.stage();
+    const traced = [...first.args, "-v", "-Wl,-t"];
+    // Without an entry, a link that is not listed runs through the driver.
+    const link = open().link(first.paths, first.directory);
+    expect(link.run(traced, false)).toBeNull();
+    expect(f.calls()).toEqual([]);
+    const linked = spawnSync(f.driver, traced, { encoding: "utf8" });
+    expect(linked.status).toBe(0);
+    link.record(traced, linked.stdout + "\n" + linked.stderr);
+    const second = f.stage();
+    const run = open()
+      .link(second.paths, second.directory)
+      .run([...second.args, "-v", "-Wl,-t"], false);
+    expect(run?.output).toContain(second.paths[0]);
+    expect(run!.driverInputs.map((input) => input.path)).toEqual(
+      expect.arrayContaining([f.ld, f.bin, f.crt, f.lib]),
+    );
+    expect(f.calls().slice(2)).toEqual([
+      `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output} -t`,
+    ]);
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "a changed startup object, library directory or linker sends the link to the driver",
+  () => {
+    for (const change of ["crt", "lib", "ld"] as const) {
+      const f = fixture();
+      const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+      const first = f.stage();
+      expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull();
+      if (change === "lib") writeFileSync(join(f.lib, "libm.so"), "new library");
+      else {
+        const path = change === "crt" ? f.crt : f.ld;
+        const before = statSync(path);
+        writeFileSync(path + ".new", readFileSync(path));
+        chmodSync(path + ".new", 0o755);
+        utimesSync(path + ".new", before.atime, before.mtime);
+        renameSync(path + ".new", path);
+      }
+      const second = f.stage();
+      const count = f.calls().length;
+      expect(open().run(second.args, second.paths, second.directory, true)).toBeNull();
+      expect(f.calls().length).toBe(count);
+    }
+  },
+);
+
+test.skipIf(process.platform === "win32")("a changed driver records the job again", () => {
+  const f = fixture();
+  const first = f.stage();
+  expect(
+    LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run(
+      first.args,
+      first.paths,
+      first.directory,
+      true,
+    ),
+  ).not.toBeNull();
+  writeFileSync(f.driver, readFileSync(f.driver, "utf8") + "\n");
+  const second = f.stage();
+  expect(
+    LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run(
+      second.args,
+      second.paths,
+      second.directory,
+      true,
+    ),
+  ).not.toBeNull();
+  expect(f.calls()[2]).toBe(`driver ${second.args.join(" ")} -v -###`);
+});
+
+test.skipIf(process.platform === "win32")(
+  "a job that fails sends this and the build's later links to the driver",
+  () => {
+    const f = fixture();
+    const first = f.stage();
+    const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+    expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull();
+    process.env["FAIL"] = "1";
+    const jobs = open();
+    const second = f.stage();
+    expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull();
+    const count = f.calls().length;
+    expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull();
+    expect(f.calls().length).toBe(count);
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "a listing without one ld.lld job naming each path once keeps the driver, and is not asked again",
+  () => {
+    for (const listing of ["two jobs", "another linker", "a path twice"] as const) {
+      const f = fixture();
+      if (listing === "two jobs") process.env["BROKEN"] = "1";
+      if (listing === "another linker")
+        process.env["LD"] = script(join(f.bin, "ld.bfd"), readFileSync(f.ld, "utf8"));
+      const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+      const first = f.stage();
+      if (listing === "a path twice") process.env["DOUBLE"] = first.paths[0];
+      expect(open().run(first.args, first.paths, first.directory, true)).toBeNull();
+      expect(f.calls()).toEqual([`driver ${first.args.join(" ")} -v -###`]);
+      // The answer is stored: the next build does not list the job again.
+      expect(open().run(first.args, first.paths, first.directory, true)).toBeNull();
+      expect(f.calls().length).toBe(1);
+      delete process.env["BROKEN"];
+      delete process.env["DOUBLE"];
+    }
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "arguments that name the stage besides the link's paths keep the driver",
+  () => {
+    const f = fixture();
+    const first = f.stage();
+    const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!;
+    const script = `-Wl,--version-script=${join(first.directory, "exports")}`;
+    expect(jobs.run([...first.args, script], first.paths, first.directory, true)).toBeNull();
+    expect(f.calls()).toEqual([]);
+  },
+);
+
+test.skipIf(process.platform === "win32")(
+  "only host Linux links through a clang driver with lld run the job",
+  () => {
+    const f = fixture();
+    const lld = ["-fuse-ld=lld"];
+    expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).not.toBeNull();
+    // A cross-target link: the arm64 target with the x64 host's helper.
+    expect(
+      LinkerJobs.open(
+        f.cache,
+        f.toolchain(LINUX_ARM64_GNU_TARGET, LINUX_X64_GNU_TARGET),
+        lld,
+        false,
+      ),
+    ).toBeNull();
+    expect(
+      LinkerJobs.open(
+        f.cache,
+        f.toolchain(LINUX_X64_GNU_TARGET, LINUX_ARM64_GNU_TARGET),
+        lld,
+        false,
+      ),
+    ).toBeNull();
+    expect(LinkerJobs.open(f.cache, f.toolchain(MACOS_ARM64_TARGET), lld, false)).toBeNull();
+    // The driver's default linker, a sanitizer link, no cache.
+    expect(LinkerJobs.open(f.cache, f.toolchain(), [], false)).toBeNull();
+    expect(LinkerJobs.open(f.cache, f.toolchain(), lld, true)).toBeNull();
+    expect(LinkerJobs.open(null, f.toolchain(), lld, false)).toBeNull();
+    // A linker that is not a clang driver.
+    const gcc = fixture("gcc");
+    expect(LinkerJobs.open(gcc.cache, gcc.toolchain(), lld, false)).toBeNull();
+    // An explicit linker, or an environment that names mutable toolchain inputs.
+    process.env["SCRIPTC_LINKER"] = f.driver;
+    expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull();
+    delete process.env["SCRIPTC_LINKER"];
+    process.env["COMPILER_PATH"] = f.bin;
+    expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull();
+  },
+);
+
+const hostTarget =
+  process.arch === "x64"
+    ? LINUX_X64_GNU_TARGET
+    : process.arch === "arm64"
+      ? LINUX_ARM64_GNU_TARGET
+      : null;
+const hasLld =
+  process.platform === "linux" &&
+  spawnSync("clang", ["-fuse-ld=lld", "-###", "-x", "c", "/dev/null"], {
+    encoding: "utf8",
+  }).stderr?.includes("ld.lld") === true;
+
+test.skipIf(hostTarget === null || !hasLld)("the job links the executable the driver links", () => {
+  const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-clang-"));
+  directories.push(root);
+  const stage = join(root, "stage");
+  mkdirSync(stage);
+  const program = join(stage, "program.o");
+  const compiled = spawnSync("clang", ["-c", "-x", "c", "-", "-o", program], {
+    input: '#include <stdio.h>\nint main(void) { puts("linked"); return 0; }\n',
+  });
+  expect(compiled.status).toBe(0);
+  const toolchain = {
+    target: hostTarget,
+    helper: hostTarget!.helper,
+    linker: "clang",
+  } as unknown as NativeToolchain;
+  const args = (output: string) => ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output];
+  const viaDriver = join(stage, "driver");
+  expect(spawnSync("clang", args(viaDriver)).status).toBe(0);
+  const cache = new NativeCache(join(root, "cache"));
+  // Recorded from the -v output of a traced link, and listed with -###.
+  const traced = (output: string) => [...args(output), "-v", "-Wl,-t"];
+  const tracedOutput = join(stage, "traced");
+  const linked = spawnSync("clang", traced(tracedOutput), { encoding: "utf8" });
+  expect(linked.status).toBe(0);
+  LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)!
+    .link([program, tracedOutput], stage)
+    .record(traced(tracedOutput), linked.stdout + "\n" + linked.stderr);
+  for (const [name, list] of [
+    ["first", false],
+    ["second", true],
+    ["third", true],
+  ] as const) {
+    const output = join(stage, name);
+    const link = LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)!.link(
+      [program, output],
+      stage,
+    );
+    const run = list ? link.run(args(output), true) : link.run(traced(output), false);
+    expect(run).not.toBeNull();
+    expect(readFileSync(output).equals(readFileSync(viaDriver))).toBe(true);
+    expect(spawnSync(output, { encoding: "utf8" }).stdout).toBe("linked\n");
+  }
+});

packages/compiler/src/native/linker-job.ts

@@ -0,0 +1,286 @@
+import { spawnSync } from "node:child_process";
+import { realpathSync } from "node:fs";
+import { basename, extname, isAbsolute, resolve } from "node:path";
+import {
+  toolchainEnvironmentCachePolicy,
+  toolchainEnvironmentFingerprint,
+} from "../backend/toolchain-environment.js";
+import type { NativeToolchain } from "./toolchain.js";
+import { contentDigest, type NativeCache } from "./cache.js";
+import {
+  commandPath,
+  identity,
+  type InputIdentity,
+  stillMatches,
+  tracePaths,
+  validIdentity,
+} from "./executable-cache.js";
+
+/** A recorded linker job. In `job`, a number is the index of one of the link's
+ * own paths (its inputs and output); `inputs` are the identities of the
+ * other files and directories the job and the driver's `-v` lines name. */
+interface LinkerJobEntry {
+  schema: "scriptc.linker-job.v1";
+  /** null when the driver's job could not be recorded: the link runs through
+   * the driver. */
+  job: (string | number)[] | null;
+  inputs: InputIdentity[];
+}
+
+/** A link that ran as its recorded job: the linker's output, and the
+ * identities the job depends on besides the link's own paths. */
+export interface LinkerJobRun {
+  output: string;
+  driverInputs: InputIdentity[];
+}
+
+/** One link's use of LinkerJobs, with its own paths and stage. */
+export interface LinkerJobLink {
+  run(args: string[], list: boolean): LinkerJobRun | null;
+  record(args: string[], output: string, driverInputs?: InputIdentity[]): void;
+}
+
+const MAX_ENTRY_BYTES = 1024 * 1024;
+
+function validEntry(value: unknown, paths: number): value is LinkerJobEntry {
+  if (value === null || typeof value !== "object") return false;
+  const entry = value as Partial<LinkerJobEntry>;
+  if (entry.schema !== "scriptc.linker-job.v1" || !Array.isArray(entry.inputs)) return false;
+  if (!entry.inputs.every(validIdentity)) return false;
+  if (entry.job === null) return true;
+  if (!Array.isArray(entry.job) || typeof entry.job[0] !== "string") return false;
+  const used = new Set<number>();
+  for (const token of entry.job) {
+    if (typeof token === "string") continue;
+    if (!Number.isInteger(token) || token < 0 || token >= paths || used.has(token)) return false;
+    used.add(token);
+  }
+  return used.size === paths;
+}
+
+function underRoots(text: string, roots: string[]): boolean {
+  return roots.some((root) => text.includes(root));
+}
+
+/** The arguments of one job line as the driver prints them: with `-###`
+ * each argument is quoted, with `-v` only the program and the arguments that
+ * contain a space, `"`, `\` or `$`; a quoted argument escapes `"`, `\` and
+ * `$` with `\`. Null for any other line. */
+function jobArguments(line: string): string[] | null {
+  const pattern = /\s*(?:"((?:\\.|[^"\\])*)"|([^\s"\\]+))/y;
+  const args: string[] = [];
+  let end = 0;
+  let match: RegExpExecArray | null;
+  while ((match = pattern.exec(line)) !== null) {
+    args.push(match[1] === undefined ? match[2]! : match[1].replace(/\\(.)/g, "$1"));
+    end = pattern.lastIndex;
+  }
+  return args.length === 0 || line.slice(end).trim() !== "" ? null : args;
+}
+
+/**
+ * Linux links through a clang driver with lld. The driver only starts ld.lld
+ * with a job it derives from its arguments, its environment, its own
+ * installation and the system's (startup objects, the GCC installation,
+ * library directories). The job is recorded once per driver identity,
+ * arguments and environment, with the identities of the files and
+ * directories it and the driver's `-v` lines name. A later link whose recorded
+ * paths still have those identities runs ld.lld with the job directly; every
+ * other link runs through the driver.
+ */
+export class LinkerJobs {
+  /** A job run that failed in this build: later links run through the
+   * driver, which reports the error. */
+  private failed = false;
+
+  private constructor(
+    private readonly cache: NativeCache,
+    private readonly driver: string,
+    private readonly driverIdentity: InputIdentity,
+  ) {}
+
+  /** Null unless the link is a host Linux link through a clang driver with
+   * `-fuse-ld=lld` (`lld`, the link's lld arguments), outside the sanitizer,
+   * with a native cache and a toolchain environment that allows complete
+   * artifacts. */
+  static open(
+    cache: NativeCache | null,
+    toolchain: NativeToolchain,
+    lld: readonly string[],
+    sanitize: boolean,
+  ): LinkerJobs | null {
+    const target = toolchain.target;
+    if (
+      cache === null ||
+      sanitize ||
+      target.platform !== "linux" ||
+      // A cross-target link keeps the driver.
+      target.llvmTriple !== toolchain.helper.defaultTarget ||
+      !lld.includes("-fuse-ld=lld") ||
+      process.env["SCRIPTC_LINKER"] !== undefined ||
+      !toolchainEnvironmentCachePolicy().completeArtifacts
+    )
+      return null;
+    try {
+      const driver = commandPath(toolchain.linker);
+      if (!/^clang(-\d+)?$/.test(basename(realpathSync(driver)))) return null;
+      return new LinkerJobs(cache, driver, identity(driver));
+    } catch {
+      return null;
+    }
+  }
+
+  /** The link whose own inputs and output are `paths`, in `stage`. */
+  link(paths: readonly string[], stage: string): LinkerJobLink {
+    return {
+      run: (args, list) => this.run(args, paths, stage, list),
+      record: (args, output, driverInputs) => this.record(args, paths, stage, output, driverInputs),
+    };
+  }
+
+  /** The entry key of the link `args` with its own `paths`, or null when
+   * the arguments name the stage besides those paths. */
+  private key(args: readonly string[], paths: readonly string[], roots: string[]): string | null {
+    const index = new Map(paths.map((path, i) => [path, i]));
+    if (index.size !== paths.length || !paths.every((path) => args.includes(path))) return null;
+    if (args.some((arg) => !index.has(arg) && underRoots(arg, roots))) return null;
+    const shape = args.map((arg) => {
+      const at = index.get(arg);
+      return at === undefined ? arg : { path: at, extension: extname(arg) };
+    });
+    return contentDigest(
+      JSON.stringify({
+        schema: 1,
+        driver: this.driverIdentity,
+        args: shape,
+        environment: toolchainEnvironmentFingerprint(),
+        cwd: process.cwd(),
+        path: process.env["PATH"] ?? null,
+      }),
+    );
+  }
+
+  /**
+   * Runs the link that the driver arguments `args` describe as its recorded
+   * job. `paths` are the link's own inputs and output among `args`, which may
+   * differ from build to build; `stage` is the directory they are in. Without
+   * an entry, `list` records the job first with the driver's `-v -###`;
+   * otherwise the caller links through the driver and passes its `-v` output
+   * to record(). Null when the link has to run through the driver: no job is
+   * recorded, a recorded path changed, or the job failed.
+   */
+  run(
+    args: readonly string[],
+    paths: readonly string[],
+    stage: string,
+    list: boolean,
+  ): LinkerJobRun | null {
+    if (this.failed) return null;
+    try {
+      const roots = [resolve(stage), realpathSync(stage)];
+      const key = this.key(args, paths, roots);
+      if (key === null) return null;
+      let entry = this.read(key, paths.length);
+      if (entry === null && list) {
+        const result = spawnSync(this.driver, [...args, "-v", "-###"], {
+          encoding: "utf8",
+          stdio: "pipe",
+        });
+        if (result.error !== undefined || result.status !== 0 || result.signal !== null)
+          return null;
+        entry = this.store(key, result.stdout + "\n" + result.stderr, paths, roots);
+      }
+      if (entry === null || entry.job === null || !stillMatches(entry.inputs)) return null;
+      const job = entry.job.map((token) => (typeof token === "number" ? paths[token]! : token));
+      const result = spawnSync(job[0]!, job.slice(1), {
+        encoding: "utf8",
+        stdio: "pipe",
+      });
+      if (result.error !== undefined || result.status !== 0 || result.signal !== null) {
+        this.failed = true;
+        return null;
+      }
+      return { output: result.stdout + "\n" + result.stderr, driverInputs: entry.inputs };
+    } catch {
+      return null;
+    }
+  }
+
+  private read(key: string, paths: number): LinkerJobEntry | null {
+    const bytes = this.cache.read("linker-job", key, MAX_ENTRY_BYTES);
+    if (bytes === null) return null;
+    try {
+      const entry: unknown = JSON.parse(bytes.toString("utf8"));
+      return validEntry(entry, paths) ? entry : null;
+    } catch {
+      return null;
+    }
+  }
+
+  /** Records the job of the link `args` from `output`, the `-v` output of
+   * the driver's successful link, unless an entry exists. `driverInputs`
+   * are the identities of the paths `output` names, when the caller has
+   * them. */
+  record(
+    args: readonly string[],
+    paths: readonly string[],
+    stage: string,
+    output: string,
+    driverInputs?: InputIdentity[],
+  ): void {
+    try {
+      const roots = [resolve(stage), realpathSync(stage)];
+      const key = this.key(args, paths, roots);
+      if (key !== null && this.read(key, paths.length) === null)
+        this.store(key, output, paths, roots, driverInputs);
+    } catch {
+      /* The next link records it. */
+    }
+  }
+
+  /** Stores the entry for `output`, the driver's `-v` lines and job. */
+  private store(
+    key: string,
+    output: string,
+    paths: readonly string[],
+    roots: string[],
+    driverInputs?: InputIdentity[],
+  ): LinkerJobEntry {
+    const lines = output.split(/\r?\n/).filter((line) => line.startsWith(' "'));
+    const job = lines.length === 1 ? this.jobOf(jobArguments(lines[0]!), paths, roots) : null;
+    const entry: LinkerJobEntry = {
+      schema: "scriptc.linker-job.v1",
+      job,
+      inputs: job === null ? [] : (driverInputs ?? tracePaths(output, true, roots).map(identity)),
+    };
+    this.cache.write("linker-job", key, JSON.stringify(entry));
+    return entry;
+  }
+
+  /** `job` with each of `paths` as its index: null unless the job runs
+   * ld.lld and names each of the paths once, as one whole argument, and names
+   * nothing else in the stage. */
+  private jobOf(
+    job: string[] | null,
+    paths: readonly string[],
+    roots: string[],
+  ): (string | number)[] | null {
+    if (job === null || job.length === 0) return null;
+    if (!isAbsolute(job[0]!) || basename(job[0]!) !== "ld.lld") return null;
+    const index = new Map(paths.map((path, i) => [path, i]));
+    const used = new Set<number>();
+    const tokens: (string | number)[] = [];
+    for (const arg of job) {
+      const at = index.get(arg);
+      if (at === undefined) {
+        if (underRoots(arg, roots)) return null;
+        tokens.push(arg);
+      } else {
+        if (used.has(at)) return null;
+        used.add(at);
+        tokens.push(at);
+      }
+    }
+    return used.size === paths.length ? tokens : null;
+  }
+}