From a4ae5f9e27f89e238ca24d6dd91273674a7ba092 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:17:20 -0500 Subject: [PATCH] Run the linker job directly for host Linux links with lld A Linux executable link runs `clang -fuse-ld=lld ...`, and the driver only starts ld.lld with a job it derives from its arguments, its environment and the installed toolchain. Starting the driver costs about 10 ms of a small program's 29 ms link. LinkerJobs (native/linker-job.ts) records that job in the native cache, with the link's own inputs and output as positions. A traced link (the Linux executable cache's `-v -Wl,-t` link) records it from the job line the driver prints with -v; a link the executable cache does not trace lists it with -v -### first. The entry also keeps the identities of the files and directories the job and the driver's -v lines name (ld.lld, startup objects, library directories, the GCC installation, the driver's directory): the set the Linux executable cache records from its traced link. It is keyed by the driver's identity, the link's arguments, the toolchain environment, PATH and the working directory. A later link with the same key whose recorded paths still have those identities runs ld.lld with the job directly, and the executable cache traces it with the job's -t output and the recorded identities. Every other link runs through the driver as before: cross-target links, links without lld, a driver that is not clang, SCRIPTC_LINKER, sanitizer links, a toolchain environment that names mutable inputs, a listing without exactly one ld.lld job naming each of the link's paths once, a recorded path whose identity changed, and a job that fails (the driver then reports the error). The linker gets the same arguments, so the executables are identical. --- packages/compiler/src/native/cache.ts | 1 + packages/compiler/src/native/compiler.ts | 8 +- .../src/native/executable-cache.test.ts | 42 +- .../compiler/src/native/executable-cache.ts | 28 +- .../compiler/src/native/linker-job.test.ts | 408 ++++++++++++++++++ packages/compiler/src/native/linker-job.ts | 286 ++++++++++++ 6 files changed, 760 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/src/native/linker-job.test.ts create mode 100644 packages/compiler/src/native/linker-job.ts diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index e204b8a6b..0250e9618 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -168,6 +168,7 @@ export class NativeCache { "sanitizer", "executable", "kept", + "linker-job", "runtime-pack", "binary", "dsym", diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index ff49e2ac9..0b56b0a53 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -70,6 +70,7 @@ import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; import { lldLinkerArgs } from "../backend/lld.js"; import { openNativeExecutableCache } from "./executable-cache.js"; +import { LinkerJobs } from "./linker-job.js"; import { nativeFileIdentity } from "./file-identity.js"; /** The first partition keeps the whole-object key used before partitioning. */ @@ -530,11 +531,14 @@ export class NativeCompiler { "-o", stagedOutput, ]; - const cacheable = executableCache?.trace(linkArgs, stage) ?? false; + // Host Linux links with lld run the linker job the driver would. + const jobs = LinkerJobs.open(this.cache, toolchain, lld, options.sanitize ?? false); + const job = jobs?.link([...plan.inputs, stagedOutput], stage); + const cacheable = executableCache?.trace(linkArgs, stage, job) ?? false; timing("link-inputs"); // A successful dependency trace already produced this executable. // Keep its verified output instead of asking the linker to repeat it. - if (!cacheable) { + if (!cacheable && (job?.run(linkArgs, true) ?? null) === null) { const driver = options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker; try { runNativeTool(driver, linkArgs); diff --git a/packages/compiler/src/native/executable-cache.test.ts b/packages/compiler/src/native/executable-cache.test.ts index 8bec9916e..4f950e5b5 100644 --- a/packages/compiler/src/native/executable-cache.test.ts +++ b/packages/compiler/src/native/executable-cache.test.ts @@ -15,7 +15,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test, vi } from "vitest"; import { contentDigest, NativeCache } from "./cache.js"; -import { NativeExecutableCache } from "./executable-cache.js"; +import { identity, NativeExecutableCache } from "./executable-cache.js"; vi.mock("node:child_process", async (original) => ({ ...(await original()), @@ -331,3 +331,43 @@ test("a Linux build that keeps its output does not read the deferred inputs", () expect(open().restore(f.destination)).toBe(true); expect(deferred).toHaveBeenCalledTimes(3); }); + +test("a Linux link that runs as its linker job is traced from the job's run, and otherwise records it", () => { + const f = fixture(); + const lib = join(f.root, "lib"); + mkdirSync(lib); + const crt = join(lib, "Scrt1.o"); + writeFileSync(crt, "startup object"); + const program = join(f.stage, "program.o"); + writeFileSync(program, "program"); + const job = { + run: vi.fn((_args: string[], _list: boolean) => ({ + output: [crt, program].join("\n") + "\n", + driverInputs: [identity(crt), identity(lib)], + })), + record: vi.fn(), + }; + const open = () => new NativeExecutableCache(f.cache, f.key, "clang", false, [f.runtime], true); + const entry = open(); + expect(entry.trace(["-o", f.output], f.stage, job)).toBe(true); + // The job runs the traced link, and the driver does not. + expect(job.run.mock.calls[0]).toEqual([["-o", f.output, "-v", "-Wl,-t"], false]); + expect(spawnSync).not.toHaveBeenCalled(); + expect(job.record).not.toHaveBeenCalled(); + entry.recordOutput(f.output); + const recorded = ( + JSON.parse(f.cache.read("kept", f.key)!.toString()) as { inputs: { path: string }[] } + ).inputs.map((saved) => saved.path); + expect(recorded).toEqual(expect.arrayContaining([crt, lib])); + expect(recorded).not.toContain(program); + expect(open().keep(f.output)).toBe(true); + // A link without a job runs through the driver, and records the job from + // the driver's output. + const record = vi.fn(); + expect(open().trace(["-o", f.output], f.stage, { run: () => null, record })).toBe(true); + expect(spawnSync).toHaveBeenCalledTimes(1); + expect(record).toHaveBeenCalledWith(["-o", f.output, "-v", "-Wl,-t"], f.library + "\n\n", [ + identity(f.sdk), + identity(f.library), + ]); +}); diff --git a/packages/compiler/src/native/executable-cache.ts b/packages/compiler/src/native/executable-cache.ts index bafd2c020..a26bc1d87 100644 --- a/packages/compiler/src/native/executable-cache.ts +++ b/packages/compiler/src/native/executable-cache.ts @@ -14,8 +14,9 @@ import type { NativeOptimization } from "../backend/optimization.js"; import type { NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; import type { NativeToolchain } from "./toolchain.js"; import { contentDigest, NativeCache } from "./cache.js"; +import type { LinkerJobLink } from "./linker-job.js"; -interface InputIdentity { +export interface InputIdentity { path: string; canonical: string; kind: "file" | "directory"; @@ -33,7 +34,7 @@ interface ExecutableEntry { symbols: string | null; } -function identity(path: string): InputIdentity { +export function identity(path: string): InputIdentity { path = resolve(path); const info = statSync(path); if (!info.isFile() && !info.isDirectory()) @@ -63,7 +64,7 @@ function sameIdentity(a: InputIdentity, b: InputIdentity): boolean { ); } -function stillMatches(inputs: InputIdentity[]): boolean { +export function stillMatches(inputs: InputIdentity[]): boolean { try { return inputs.every((input) => sameIdentity(identity(input.path), input)); } catch { @@ -71,7 +72,7 @@ function stillMatches(inputs: InputIdentity[]): boolean { } } -function commandPath(command: string): string { +export function commandPath(command: string): string { if (command.includes("/") || command.includes("\\")) return resolve(command); for (const directory of (process.env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) { const candidate = join(directory || process.cwd(), command); @@ -92,7 +93,7 @@ function toolOutput(executable: string, args: string[]): string { return result.stdout + "\n" + result.stderr; } -function tracePaths(output: string, driver: boolean, roots: string[]): string[] { +export function tracePaths(output: string, driver: boolean, roots: string[]): string[] { const paths = new Set(); for (const line of output.split(/\r?\n/)) { for (const candidate of driver ? driverTraceCandidates(line) : linkTraceCandidate(line)) { @@ -134,7 +135,7 @@ interface KeptEntry { inputs: InputIdentity[]; } -function validIdentity(input: InputIdentity): boolean { +export function validIdentity(input: InputIdentity): boolean { return ( input !== null && typeof input === "object" && @@ -290,8 +291,10 @@ export class NativeExecutableCache { * Private build inputs already belong to the LLVM/runtime cache identity. * A true result means the requested output was linked successfully and its * inputs remained stable. The caller can publish that output directly. On - * false, the caller links the output itself. */ - trace(args: string[], stage: string): boolean { + * false, the caller links the output itself. On Linux, `job` runs the + * traced link as its recorded linker job when it can, and otherwise records + * the job the driver ran (see LinkerJobs). */ + trace(args: string[], stage: string, job?: LinkerJobLink): boolean { this.traced = false; try { this.addDeferredInputs(); @@ -300,8 +303,13 @@ export class NativeExecutableCache { ? [] : tracePaths(toolOutput(this.linker, [...args, "-###"]), true, roots).map(identity); // On Linux, -v prints the jobs that -### would, while the link runs. - const linked = toolOutput(this.linker, [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"]); - if (this.linux) driverInputs = tracePaths(linked, true, roots).map(identity); + const traced = [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"]; + const direct = this.linux ? (job?.run(traced, false) ?? null) : null; + const linked = direct?.output ?? toolOutput(this.linker, traced); + if (this.linux) { + driverInputs = direct?.driverInputs ?? tracePaths(linked, true, roots).map(identity); + if (direct === null) job?.record(traced, linked, driverInputs); + } const paths = tracePaths(linked, false, roots); if (paths.length === 0 || !stillMatches(this.inputs) || !stillMatches(driverInputs)) return false; diff --git a/packages/compiler/src/native/linker-job.test.ts b/packages/compiler/src/native/linker-job.test.ts new file mode 100644 index 000000000..b9d718c03 --- /dev/null +++ b/packages/compiler/src/native/linker-job.test.ts @@ -0,0 +1,408 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test } from "vitest"; +import { + LINUX_ARM64_GNU_TARGET, + LINUX_X64_GNU_TARGET, + MACOS_ARM64_TARGET, + type NativeTargetSpec, +} from "../backend/targets.js"; +import { NativeCache } from "./cache.js"; +import { LinkerJobs } from "./linker-job.js"; +import type { NativeToolchain } from "./toolchain.js"; + +const directories: string[] = []; +const saved = new Map(); +const variables = [ + "SCRIPTC_LINKER", + "COMPILER_PATH", + "LOG", + "LD", + "CRT", + "LIB", + "FAIL", + "BROKEN", + "DOUBLE", +]; +beforeEach(() => { + for (const name of variables) { + saved.set(name, process.env[name]); + delete process.env[name]; + } +}); +afterEach(() => { + for (const [name, value] of saved) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + saved.clear(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function script(path: string, text: string): string { + writeFileSync(path, text); + chmodSync(path, 0o755); + return path; +} + +/** A driver named `name` and an ld.lld that log their arguments to $LOG. The + * driver's job runs $LD with `-pie`, the startup object $CRT, `-L$LIB` and + * its own arguments (without `-fuse-ld=lld` and `-v`, `-Wl,` removed). With + * -### it prints that job as clang does, every argument quoted; with -v it + * prints the job it runs as clang does, only the program quoted. With + * $BROKEN, -### prints the job twice; with $DOUBLE, it names the path $DOUBLE + * once more. The linker writes its inputs to its output, prints them with + * `-t`, and fails with $FAIL. */ +function fixture(name = "clang") { + const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-")); + directories.push(root); + const bin = join(root, "bin"); + const lib = join(root, "lib"); + // The recorded inputs include the parents of the toolchain's paths (here + // `root`); stages are elsewhere, as a build's are. + const stages = join(root, "stages"); + for (const directory of [bin, lib, stages]) mkdirSync(directory); + const log = join(root, "log"); + writeFileSync(log, ""); + const driver = script( + join(bin, name), + `#!/bin/sh +echo "driver $*" >> "$LOG" +job=" \\"$LD\\" \\"-pie\\" \\"$CRT\\" \\"-L$LIB\\"" +raw=" \\"$LD\\" -pie $CRT -L$LIB" +listing=0; verbose=0 +for a; do + case "$a" in + -'###') listing=1 ;; + -v) verbose=1 ;; + -fuse-ld=lld) ;; + -Wl,*) job="$job \\"\${a#-Wl,}\\""; raw="$raw \${a#-Wl,}" ;; + *) job="$job \\"$a\\""; raw="$raw $a" ;; + esac +done +if [ $listing = 1 ]; then + echo "InstalledDir: $(dirname "$0")" >&2 + [ -z "$DOUBLE" ] && echo "$job" >&2 + [ -n "$BROKEN" ] && echo "$job" >&2 + [ -n "$DOUBLE" ] && echo "$job \\"$DOUBLE\\"" >&2 && exit 0 + exit 0 +fi +[ $verbose = 1 ] && echo "$raw" >&2 +eval "exec $job" +`, + ); + const ld = script( + join(bin, "ld.lld"), + `#!/bin/sh +echo "ld.lld $*" >> "$LOG" +[ -n "$FAIL" ] && exit 1 +out=""; prev="" +for a; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done +printf '%s\\n' "$@" > "$out" +for a; do [ "$a" = "-t" ] && printf '%s\\n' "$@" | grep '^/'; done +exit 0 +`, + ); + const crt = join(lib, "Scrt1.o"); + writeFileSync(crt, "startup object"); + Object.assign(process.env, { LOG: log, LD: ld, CRT: crt, LIB: lib }); + // A directory's identity is its stat: an old time makes a later change visible. + utimesSync(lib, 0, 0); + const cache = new NativeCache(join(root, "cache")); + const toolchain = (target: NativeTargetSpec = LINUX_X64_GNU_TARGET, helper = target) => + ({ + target, + helper: helper.helper, + linker: driver, + }) as unknown as NativeToolchain; + let count = 0; + /** A stage with a program object, the link's paths and its driver arguments. */ + const stage = () => { + const directory = join(stages, `stage-${++count}`); + mkdirSync(directory); + const program = join(directory, "program.o"); + const output = join(directory, "out"); + writeFileSync(program, "program"); + return { + directory, + output, + paths: [program, output], + args: ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output], + }; + }; + const calls = () => + readFileSync(log, "utf8") + .split("\n") + .filter((line) => line !== ""); + return { root, bin, lib, crt, ld, driver, cache, toolchain, stage, calls }; +} + +test.skipIf(process.platform === "win32")( + "a link records the driver's job once, and later links run ld.lld with their own paths", + () => { + const f = fixture(); + const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(jobs).not.toBeNull(); + const first = f.stage(); + const run = jobs.run(first.args, first.paths, first.directory, true); + expect(run).not.toBeNull(); + // The driver only listed the job; ld.lld linked. + expect(f.calls()).toEqual([ + `driver ${first.args.join(" ")} -v -###`, + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${first.paths[0]} -lm -o ${first.output}`, + ]); + expect(readFileSync(first.output, "utf8")).toContain(first.paths[0]); + const recorded = run!.driverInputs.map((input) => input.path); + expect(recorded).toEqual(expect.arrayContaining([f.ld, f.bin, f.crt, f.lib])); + expect(recorded.some((path) => path.startsWith(first.directory))).toBe(false); + // Another build, with other paths, runs the recorded job without the driver. + const second = f.stage(); + const again = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(again.run(second.args, second.paths, second.directory, true)).not.toBeNull(); + expect(f.calls().slice(2)).toEqual([ + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output}`, + ]); + // Other arguments are another job. + const third = f.stage(); + expect( + again.run([...third.args, "-Wl,-t"], third.paths, third.directory, true)?.output, + ).toContain(third.paths[0]); + expect(f.calls().slice(3)[0]).toBe(`driver ${third.args.join(" ")} -Wl,-t -v -###`); + }, +); + +test.skipIf(process.platform === "win32")( + "a driver link records its job from its -v output, and later links run it", + () => { + const f = fixture(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + const traced = [...first.args, "-v", "-Wl,-t"]; + // Without an entry, a link that is not listed runs through the driver. + const link = open().link(first.paths, first.directory); + expect(link.run(traced, false)).toBeNull(); + expect(f.calls()).toEqual([]); + const linked = spawnSync(f.driver, traced, { encoding: "utf8" }); + expect(linked.status).toBe(0); + link.record(traced, linked.stdout + "\n" + linked.stderr); + const second = f.stage(); + const run = open() + .link(second.paths, second.directory) + .run([...second.args, "-v", "-Wl,-t"], false); + expect(run?.output).toContain(second.paths[0]); + expect(run!.driverInputs.map((input) => input.path)).toEqual( + expect.arrayContaining([f.ld, f.bin, f.crt, f.lib]), + ); + expect(f.calls().slice(2)).toEqual([ + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output} -t`, + ]); + }, +); + +test.skipIf(process.platform === "win32")( + "a changed startup object, library directory or linker sends the link to the driver", + () => { + for (const change of ["crt", "lib", "ld"] as const) { + const f = fixture(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull(); + if (change === "lib") writeFileSync(join(f.lib, "libm.so"), "new library"); + else { + const path = change === "crt" ? f.crt : f.ld; + const before = statSync(path); + writeFileSync(path + ".new", readFileSync(path)); + chmodSync(path + ".new", 0o755); + utimesSync(path + ".new", before.atime, before.mtime); + renameSync(path + ".new", path); + } + const second = f.stage(); + const count = f.calls().length; + expect(open().run(second.args, second.paths, second.directory, true)).toBeNull(); + expect(f.calls().length).toBe(count); + } + }, +); + +test.skipIf(process.platform === "win32")("a changed driver records the job again", () => { + const f = fixture(); + const first = f.stage(); + expect( + LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run( + first.args, + first.paths, + first.directory, + true, + ), + ).not.toBeNull(); + writeFileSync(f.driver, readFileSync(f.driver, "utf8") + "\n"); + const second = f.stage(); + expect( + LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run( + second.args, + second.paths, + second.directory, + true, + ), + ).not.toBeNull(); + expect(f.calls()[2]).toBe(`driver ${second.args.join(" ")} -v -###`); +}); + +test.skipIf(process.platform === "win32")( + "a job that fails sends this and the build's later links to the driver", + () => { + const f = fixture(); + const first = f.stage(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull(); + process.env["FAIL"] = "1"; + const jobs = open(); + const second = f.stage(); + expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull(); + const count = f.calls().length; + expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull(); + expect(f.calls().length).toBe(count); + }, +); + +test.skipIf(process.platform === "win32")( + "a listing without one ld.lld job naming each path once keeps the driver, and is not asked again", + () => { + for (const listing of ["two jobs", "another linker", "a path twice"] as const) { + const f = fixture(); + if (listing === "two jobs") process.env["BROKEN"] = "1"; + if (listing === "another linker") + process.env["LD"] = script(join(f.bin, "ld.bfd"), readFileSync(f.ld, "utf8")); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + if (listing === "a path twice") process.env["DOUBLE"] = first.paths[0]; + expect(open().run(first.args, first.paths, first.directory, true)).toBeNull(); + expect(f.calls()).toEqual([`driver ${first.args.join(" ")} -v -###`]); + // The answer is stored: the next build does not list the job again. + expect(open().run(first.args, first.paths, first.directory, true)).toBeNull(); + expect(f.calls().length).toBe(1); + delete process.env["BROKEN"]; + delete process.env["DOUBLE"]; + } + }, +); + +test.skipIf(process.platform === "win32")( + "arguments that name the stage besides the link's paths keep the driver", + () => { + const f = fixture(); + const first = f.stage(); + const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const script = `-Wl,--version-script=${join(first.directory, "exports")}`; + expect(jobs.run([...first.args, script], first.paths, first.directory, true)).toBeNull(); + expect(f.calls()).toEqual([]); + }, +); + +test.skipIf(process.platform === "win32")( + "only host Linux links through a clang driver with lld run the job", + () => { + const f = fixture(); + const lld = ["-fuse-ld=lld"]; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).not.toBeNull(); + // A cross-target link: the arm64 target with the x64 host's helper. + expect( + LinkerJobs.open( + f.cache, + f.toolchain(LINUX_ARM64_GNU_TARGET, LINUX_X64_GNU_TARGET), + lld, + false, + ), + ).toBeNull(); + expect( + LinkerJobs.open( + f.cache, + f.toolchain(LINUX_X64_GNU_TARGET, LINUX_ARM64_GNU_TARGET), + lld, + false, + ), + ).toBeNull(); + expect(LinkerJobs.open(f.cache, f.toolchain(MACOS_ARM64_TARGET), lld, false)).toBeNull(); + // The driver's default linker, a sanitizer link, no cache. + expect(LinkerJobs.open(f.cache, f.toolchain(), [], false)).toBeNull(); + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, true)).toBeNull(); + expect(LinkerJobs.open(null, f.toolchain(), lld, false)).toBeNull(); + // A linker that is not a clang driver. + const gcc = fixture("gcc"); + expect(LinkerJobs.open(gcc.cache, gcc.toolchain(), lld, false)).toBeNull(); + // An explicit linker, or an environment that names mutable toolchain inputs. + process.env["SCRIPTC_LINKER"] = f.driver; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull(); + delete process.env["SCRIPTC_LINKER"]; + process.env["COMPILER_PATH"] = f.bin; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull(); + }, +); + +const hostTarget = + process.arch === "x64" + ? LINUX_X64_GNU_TARGET + : process.arch === "arm64" + ? LINUX_ARM64_GNU_TARGET + : null; +const hasLld = + process.platform === "linux" && + spawnSync("clang", ["-fuse-ld=lld", "-###", "-x", "c", "/dev/null"], { + encoding: "utf8", + }).stderr?.includes("ld.lld") === true; + +test.skipIf(hostTarget === null || !hasLld)("the job links the executable the driver links", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-clang-")); + directories.push(root); + const stage = join(root, "stage"); + mkdirSync(stage); + const program = join(stage, "program.o"); + const compiled = spawnSync("clang", ["-c", "-x", "c", "-", "-o", program], { + input: '#include \nint main(void) { puts("linked"); return 0; }\n', + }); + expect(compiled.status).toBe(0); + const toolchain = { + target: hostTarget, + helper: hostTarget!.helper, + linker: "clang", + } as unknown as NativeToolchain; + const args = (output: string) => ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output]; + const viaDriver = join(stage, "driver"); + expect(spawnSync("clang", args(viaDriver)).status).toBe(0); + const cache = new NativeCache(join(root, "cache")); + // Recorded from the -v output of a traced link, and listed with -###. + const traced = (output: string) => [...args(output), "-v", "-Wl,-t"]; + const tracedOutput = join(stage, "traced"); + const linked = spawnSync("clang", traced(tracedOutput), { encoding: "utf8" }); + expect(linked.status).toBe(0); + LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)! + .link([program, tracedOutput], stage) + .record(traced(tracedOutput), linked.stdout + "\n" + linked.stderr); + for (const [name, list] of [ + ["first", false], + ["second", true], + ["third", true], + ] as const) { + const output = join(stage, name); + const link = LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)!.link( + [program, output], + stage, + ); + const run = list ? link.run(args(output), true) : link.run(traced(output), false); + expect(run).not.toBeNull(); + expect(readFileSync(output).equals(readFileSync(viaDriver))).toBe(true); + expect(spawnSync(output, { encoding: "utf8" }).stdout).toBe("linked\n"); + } +}); diff --git a/packages/compiler/src/native/linker-job.ts b/packages/compiler/src/native/linker-job.ts new file mode 100644 index 000000000..e1195e4c4 --- /dev/null +++ b/packages/compiler/src/native/linker-job.ts @@ -0,0 +1,286 @@ +import { spawnSync } from "node:child_process"; +import { realpathSync } from "node:fs"; +import { basename, extname, isAbsolute, resolve } from "node:path"; +import { + toolchainEnvironmentCachePolicy, + toolchainEnvironmentFingerprint, +} from "../backend/toolchain-environment.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import { + commandPath, + identity, + type InputIdentity, + stillMatches, + tracePaths, + validIdentity, +} from "./executable-cache.js"; + +/** A recorded linker job. In `job`, a number is the index of one of the link's + * own paths (its inputs and output); `inputs` are the identities of the + * other files and directories the job and the driver's `-v` lines name. */ +interface LinkerJobEntry { + schema: "scriptc.linker-job.v1"; + /** null when the driver's job could not be recorded: the link runs through + * the driver. */ + job: (string | number)[] | null; + inputs: InputIdentity[]; +} + +/** A link that ran as its recorded job: the linker's output, and the + * identities the job depends on besides the link's own paths. */ +export interface LinkerJobRun { + output: string; + driverInputs: InputIdentity[]; +} + +/** One link's use of LinkerJobs, with its own paths and stage. */ +export interface LinkerJobLink { + run(args: string[], list: boolean): LinkerJobRun | null; + record(args: string[], output: string, driverInputs?: InputIdentity[]): void; +} + +const MAX_ENTRY_BYTES = 1024 * 1024; + +function validEntry(value: unknown, paths: number): value is LinkerJobEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + if (entry.schema !== "scriptc.linker-job.v1" || !Array.isArray(entry.inputs)) return false; + if (!entry.inputs.every(validIdentity)) return false; + if (entry.job === null) return true; + if (!Array.isArray(entry.job) || typeof entry.job[0] !== "string") return false; + const used = new Set(); + for (const token of entry.job) { + if (typeof token === "string") continue; + if (!Number.isInteger(token) || token < 0 || token >= paths || used.has(token)) return false; + used.add(token); + } + return used.size === paths; +} + +function underRoots(text: string, roots: string[]): boolean { + return roots.some((root) => text.includes(root)); +} + +/** The arguments of one job line as the driver prints them: with `-###` + * each argument is quoted, with `-v` only the program and the arguments that + * contain a space, `"`, `\` or `$`; a quoted argument escapes `"`, `\` and + * `$` with `\`. Null for any other line. */ +function jobArguments(line: string): string[] | null { + const pattern = /\s*(?:"((?:\\.|[^"\\])*)"|([^\s"\\]+))/y; + const args: string[] = []; + let end = 0; + let match: RegExpExecArray | null; + while ((match = pattern.exec(line)) !== null) { + args.push(match[1] === undefined ? match[2]! : match[1].replace(/\\(.)/g, "$1")); + end = pattern.lastIndex; + } + return args.length === 0 || line.slice(end).trim() !== "" ? null : args; +} + +/** + * Linux links through a clang driver with lld. The driver only starts ld.lld + * with a job it derives from its arguments, its environment, its own + * installation and the system's (startup objects, the GCC installation, + * library directories). The job is recorded once per driver identity, + * arguments and environment, with the identities of the files and + * directories it and the driver's `-v` lines name. A later link whose recorded + * paths still have those identities runs ld.lld with the job directly; every + * other link runs through the driver. + */ +export class LinkerJobs { + /** A job run that failed in this build: later links run through the + * driver, which reports the error. */ + private failed = false; + + private constructor( + private readonly cache: NativeCache, + private readonly driver: string, + private readonly driverIdentity: InputIdentity, + ) {} + + /** Null unless the link is a host Linux link through a clang driver with + * `-fuse-ld=lld` (`lld`, the link's lld arguments), outside the sanitizer, + * with a native cache and a toolchain environment that allows complete + * artifacts. */ + static open( + cache: NativeCache | null, + toolchain: NativeToolchain, + lld: readonly string[], + sanitize: boolean, + ): LinkerJobs | null { + const target = toolchain.target; + if ( + cache === null || + sanitize || + target.platform !== "linux" || + // A cross-target link keeps the driver. + target.llvmTriple !== toolchain.helper.defaultTarget || + !lld.includes("-fuse-ld=lld") || + process.env["SCRIPTC_LINKER"] !== undefined || + !toolchainEnvironmentCachePolicy().completeArtifacts + ) + return null; + try { + const driver = commandPath(toolchain.linker); + if (!/^clang(-\d+)?$/.test(basename(realpathSync(driver)))) return null; + return new LinkerJobs(cache, driver, identity(driver)); + } catch { + return null; + } + } + + /** The link whose own inputs and output are `paths`, in `stage`. */ + link(paths: readonly string[], stage: string): LinkerJobLink { + return { + run: (args, list) => this.run(args, paths, stage, list), + record: (args, output, driverInputs) => this.record(args, paths, stage, output, driverInputs), + }; + } + + /** The entry key of the link `args` with its own `paths`, or null when + * the arguments name the stage besides those paths. */ + private key(args: readonly string[], paths: readonly string[], roots: string[]): string | null { + const index = new Map(paths.map((path, i) => [path, i])); + if (index.size !== paths.length || !paths.every((path) => args.includes(path))) return null; + if (args.some((arg) => !index.has(arg) && underRoots(arg, roots))) return null; + const shape = args.map((arg) => { + const at = index.get(arg); + return at === undefined ? arg : { path: at, extension: extname(arg) }; + }); + return contentDigest( + JSON.stringify({ + schema: 1, + driver: this.driverIdentity, + args: shape, + environment: toolchainEnvironmentFingerprint(), + cwd: process.cwd(), + path: process.env["PATH"] ?? null, + }), + ); + } + + /** + * Runs the link that the driver arguments `args` describe as its recorded + * job. `paths` are the link's own inputs and output among `args`, which may + * differ from build to build; `stage` is the directory they are in. Without + * an entry, `list` records the job first with the driver's `-v -###`; + * otherwise the caller links through the driver and passes its `-v` output + * to record(). Null when the link has to run through the driver: no job is + * recorded, a recorded path changed, or the job failed. + */ + run( + args: readonly string[], + paths: readonly string[], + stage: string, + list: boolean, + ): LinkerJobRun | null { + if (this.failed) return null; + try { + const roots = [resolve(stage), realpathSync(stage)]; + const key = this.key(args, paths, roots); + if (key === null) return null; + let entry = this.read(key, paths.length); + if (entry === null && list) { + const result = spawnSync(this.driver, [...args, "-v", "-###"], { + encoding: "utf8", + stdio: "pipe", + }); + if (result.error !== undefined || result.status !== 0 || result.signal !== null) + return null; + entry = this.store(key, result.stdout + "\n" + result.stderr, paths, roots); + } + if (entry === null || entry.job === null || !stillMatches(entry.inputs)) return null; + const job = entry.job.map((token) => (typeof token === "number" ? paths[token]! : token)); + const result = spawnSync(job[0]!, job.slice(1), { + encoding: "utf8", + stdio: "pipe", + }); + if (result.error !== undefined || result.status !== 0 || result.signal !== null) { + this.failed = true; + return null; + } + return { output: result.stdout + "\n" + result.stderr, driverInputs: entry.inputs }; + } catch { + return null; + } + } + + private read(key: string, paths: number): LinkerJobEntry | null { + const bytes = this.cache.read("linker-job", key, MAX_ENTRY_BYTES); + if (bytes === null) return null; + try { + const entry: unknown = JSON.parse(bytes.toString("utf8")); + return validEntry(entry, paths) ? entry : null; + } catch { + return null; + } + } + + /** Records the job of the link `args` from `output`, the `-v` output of + * the driver's successful link, unless an entry exists. `driverInputs` + * are the identities of the paths `output` names, when the caller has + * them. */ + record( + args: readonly string[], + paths: readonly string[], + stage: string, + output: string, + driverInputs?: InputIdentity[], + ): void { + try { + const roots = [resolve(stage), realpathSync(stage)]; + const key = this.key(args, paths, roots); + if (key !== null && this.read(key, paths.length) === null) + this.store(key, output, paths, roots, driverInputs); + } catch { + /* The next link records it. */ + } + } + + /** Stores the entry for `output`, the driver's `-v` lines and job. */ + private store( + key: string, + output: string, + paths: readonly string[], + roots: string[], + driverInputs?: InputIdentity[], + ): LinkerJobEntry { + const lines = output.split(/\r?\n/).filter((line) => line.startsWith(' "')); + const job = lines.length === 1 ? this.jobOf(jobArguments(lines[0]!), paths, roots) : null; + const entry: LinkerJobEntry = { + schema: "scriptc.linker-job.v1", + job, + inputs: job === null ? [] : (driverInputs ?? tracePaths(output, true, roots).map(identity)), + }; + this.cache.write("linker-job", key, JSON.stringify(entry)); + return entry; + } + + /** `job` with each of `paths` as its index: null unless the job runs + * ld.lld and names each of the paths once, as one whole argument, and names + * nothing else in the stage. */ + private jobOf( + job: string[] | null, + paths: readonly string[], + roots: string[], + ): (string | number)[] | null { + if (job === null || job.length === 0) return null; + if (!isAbsolute(job[0]!) || basename(job[0]!) !== "ld.lld") return null; + const index = new Map(paths.map((path, i) => [path, i])); + const used = new Set(); + const tokens: (string | number)[] = []; + for (const arg of job) { + const at = index.get(arg); + if (at === undefined) { + if (underRoots(arg, roots)) return null; + tokens.push(arg); + } else { + if (used.has(at)) return null; + used.add(at); + tokens.push(at); + } + } + return used.size === paths.length ? tokens : null; + } +}