From a263f88db5284208977f07f6aaee477be7c01aed Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 20 Aug 2026 20:36:41 -0500 Subject: [PATCH] fix: harden program shard caching --- .../compiler/src/backend/cc-cache.test.ts | 152 +++++++++++++++++- packages/compiler/src/backend/cc.ts | 120 ++++++++++---- 2 files changed, 238 insertions(+), 34 deletions(-) diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index efbe6bd34..acf5ecdb6 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -64,10 +64,22 @@ const test = Object.assign( const trustInstrumentedCompilerWrapper = (): void => { process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] = "1"; }; -const zigExecutable = (process.env["PATH"] ?? "") - .split(delimiter) - .map((entry) => join(entry === "" ? process.cwd() : entry, "zig")) - .find((candidate) => existsSync(candidate)); +const executableOnPath = (name: string): string | undefined => + (process.env["PATH"] ?? "") + .split(delimiter) + .map((entry) => join(entry === "" ? process.cwd() : entry, name)) + .find((candidate) => existsSync(candidate)); +const zigExecutable = executableOnPath("zig"); +const clangExecutable = executableOnPath("clang"); +const arExecutable = executableOnPath("ar"); +const ldExecutable = executableOnPath("ld"); +const objcopyExecutable = executableOnPath("objcopy"); +const nativeShardMergeAvailable = + process.platform === "darwin" + ? ldExecutable !== undefined + : process.platform === "linux" + ? ldExecutable !== undefined && objcopyExecutable !== undefined + : process.platform === "win32" && process.arch === "x64"; const completeArtifacts = async (root: string, kind: "bin" | "lib"): Promise => (await readdir(join(root, kind))).filter((name) => !name.endsWith(".sha256")); const cacheTreeBytes = async (root: string): Promise => { @@ -3000,7 +3012,7 @@ test("library identity edits reuse the cached large program object", async () => } }); -test.skipIf(process.platform === "win32")( +test.skipIf(!nativeShardMergeAvailable)( "LLVM library shards reuse unaffected program objects after a localized edit", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shards-")); @@ -3086,6 +3098,136 @@ entry: }, ); +test.skipIf(!nativeShardMergeAvailable)( + "LLVM merged-object and archive caches separate shard projections and public-symbol keep sets", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shard-abi-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.ll"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const programSource = [ + "define i64 @foo() {", + "entry:", + " ret i64 1", + "}", + "", + "define i64 @bar() {", + "entry:", + " ret i64 2", + "}", + "", + ].join("\n"); + const programShards = [ + { + name: "program-f000.ll", + source: "define i64 @foo() {\nentry:\n ret i64 1\n}\ndeclare i64 @bar()\n", + }, + { + name: "program-f001.ll", + source: "declare i64 @foo()\ndefine i64 @bar() {\nentry:\n ret i64 2\n}\n", + }, + ]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await mkdir(cacheRoot, { mode: 0o700 }); + await writeFile(cPath, programSource); + const build = async ( + symbol: "foo" | "bar", + outPath: string, + shards = programShards, + ): Promise => { + await compileLibArchive({ + cPath, + programSource, + programShards: shards, + programPublicSymbols: [symbol], + outPath, + cacheIdentity: TEST_CACHE_IDENTITY, + optimization: "dev", + }); + }; + await build("foo", join(dir, "foo.lib.a")); + const barArchive = join(dir, "bar.lib.a"); + await build("bar", barArchive); + + expect(await completeArtifacts(cacheRoot, "lib")).toHaveLength(2); + expect( + (await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256")), + ).toHaveLength(2); + const probeSource = join(dir, "probe.c"); + const probe = join(dir, "probe"); + await writeFile(probeSource, "long long bar(void);\nint main(void) { return bar() != 2; }\n"); + execFileSync("clang", [probeSource, barArchive, "-lm", "-o", probe]); + expect(execFileSync(probe, { encoding: "utf8" })).toBe(""); + + // The canonical TU and ABI stay fixed, but the exact shard projection + // changes. The merged-object tier must not reuse the previous layout. + await build("bar", join(dir, "bar-commented.lib.a"), [ + { ...programShards[0]!, source: `; projection edit\n${programShards[0]!.source}` }, + programShards[1]!, + ]); + expect(await completeArtifacts(cacheRoot, "lib")).toHaveLength(3); + expect( + (await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256")), + ).toHaveLength(3); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } + }, +); + +test.skipIf( + process.platform === "win32" || clangExecutable === undefined || arExecutable === undefined, +)("LLVM library shards fall back to the canonical TU without host merge tools", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shard-fallback-")); + scratch.push(dir); + const binDir = join(dir, "bin"); + const cPath = join(dir, "program.ll"); + const outPath = join(dir, "program.lib.a"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldPath = process.env["PATH"]; + const programSource = "define i64 @public_value() {\nentry:\n ret i64 7\n}\n"; + try { + await mkdir(binDir); + await Promise.all([ + symlink(clangExecutable!, join(binDir, "clang")), + symlink(arExecutable!, join(binDir, "ar")), + ]); + process.env["PATH"] = binDir; + process.env["SCRIPTC_NO_CACHE"] = "1"; + delete process.env["SCRIPTC_CACHE_DIR"]; + await writeFile(cPath, programSource); + await compileLibArchive({ + cPath, + programSource, + // If compilation reaches either shard, this build fails. Missing ld + // (and objcopy on Linux) must select the valid canonical TU instead. + programShards: [ + { name: "program-f000.ll", source: "not valid LLVM IR\n" }, + { name: "program-f001.ll", source: "also not valid LLVM IR\n" }, + ], + programPublicSymbols: ["public_value"], + outPath, + optimization: "dev", + }); + expect(await readFile(outPath)).not.toHaveLength(0); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldPath === undefined) delete process.env["PATH"]; + else process.env["PATH"] = oldPath; + } +}); + test.skipIf(process.platform === "win32" || zigExecutable === undefined)( "cross-ELF localized archives retain unreferenced identity roots", async () => { diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 0b7d13a17..fa1fd1019 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1464,9 +1464,79 @@ export interface LibArchiveOptions { textDecoderLegacy?: boolean; } +function updateProgramShardCacheIdentity( + hash: ReturnType, + shards: readonly { name: string; source: string }[] | undefined, + publicSymbols: readonly string[] | undefined, +): void { + hash.update("\0program-shards\0"); + if (shards === undefined) { + hash.update("\0"); + } else { + hash.update("\0"); + for (const shard of shards) { + hash.update(shard.name).update("\0").update(shard.source).update("\0"); + } + } + hash.update("\0program-public-symbols\0"); + if (publicSymbols === undefined) { + hash.update("\0"); + } else { + hash.update("\0"); + for (const symbol of publicSymbols) hash.update(symbol).update("\0"); + } +} + +/** Whether this target can turn LLVM shard objects back into the canonical + * program member. Sharding is only a build optimization: a missing host merge + * tool or an object class the in-process localizers do not support must retain + * the ordinary single-TU compile instead of making a valid library fail. */ +async function libraryProgramShardMergeAvailable(driver: CcDriver): Promise { + const platform = targetPlatform(driver); + // Mach-O merging uses the host's ld64. A Darwin target produced from a + // non-Darwin host can still compile as one canonical Zig TU, but the host's + // ELF/COFF linker cannot combine those objects. + if (platform === "darwin") { + return process.platform === "darwin" && await resolvedTool("ld") !== null; + } + if (platform === "linux") { + if (driver.target === null) { + const [ld, objcopy] = await Promise.all([resolvedTool("ld"), resolvedTool("objcopy")]); + return ld !== null && objcopy !== null; + } + const arch = driver.target.split("-", 1)[0]; + return arch === "x86_64" || arch === "aarch64"; + } + if (platform === "win32") { + const arch = driver.target?.split("-", 1)[0] ?? process.arch; + return arch === "x86_64" || arch === "x64"; + } + return false; +} + export async function compileLibArchive(opts: LibArchiveOptions): Promise { const rtDir = runtimeSrcDir(); const driver = resolveCc(); + const shardNames = new Set(); + const programShardsValid = opts.programShards?.every((shard) => { + if ( + basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") || + shardNames.has(shard.name) + ) return false; + shardNames.add(shard.name); + return true; + }) === true; + const programShardsRequested = + opts.cPath.endsWith(".ll") && + programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 && + opts.programPublicSymbols !== undefined + ? opts.programShards + : null; + const programShards = + programShardsRequested !== null && await libraryProgramShardMergeAvailable(driver) + ? programShardsRequested + : null; + const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols; const sanitize = opts.sanitize ?? false; const optimization = opts.optimization ?? "release"; const regex = opts.regex ?? false; @@ -1616,10 +1686,11 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise const av = await toolVersionOnce(arArgv, toolchainEnv, true); archiverVersion = av; const key = createHash("sha256") - // v8 adds the optional native program-shard set. The canonical TU - // still keys source semantics; shard bytes key the exact archive - // object layout so single- and multi-TU producers never collide. - .update("lib-v8\0") + // v9 adds the public-symbol keep set beside the optional native + // shards. The canonical TU still keys source semantics; shard and + // keep bytes key the exact merged program object so ABI projections + // and single-/multi-TU producers never collide. + .update("lib-v9\0") .update(cacheTargetIdentity(driver)).update("\0") .update(toolchainEnv).update("\0") .update(implicitToolchain!).update("\0") @@ -1639,11 +1710,12 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise // latter. Archive members also inherit the TU's basename. .update(opts.cPath).update("\0") .update(resolve(opts.cPath)).update("\0") - .update(programBytes) - .update("\0program-shards\0"); - for (const shard of opts.programShards ?? []) { - key.update(shard.name).update("\0").update(shard.source).update("\0"); - } + .update(programBytes); + updateProgramShardCacheIdentity( + key, + programShards ?? undefined, + programPublicSymbols, + ); const keyHex = key .update("\0identity\0") .update(identityBytes === null ? "" : "").update("\0") @@ -1725,21 +1797,6 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise cachedProgramBytes === null ? opts.cPath : join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`); - const shardNames = new Set(); - const programShardsValid = opts.programShards?.every((shard) => { - if ( - basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") || - shardNames.has(shard.name) - ) return false; - shardNames.add(shard.name); - return true; - }) === true; - const programShards = - opts.cPath.endsWith(".ll") && - programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 && - opts.programPublicSymbols !== undefined - ? opts.programShards - : null; if (cachedProgramBytes !== null && programShards === null) { await writeFile(programSource, cachedProgramBytes); } @@ -1749,7 +1806,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise implicitToolchain !== null && programCompilerInvocation !== null ) { const programKey = createHash("sha256") - .update("lib-program-obj-v1\0") + .update("lib-program-obj-v2\0") .update(cacheTargetIdentity(driver)).update("\0") .update(toolchainEnv).update("\0") .update(implicitToolchain).update("\0") @@ -1762,9 +1819,14 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(programCompilerArgs.join("\x1f")).update("\0") .update(opts.cPath).update("\0") .update(resolve(opts.cPath)).update("\0") - .update(cachedProgramBytes) - .digest("hex"); - cachedProgramObject = join(root, "program-obj", programKey); + .update(cachedProgramBytes); + updateProgramShardCacheIdentity( + programKey, + programShards ?? undefined, + programPublicSymbols, + ); + const programKeyHex = programKey.digest("hex"); + cachedProgramObject = join(root, "program-obj", programKeyHex); } const stagedProgramObject = join(buildDir, `${stem}.program.o`); let programObject: string; @@ -1824,7 +1886,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise buildDir, shardEntries.map((entry) => entry.staged), [], - opts.programPublicSymbols!, + programPublicSymbols!, `${stem}.program`, ); // Keep the canonical archive member spelling. The fact that native