fix: harden program shard caching

2026-10-26 06:34:20 +08:00 · 2026-08-20 20:36:41 -05:00
parent 9b9ca95b68
commit a263f88db5
2 changed files with 238 additions and 34 deletions
@@ -64,10 +64,22 @@ const test = Object.assign(
 const trustInstrumentedCompilerWrapper = (): void => {
  process.env["SCRIPTC_TEST_TRUST_COMPILER_WRAPPER"] = "1";
 };
-const zigExecutable = (process.env["PATH"] ?? "")
-  .split(delimiter)
-  .map((entry) => join(entry === "" ? process.cwd() : entry, "zig"))
-  .find((candidate) => existsSync(candidate));
+const executableOnPath = (name: string): string | undefined =>
+  (process.env["PATH"] ?? "")
+    .split(delimiter)
+    .map((entry) => join(entry === "" ? process.cwd() : entry, name))
+    .find((candidate) => existsSync(candidate));
+const zigExecutable = executableOnPath("zig");
+const clangExecutable = executableOnPath("clang");
+const arExecutable = executableOnPath("ar");
+const ldExecutable = executableOnPath("ld");
+const objcopyExecutable = executableOnPath("objcopy");
+const nativeShardMergeAvailable =
+  process.platform === "darwin"
+    ? ldExecutable !== undefined
+    : process.platform === "linux"
+      ? ldExecutable !== undefined && objcopyExecutable !== undefined
+      : process.platform === "win32" && process.arch === "x64";
 const completeArtifacts = async (root: string, kind: "bin" | "lib"): Promise<string[]> =>
  (await readdir(join(root, kind))).filter((name) => !name.endsWith(".sha256"));
 const cacheTreeBytes = async (root: string): Promise<number> => {
@@ -3000,7 +3012,7 @@ test("library identity edits reuse the cached large program object", async () =>
  }
 });

-test.skipIf(process.platform === "win32")(
+test.skipIf(!nativeShardMergeAvailable)(
  "LLVM library shards reuse unaffected program objects after a localized edit",
  async () => {
    const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shards-"));
@@ -3086,6 +3098,136 @@ entry:
  },
 );

+test.skipIf(!nativeShardMergeAvailable)(
+  "LLVM merged-object and archive caches separate shard projections and public-symbol keep sets",
+  async () => {
+    const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shard-abi-"));
+    scratch.push(dir);
+    const cacheRoot = join(dir, "cache");
+    const cPath = join(dir, "program.ll");
+    const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
+    const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
+    const programSource = [
+      "define i64 @foo() {",
+      "entry:",
+      "  ret i64 1",
+      "}",
+      "",
+      "define i64 @bar() {",
+      "entry:",
+      "  ret i64 2",
+      "}",
+      "",
+    ].join("\n");
+    const programShards = [
+      {
+        name: "program-f000.ll",
+        source: "define i64 @foo() {\nentry:\n  ret i64 1\n}\ndeclare i64 @bar()\n",
+      },
+      {
+        name: "program-f001.ll",
+        source: "declare i64 @foo()\ndefine i64 @bar() {\nentry:\n  ret i64 2\n}\n",
+      },
+    ];
+    try {
+      process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
+      delete process.env["SCRIPTC_NO_CACHE"];
+      await mkdir(cacheRoot, { mode: 0o700 });
+      await writeFile(cPath, programSource);
+      const build = async (
+        symbol: "foo" | "bar",
+        outPath: string,
+        shards = programShards,
+      ): Promise<void> => {
+        await compileLibArchive({
+          cPath,
+          programSource,
+          programShards: shards,
+          programPublicSymbols: [symbol],
+          outPath,
+          cacheIdentity: TEST_CACHE_IDENTITY,
+          optimization: "dev",
+        });
+      };
+      await build("foo", join(dir, "foo.lib.a"));
+      const barArchive = join(dir, "bar.lib.a");
+      await build("bar", barArchive);
+
+      expect(await completeArtifacts(cacheRoot, "lib")).toHaveLength(2);
+      expect(
+        (await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256")),
+      ).toHaveLength(2);
+      const probeSource = join(dir, "probe.c");
+      const probe = join(dir, "probe");
+      await writeFile(probeSource, "long long bar(void);\nint main(void) { return bar() != 2; }\n");
+      execFileSync("clang", [probeSource, barArchive, "-lm", "-o", probe]);
+      expect(execFileSync(probe, { encoding: "utf8" })).toBe("");
+
+      // The canonical TU and ABI stay fixed, but the exact shard projection
+      // changes. The merged-object tier must not reuse the previous layout.
+      await build("bar", join(dir, "bar-commented.lib.a"), [
+        { ...programShards[0]!, source: `; projection edit\n${programShards[0]!.source}` },
+        programShards[1]!,
+      ]);
+      expect(await completeArtifacts(cacheRoot, "lib")).toHaveLength(3);
+      expect(
+        (await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256")),
+      ).toHaveLength(3);
+    } finally {
+      if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
+      else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
+      if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
+      else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
+    }
+  },
+);
+
+test.skipIf(
+  process.platform === "win32" || clangExecutable === undefined || arExecutable === undefined,
+)("LLVM library shards fall back to the canonical TU without host merge tools", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-shard-fallback-"));
+  scratch.push(dir);
+  const binDir = join(dir, "bin");
+  const cPath = join(dir, "program.ll");
+  const outPath = join(dir, "program.lib.a");
+  const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
+  const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
+  const oldPath = process.env["PATH"];
+  const programSource = "define i64 @public_value() {\nentry:\n  ret i64 7\n}\n";
+  try {
+    await mkdir(binDir);
+    await Promise.all([
+      symlink(clangExecutable!, join(binDir, "clang")),
+      symlink(arExecutable!, join(binDir, "ar")),
+    ]);
+    process.env["PATH"] = binDir;
+    process.env["SCRIPTC_NO_CACHE"] = "1";
+    delete process.env["SCRIPTC_CACHE_DIR"];
+    await writeFile(cPath, programSource);
+    await compileLibArchive({
+      cPath,
+      programSource,
+      // If compilation reaches either shard, this build fails. Missing ld
+      // (and objcopy on Linux) must select the valid canonical TU instead.
+      programShards: [
+        { name: "program-f000.ll", source: "not valid LLVM IR\n" },
+        { name: "program-f001.ll", source: "also not valid LLVM IR\n" },
+      ],
+      programPublicSymbols: ["public_value"],
+      outPath,
+      optimization: "dev",
+    });
+    expect(await readFile(outPath)).not.toHaveLength(0);
+  } finally {
+    if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
+    else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
+    if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
+    else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
+    if (oldPath === undefined) delete process.env["PATH"];
+    else process.env["PATH"] = oldPath;
+  }
+});
+
 test.skipIf(process.platform === "win32" || zigExecutable === undefined)(
  "cross-ELF localized archives retain unreferenced identity roots",
  async () => {
@@ -1464,9 +1464,79 @@ export interface LibArchiveOptions {
  textDecoderLegacy?: boolean;
 }

+function updateProgramShardCacheIdentity(
+  hash: ReturnType<typeof createHash>,
+  shards: readonly { name: string; source: string }[] | undefined,
+  publicSymbols: readonly string[] | undefined,
+): void {
+  hash.update("\0program-shards\0");
+  if (shards === undefined) {
+    hash.update("<none>\0");
+  } else {
+    hash.update("<present>\0");
+    for (const shard of shards) {
+      hash.update(shard.name).update("\0").update(shard.source).update("\0");
+    }
+  }
+  hash.update("\0program-public-symbols\0");
+  if (publicSymbols === undefined) {
+    hash.update("<none>\0");
+  } else {
+    hash.update("<present>\0");
+    for (const symbol of publicSymbols) hash.update(symbol).update("\0");
+  }
+}
+
+/** Whether this target can turn LLVM shard objects back into the canonical
+ * program member. Sharding is only a build optimization: a missing host merge
+ * tool or an object class the in-process localizers do not support must retain
+ * the ordinary single-TU compile instead of making a valid library fail. */
+async function libraryProgramShardMergeAvailable(driver: CcDriver): Promise<boolean> {
+  const platform = targetPlatform(driver);
+  // Mach-O merging uses the host's ld64. A Darwin target produced from a
+  // non-Darwin host can still compile as one canonical Zig TU, but the host's
+  // ELF/COFF linker cannot combine those objects.
+  if (platform === "darwin") {
+    return process.platform === "darwin" && await resolvedTool("ld") !== null;
+  }
+  if (platform === "linux") {
+    if (driver.target === null) {
+      const [ld, objcopy] = await Promise.all([resolvedTool("ld"), resolvedTool("objcopy")]);
+      return ld !== null && objcopy !== null;
+    }
+    const arch = driver.target.split("-", 1)[0];
+    return arch === "x86_64" || arch === "aarch64";
+  }
+  if (platform === "win32") {
+    const arch = driver.target?.split("-", 1)[0] ?? process.arch;
+    return arch === "x86_64" || arch === "x64";
+  }
+  return false;
+}
+
 export async function compileLibArchive(opts: LibArchiveOptions): Promise<void> {
  const rtDir = runtimeSrcDir();
  const driver = resolveCc();
+  const shardNames = new Set<string>();
+  const programShardsValid = opts.programShards?.every((shard) => {
+    if (
+      basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
+      shardNames.has(shard.name)
+    ) return false;
+    shardNames.add(shard.name);
+    return true;
+  }) === true;
+  const programShardsRequested =
+    opts.cPath.endsWith(".ll") &&
+    programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
+    opts.programPublicSymbols !== undefined
+      ? opts.programShards
+      : null;
+  const programShards =
+    programShardsRequested !== null && await libraryProgramShardMergeAvailable(driver)
+      ? programShardsRequested
+      : null;
+  const programPublicSymbols = programShards === null ? undefined : opts.programPublicSymbols;
  const sanitize = opts.sanitize ?? false;
  const optimization = opts.optimization ?? "release";
  const regex = opts.regex ?? false;
@@ -1616,10 +1686,11 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
        const av = await toolVersionOnce(arArgv, toolchainEnv, true);
        archiverVersion = av;
        const key = createHash("sha256")
-          // v8 adds the optional native program-shard set. The canonical TU
-          // still keys source semantics; shard bytes key the exact archive
-          // object layout so single- and multi-TU producers never collide.
-          .update("lib-v8\0")
+          // v9 adds the public-symbol keep set beside the optional native
+          // shards. The canonical TU still keys source semantics; shard and
+          // keep bytes key the exact merged program object so ABI projections
+          // and single-/multi-TU producers never collide.
+          .update("lib-v9\0")
          .update(cacheTargetIdentity(driver)).update("\0")
          .update(toolchainEnv).update("\0")
          .update(implicitToolchain!).update("\0")
@@ -1639,11 +1710,12 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
          // latter. Archive members also inherit the TU's basename.
          .update(opts.cPath).update("\0")
          .update(resolve(opts.cPath)).update("\0")
-          .update(programBytes)
-          .update("\0program-shards\0");
-        for (const shard of opts.programShards ?? []) {
-          key.update(shard.name).update("\0").update(shard.source).update("\0");
-        }
+          .update(programBytes);
+        updateProgramShardCacheIdentity(
+          key,
+          programShards ?? undefined,
+          programPublicSymbols,
+        );
        const keyHex = key
          .update("\0identity\0")
          .update(identityBytes === null ? "<none>" : "<generated>").update("\0")
@@ -1725,21 +1797,6 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
        cachedProgramBytes === null
          ? opts.cPath
          : join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`);
-      const shardNames = new Set<string>();
-      const programShardsValid = opts.programShards?.every((shard) => {
-        if (
-          basename(shard.name) !== shard.name || !shard.name.endsWith(".ll") ||
-          shardNames.has(shard.name)
-        ) return false;
-        shardNames.add(shard.name);
-        return true;
-      }) === true;
-      const programShards =
-        opts.cPath.endsWith(".ll") &&
-        programShardsValid && opts.programShards !== undefined && opts.programShards.length > 1 &&
-        opts.programPublicSymbols !== undefined
-          ? opts.programShards
-          : null;
      if (cachedProgramBytes !== null && programShards === null) {
        await writeFile(programSource, cachedProgramBytes);
      }
@@ -1749,7 +1806,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
        implicitToolchain !== null && programCompilerInvocation !== null
      ) {
        const programKey = createHash("sha256")
-          .update("lib-program-obj-v1\0")
+          .update("lib-program-obj-v2\0")
          .update(cacheTargetIdentity(driver)).update("\0")
          .update(toolchainEnv).update("\0")
          .update(implicitToolchain).update("\0")
@@ -1762,9 +1819,14 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
          .update(programCompilerArgs.join("\x1f")).update("\0")
          .update(opts.cPath).update("\0")
          .update(resolve(opts.cPath)).update("\0")
-          .update(cachedProgramBytes)
-          .digest("hex");
-        cachedProgramObject = join(root, "program-obj", programKey);
+          .update(cachedProgramBytes);
+        updateProgramShardCacheIdentity(
+          programKey,
+          programShards ?? undefined,
+          programPublicSymbols,
+        );
+        const programKeyHex = programKey.digest("hex");
+        cachedProgramObject = join(root, "program-obj", programKeyHex);
      }
      const stagedProgramObject = join(buildDir, `${stem}.program.o`);
      let programObject: string;
@@ -1824,7 +1886,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
          buildDir,
          shardEntries.map((entry) => entry.staged),
          [],
-          opts.programPublicSymbols!,
+          programPublicSymbols!,
          `${stem}.program`,
        );
        // Keep the canonical archive member spelling. The fact that native