Leave out a header the compiler rejects in the implicit toolchain probe

The implicit toolchain probe preprocesses every system header that the
runtime and vendored sources spell in one source file. When the compiler
rejected one of them, the whole probe failed, and the build continued
without the native cache and without program sharding.

When the compiler rejects the probe source, each header is now
preprocessed on its own, and a header that the compiler rejects on its
own is left out of the dependency set, as an absent header is. The probe
still fails when the compiler rejects every header.

packages/compiler/src/backend/native-toolchain.test.ts

@@ -39,6 +39,8 @@ import {
   warmNativeCaches,
 } from "./native-toolchain.js";
 import { splitLlvmProgram } from "./llvm/split.js";
+import { implicitToolchainFingerprints } from "./native/compiler-fingerprint.js";
+import { fingerprintDependencyPaths } from "./native/dependency-files.js";
 
 const scratch: string[] = [];
 const TEST_CACHE_IDENTITY = "cc-cache-tests-v1";
@@ -672,6 +674,29 @@ test.skipIf(clangExecutable === undefined)(
   },
 );
 
+test.skipIf(clangExecutable === undefined)(
+  "the implicit toolchain probe leaves out a header the compiler rejects",
+  async () => {
+    // A system header that the runtime spells, shadowed by one that is an
+    // #error, as clang 18's <riscv_vector.h> is outside RISC-V. The probe
+    // still covers the other headers.
+    const dir = await mkdtemp(join(tmpdir(), "scriptc-probe-header-"));
+    scratch.push(dir);
+    await mkdir(join(dir, "sys"));
+    const rejected = join(dir, "sys", "auxv.h");
+    await writeFile(rejected, '#error "not for this target"\n');
+    const includes = await implicitDependencyProbeIncludes(runtimeSrcDir());
+    expect(includes).toContain("<sys/auxv.h>");
+    const fingerprints = await implicitToolchainFingerprints(
+      { argv: [clangExecutable!, "-isystem", dir], targetArgs: [], target: null },
+      "probe-rejected-header",
+    );
+    const dependencies = fingerprintDependencyPaths(fingerprints.complete);
+    expect(dependencies).not.toContain(rejected);
+    expect(dependencies.some((path) => path.endsWith("/stdatomic.h"))).toBe(true);
+  },
+);
+
 test.skipIf(
   process.platform === "win32" ||
     clangExecutable === undefined ||

packages/compiler/src/backend/native/compiler-fingerprint.ts

@@ -399,11 +399,8 @@ async function implicitToolchainFingerprintsFresh(
       ];
       const [dependencyResults, linker, assembler, compilerInvocation] = await Promise.all([
         Promise.all(
-          sources.map((source) =>
-            execFileAsync(compiler, [...probeArgs, source], {
-              cwd: probeDir,
-              maxBuffer: 16 * 1024 * 1024,
-            }),
+          sources.map((source, index) =>
+            probeIncludeDependencies(compiler, probeArgs, probeDir, source, sourceGroups[index]!),
           ),
         ),
         execFileAsync(compiler, [...prefix, "-print-prog-name=ld"], { cwd: probeDir }),
@@ -423,14 +420,7 @@ async function implicitToolchainFingerprintsFresh(
       const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter(
         (value, index, all) => value !== "" && all.indexOf(value) === index,
       );
-      const sourceSet = new Set(sources);
-      const dependencyPaths = [
-        ...new Set(
-          dependencyResults
-            .flatMap((dependencies) => parseMakeDependencies(dependencies.stdout))
-            .filter((path) => !sourceSet.has(path)),
-        ),
-      ].sort();
+      const dependencyPaths = [...new Set(dependencyResults.flat())].sort();
       probe = {
         compilerIdentity: compilerIdentity ?? `<unresolved>\0${compiler}`,
         compilerInvocation: normalizedProbeInvocation(compilerInvocation, probeDir),
@@ -513,6 +503,51 @@ async function implicitToolchainFingerprintsFresh(
   };
 }
 
+/** Headers the dependency probe preprocesses at a time when it probes them
+ * one by one. */
+const SINGLE_HEADER_PROBES = 8;
+
+/** The files the compiler reads to preprocess `source`, a probe source that
+ * includes `includes`, apart from the probe's own sources. When the compiler
+ * rejects `source`, for example for a header it finds that is an #error
+ * outside its target (<riscv_vector.h> through clang 18 without the RISC-V
+ * vector extension), each header is preprocessed on its own, and a header
+ * that the compiler rejects on its own is left out, as an absent one is.
+ * Throws the compiler's error when it rejects every header. */
+async function probeIncludeDependencies(
+  compiler: string,
+  probeArgs: readonly string[],
+  probeDir: string,
+  source: string,
+  includes: readonly string[],
+): Promise<string[]> {
+  const dependencies = async (path: string): Promise<string[]> => {
+    const result = await execFileAsync(compiler, [...probeArgs, path], {
+      cwd: probeDir,
+      maxBuffer: 16 * 1024 * 1024,
+    });
+    return parseMakeDependencies(result.stdout).filter((dependency) => dependency !== path);
+  };
+  try {
+    return await dependencies(source);
+  } catch (error) {
+    const found: string[][] = [];
+    for (let start = 0; start < includes.length; start += SINGLE_HEADER_PROBES) {
+      const batch = includes.slice(start, start + SINGLE_HEADER_PROBES);
+      const results = await Promise.all(
+        batch.map(async (include, offset) => {
+          const single = `${source.slice(0, -2)}-${start + offset}.c`;
+          await writeFile(single, implicitDependencyIncludeDirective(include));
+          return dependencies(single).catch(() => null);
+        }),
+      );
+      for (const result of results) if (result !== null) found.push(result);
+    }
+    if (found.length === 0) throw error;
+    return found.flat();
+  }
+}
+
 const stableImplicitToolchainMemos = new Map<string, Promise<ImplicitToolchainFingerprints>>();
 export function implicitToolchainFingerprints(
   driver: Pick<CcDriver, "argv" | "targetArgs" | "target">,