From e4fe30e08446700fe21983937b6570744ee52d4b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 21:16:15 -0500 Subject: [PATCH] Leave out a header the compiler rejects in the implicit toolchain probe The implicit toolchain probe preprocesses every system header that the runtime and vendored sources spell in one source file. When the compiler rejected one of them, the whole probe failed, and the build continued without the native cache and without program sharding. When the compiler rejects the probe source, each header is now preprocessed on its own, and a header that the compiler rejects on its own is left out of the dependency set, as an absent header is. The probe still fails when the compiler rejects every header. --- .../src/backend/native-toolchain.test.ts | 25 ++++++++ .../backend/native/compiler-fingerprint.ts | 61 +++++++++++++++---- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index 8fb86d4af..699847a2a 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -39,6 +39,8 @@ import { warmNativeCaches, } from "./native-toolchain.js"; import { splitLlvmProgram } from "./llvm/split.js"; +import { implicitToolchainFingerprints } from "./native/compiler-fingerprint.js"; +import { fingerprintDependencyPaths } from "./native/dependency-files.js"; const scratch: string[] = []; const TEST_CACHE_IDENTITY = "cc-cache-tests-v1"; @@ -672,6 +674,29 @@ test.skipIf(clangExecutable === undefined)( }, ); +test.skipIf(clangExecutable === undefined)( + "the implicit toolchain probe leaves out a header the compiler rejects", + async () => { + // A system header that the runtime spells, shadowed by one that is an + // #error, as clang 18's is outside RISC-V. The probe + // still covers the other headers. + const dir = await mkdtemp(join(tmpdir(), "scriptc-probe-header-")); + scratch.push(dir); + await mkdir(join(dir, "sys")); + const rejected = join(dir, "sys", "auxv.h"); + await writeFile(rejected, '#error "not for this target"\n'); + const includes = await implicitDependencyProbeIncludes(runtimeSrcDir()); + expect(includes).toContain(""); + const fingerprints = await implicitToolchainFingerprints( + { argv: [clangExecutable!, "-isystem", dir], targetArgs: [], target: null }, + "probe-rejected-header", + ); + const dependencies = fingerprintDependencyPaths(fingerprints.complete); + expect(dependencies).not.toContain(rejected); + expect(dependencies.some((path) => path.endsWith("/stdatomic.h"))).toBe(true); + }, +); + test.skipIf( process.platform === "win32" || clangExecutable === undefined || diff --git a/packages/compiler/src/backend/native/compiler-fingerprint.ts b/packages/compiler/src/backend/native/compiler-fingerprint.ts index 4be20a4d0..3df6cb449 100644 --- a/packages/compiler/src/backend/native/compiler-fingerprint.ts +++ b/packages/compiler/src/backend/native/compiler-fingerprint.ts @@ -399,11 +399,8 @@ async function implicitToolchainFingerprintsFresh( ]; const [dependencyResults, linker, assembler, compilerInvocation] = await Promise.all([ Promise.all( - sources.map((source) => - execFileAsync(compiler, [...probeArgs, source], { - cwd: probeDir, - maxBuffer: 16 * 1024 * 1024, - }), + sources.map((source, index) => + probeIncludeDependencies(compiler, probeArgs, probeDir, source, sourceGroups[index]!), ), ), execFileAsync(compiler, [...prefix, "-print-prog-name=ld"], { cwd: probeDir }), @@ -423,14 +420,7 @@ async function implicitToolchainFingerprintsFresh( const toolSpellings = [linker.stdout.trim(), assemblerSpelling].filter( (value, index, all) => value !== "" && all.indexOf(value) === index, ); - const sourceSet = new Set(sources); - const dependencyPaths = [ - ...new Set( - dependencyResults - .flatMap((dependencies) => parseMakeDependencies(dependencies.stdout)) - .filter((path) => !sourceSet.has(path)), - ), - ].sort(); + const dependencyPaths = [...new Set(dependencyResults.flat())].sort(); probe = { compilerIdentity: compilerIdentity ?? `\0${compiler}`, compilerInvocation: normalizedProbeInvocation(compilerInvocation, probeDir), @@ -513,6 +503,51 @@ async function implicitToolchainFingerprintsFresh( }; } +/** Headers the dependency probe preprocesses at a time when it probes them + * one by one. */ +const SINGLE_HEADER_PROBES = 8; + +/** The files the compiler reads to preprocess `source`, a probe source that + * includes `includes`, apart from the probe's own sources. When the compiler + * rejects `source`, for example for a header it finds that is an #error + * outside its target ( through clang 18 without the RISC-V + * vector extension), each header is preprocessed on its own, and a header + * that the compiler rejects on its own is left out, as an absent one is. + * Throws the compiler's error when it rejects every header. */ +async function probeIncludeDependencies( + compiler: string, + probeArgs: readonly string[], + probeDir: string, + source: string, + includes: readonly string[], +): Promise { + const dependencies = async (path: string): Promise => { + const result = await execFileAsync(compiler, [...probeArgs, path], { + cwd: probeDir, + maxBuffer: 16 * 1024 * 1024, + }); + return parseMakeDependencies(result.stdout).filter((dependency) => dependency !== path); + }; + try { + return await dependencies(source); + } catch (error) { + const found: string[][] = []; + for (let start = 0; start < includes.length; start += SINGLE_HEADER_PROBES) { + const batch = includes.slice(start, start + SINGLE_HEADER_PROBES); + const results = await Promise.all( + batch.map(async (include, offset) => { + const single = `${source.slice(0, -2)}-${start + offset}.c`; + await writeFile(single, implicitDependencyIncludeDirective(include)); + return dependencies(single).catch(() => null); + }), + ); + for (const result of results) if (result !== null) found.push(result); + } + if (found.length === 0) throw error; + return found.flat(); + } +} + const stableImplicitToolchainMemos = new Map>(); export function implicitToolchainFingerprints( driver: Pick,