From 53e07fc55ca9e5dcd6c60d5ba520ccc4e294beb7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 1 Aug 2026 15:25:04 -0500 Subject: [PATCH] fix: harden native build caches --- .../compiler/src/backend/cc-cache.test.ts | 314 +++++++++++++++++- packages/compiler/src/backend/cc.ts | 287 +++++++++++++--- packages/runtime/vendor/README.md | 8 +- tests/harness/README.md | 2 +- 4 files changed, 552 insertions(+), 59 deletions(-) diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index ce4373327..9c6c0b168 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -1,18 +1,21 @@ import { execFileSync } from "node:child_process"; import { existsSync } from "node:fs"; -import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { chmod, cp, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { afterAll, expect, test } from "vitest"; import { cacheTargetIdentity, + ccVersionOnce, compileC, compileLibArchive, resolveBuildCacheRoot, runtimeFingerprint, + runtimeSrcDir, stageRuntimeObjects, toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint, + vendorCacheBuildIdentity, vendorCacheTargetFlavor, } from "./cc.js"; @@ -84,8 +87,53 @@ test("the toolchain environment joins cache identities", () => { completeArtifacts: false, runtimeObjects: false, }); + expect(toolchainEnvironmentCachePolicy({ CFLAGS: "-I/headers" })).toEqual({ + completeArtifacts: false, + runtimeObjects: false, + }); + + expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe( + vendorCacheBuildIdentity(base, "compiler-two"), + ); + expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe( + vendorCacheBuildIdentity( + toolchainEnvironmentFingerprint({ MACOSX_DEPLOYMENT_TARGET: "11.0" }), + "compiler-one", + ), + ); }); +test.skipIf(process.platform === "win32")( + "compiler version probes use and remove a private working directory", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-version-probe-")); + scratch.push(dir); + const probeTmp = join(dir, "tmp"); + const fakeCompiler = join(dir, "fake-zig"); + await mkdir(probeTmp); + await writeFile( + fakeCompiler, + `#!/bin/sh +: > a.o +printf 'fake zig cc version probe\n' +`, + ); + await chmod(fakeCompiler, 0o755); + const oldTmpdir = process.env["TMPDIR"]; + try { + process.env["TMPDIR"] = probeTmp; + // Zig 0.16 creates cwd/a.o for `zig cc --version`. Model that side + // effect directly so this regression does not depend on Zig being + // installed in every generic test shard. + await ccVersionOnce([fakeCompiler, "cc"], `version-probe-${dir}`); + expect(await readdir(probeTmp)).toEqual([]); + } finally { + if (oldTmpdir === undefined) delete process.env["TMPDIR"]; + else process.env["TMPDIR"] = oldTmpdir; + } + }, +); + test("the runtime fingerprint includes the textually included Ryū sources", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-fingerprint-")); scratch.push(dir); @@ -314,6 +362,61 @@ test("mutable compiler inputs bypass caches when files change in place", async ( } }); +test.skipIf(process.platform !== "darwin")( + "vendor object caches separate deployment-target environments", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-vendor-environment-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const vendorCacheRoot = join(dir, "vendor-cache"); + const cPath = join(dir, "program.c"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldVendorCacheDir = process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"]; + const oldDeploymentTarget = process.env["MACOSX_DEPLOYMENT_TARGET"]; + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = vendorCacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await writeFile(cPath, "int main(void) { return 0; }\n"); + + process.env["MACOSX_DEPLOYMENT_TARGET"] = "14.0"; + await compileC({ + cPath, + outPath: join(dir, "newer"), + cacheIdentity: TEST_CACHE_IDENTITY, + regex: true, + }); + process.env["MACOSX_DEPLOYMENT_TARGET"] = "11.0"; + await compileC({ + cPath, + outPath: join(dir, "older"), + cacheIdentity: TEST_CACHE_IDENTITY, + regex: true, + }); + + const lreCaches = (await readdir(vendorCacheRoot)).filter((name) => name.includes("-lre-")); + expect(lreCaches).toHaveLength(2); + const minVersions = lreCaches.map((cache) => + execFileSync("vtool", ["-show-build", join(vendorCacheRoot, cache, "libregexp.o")], { + encoding: "utf8", + }).match(/minos\s+(\S+)/)?.[1], + ); + expect(new Set(minVersions)).toEqual(new Set(["11.0", "14.0"])); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldVendorCacheDir === undefined) delete process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"]; + else process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = oldVendorCacheDir; + if (oldDeploymentTarget === undefined) delete process.env["MACOSX_DEPLOYMENT_TARGET"]; + else process.env["MACOSX_DEPLOYMENT_TARGET"] = oldDeploymentTarget; + } + }, +); + test("complete binary hits precede missing vendor prerequisite materialization", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-cache-vendor-hit-")); scratch.push(dir); @@ -493,7 +596,18 @@ done exec "$SCRIPTC_TEST_REAL_CLANG" "$@" `, ); - await chmod(wrapper, 0o755); + // ccache availability is memoized across this file. If an earlier test + // found it, keep the controlled compiler wrapper in the path by making + // a cache shim that simply forwards its compiler argv. + const ccacheWrapper = join(binDir, "ccache"); + await writeFile( + ccacheWrapper, + `#!/bin/sh +if [ "$1" = "--version" ]; then exit 1; fi +exec "$@" +`, + ); + await Promise.all([chmod(wrapper, 0o755), chmod(ccacheWrapper, 0o755)]); await writeFile(cPath, source("one")); process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; process.env["SCRIPTC_TEST_REAL_CLANG"] = originalClang!; @@ -584,6 +698,129 @@ exec "$SCRIPTC_TEST_REAL_CLANG" "$@" }, ); +test.skipIf(process.platform === "win32")( + "runtime edits during object compilation cannot poison the old fingerprint", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-cache-race-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const fakeRuntime = join(dir, "runtime", "src"); + const originalRuntime = runtimeSrcDir(); + await Promise.all([ + cp(originalRuntime, fakeRuntime, { recursive: true }), + mkdir(join(dir, "runtime", "vendor"), { recursive: true }).then(() => + cp(join(originalRuntime, "..", "vendor", "ryu"), join(dir, "runtime", "vendor", "ryu"), { + recursive: true, + }), + ), + ]); + + const binDir = join(dir, "bin"); + const cPath = join(dir, "program.c"); + const raceSource = join(fakeRuntime, "scr_number.c"); + const signal = join(dir, "runtime-compile-started"); + const release = join(dir, "runtime-compile-release"); + const marker = "scriptc-runtime-race-marker-unique"; + const originalRuntimeSource = await readFile(raceSource, "utf8"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldRuntimeDir = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"]; + const oldDisableCcache = process.env["SCRIPTC_TEST_DISABLE_CCACHE"]; + const oldPath = process.env["PATH"]; + const oldRealClang = process.env["SCRIPTC_TEST_REAL_CLANG"]; + const oldRaceSource = process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"]; + const oldRaceSignal = process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"]; + const oldRaceRelease = process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"]; + const realClang = (oldPath ?? "") + .split(delimiter) + .map((entry) => join(entry === "" ? process.cwd() : entry, "clang")) + .find((candidate) => existsSync(candidate)); + expect(realClang).toBeDefined(); + + try { + await mkdir(binDir); + const wrapper = join(binDir, "clang"); + await writeFile( + wrapper, + `#!/bin/sh +for arg in "$@"; do + if [ "$arg" = "$SCRIPTC_TEST_RUNTIME_RACE_SOURCE" ]; then + : > "$SCRIPTC_TEST_RUNTIME_RACE_SIGNAL" + while [ ! -e "$SCRIPTC_TEST_RUNTIME_RACE_RELEASE" ]; do sleep 0.01; done + break + fi +done +exec "$SCRIPTC_TEST_REAL_CLANG" "$@" +`, + ); + await chmod(wrapper, 0o755); + await writeFile(cPath, "int main(void) { return 0; }\n"); + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = fakeRuntime; + process.env["SCRIPTC_TEST_REAL_CLANG"] = realClang!; + process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = raceSource; + process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = signal; + process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = release; + // Keep ccache out of this controlled race: its own object store is not + // the cache under test and can obscure which compiler read happened. + process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = "1"; + process.env["PATH"] = `${binDir}${delimiter}/usr/bin${delimiter}/bin`; + delete process.env["SCRIPTC_NO_CACHE"]; + + const firstOut = join(dir, "first"); + const firstBuild = compileC({ + cPath, + outPath: firstOut, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + const deadline = Date.now() + 10_000; + while (!existsSync(signal)) { + if (Date.now() > deadline) throw new Error("timed out waiting for runtime compile"); + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + await writeFile( + raceSource, + `${originalRuntimeSource}\nconst char scriptc_runtime_race_marker[] = "${marker}";\n`, + ); + await writeFile(release, "go"); + await firstBuild; + expect((await readFile(firstOut)).includes(Buffer.from(marker))).toBe(true); + + // Restore the fingerprint, then force a program-cache miss. The old + // fingerprint must not have retained the object built from edited bytes. + await writeFile(raceSource, originalRuntimeSource); + await writeFile(cPath, "int main(void) { return 0; } /* second */\n"); + const secondOut = join(dir, "second"); + await compileC({ + cPath, + outPath: secondOut, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + expect((await readFile(secondOut)).includes(Buffer.from(marker))).toBe(false); + expect(await readdir(join(cacheRoot, "bin"))).toHaveLength(1); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldRuntimeDir === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"]; + else process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = oldRuntimeDir; + if (oldDisableCcache === undefined) delete process.env["SCRIPTC_TEST_DISABLE_CCACHE"]; + else process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = oldDisableCcache; + if (oldPath === undefined) delete process.env["PATH"]; + else process.env["PATH"] = oldPath; + if (oldRealClang === undefined) delete process.env["SCRIPTC_TEST_REAL_CLANG"]; + else process.env["SCRIPTC_TEST_REAL_CLANG"] = oldRealClang; + if (oldRaceSource === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"]; + else process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = oldRaceSource; + if (oldRaceSignal === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"]; + else process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = oldRaceSignal; + if (oldRaceRelease === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"]; + else process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = oldRaceRelease; + } + }, +); + test("explicit native link inputs relink while runtime objects remain cached", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-link-input-cache-")); scratch.push(dir); @@ -704,6 +941,77 @@ test("system libraries relink after an in-place rebuild while runtime objects re } }); +test("damaged runtime objects are rebuilt before linking or archiving", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-object-integrity-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const corruption = Buffer.from("scriptc-corrupt-object-unique-marker"); + + const objectSets = async (): Promise => + (await readdir(join(cacheRoot, "obj"), { withFileTypes: true })) + .filter((entry) => entry.isDirectory() && !entry.name.startsWith("build-")) + .map((entry) => entry.name); + const corruptOneObject = async (set: string): Promise => { + const objectDir = join(cacheRoot, "obj", set); + const object = (await readdir(objectDir)).find((name) => name.endsWith(".o")); + expect(object).toBeDefined(); + await writeFile(join(objectDir, object!), corruption); + expect((await stat(`${join(objectDir, object!)}.sha256`)).isFile()).toBe(true); + return join(objectDir, object!); + }; + + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await writeFile(cPath, '#include \nint main(void) { puts("one"); return 0; }\n'); + await compileC({ + cPath, + outPath: join(dir, "one"), + cacheIdentity: TEST_CACHE_IDENTITY, + }); + const [executableSet] = await objectSets(); + const damagedExecutableObject = await corruptOneObject(executableSet!); + + await writeFile(cPath, '#include \nint main(void) { puts("two"); return 0; }\n'); + const repairedOut = join(dir, "two"); + await compileC({ + cPath, + outPath: repairedOut, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + expect(execFileSync(repairedOut, { encoding: "utf8" }).trim()).toBe("two"); + expect(await readFile(damagedExecutableObject)).not.toEqual(corruption); + + const beforeLibrary = new Set(await objectSets()); + await writeFile(cPath, "int scriptc_integrity_probe = 1;\n"); + await compileLibArchive({ + cPath, + outPath: join(dir, "one.lib.a"), + cacheIdentity: TEST_CACHE_IDENTITY, + }); + const librarySet = (await objectSets()).find((set) => !beforeLibrary.has(set)); + expect(librarySet).toBeDefined(); + await corruptOneObject(librarySet!); + + await writeFile(cPath, "int scriptc_integrity_probe = 2;\n"); + const repairedArchive = join(dir, "two.lib.a"); + await compileLibArchive({ + cPath, + outPath: repairedArchive, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + expect((await readFile(repairedArchive)).includes(corruption)).toBe(false); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } +}); + test("library archives hit by content, invalidate on edits, and reuse runtime objects", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-cache-")); scratch.push(dir); @@ -731,7 +1039,7 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob const objectSet = objectSets.find((entry) => entry.isDirectory() && !entry.name.startsWith("build-")); expect(objectSet).toBeDefined(); const objectDir = join(cacheRoot, "obj", objectSet!.name); - const objectNames = await readdir(objectDir); + const objectNames = (await readdir(objectDir)).filter((name) => name.endsWith(".o")); expect(objectNames.length).toBeGreaterThan(10); const pinnedTime = new Date("2000-01-01T00:00:00.000Z"); await Promise.all(objectNames.map((name) => utimes(join(objectDir, name), pinnedTime, pinnedTime))); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 74d3f7a9b..d75fa314b 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -59,6 +59,14 @@ const TOOLCHAIN_ENV_KEYS = [ "CCC_ADD_ARGS", "CLANG_CONFIG_FILE_SYSTEM_DIR", "CLANG_CONFIG_FILE_USER_DIR", + "CC", + "CFLAGS", + "CPPFLAGS", + "LDFLAGS", + "AR", + "RANLIB", + "CMAKE_GENERATOR", + "CMAKE_TOOLCHAIN_FILE", "SOURCE_DATE_EPOCH", "ZERO_AR_DATE", "LANG", @@ -90,6 +98,17 @@ const MUTABLE_COMPILE_ENV_KEYS = [ "CCC_ADD_ARGS", "CLANG_CONFIG_FILE_SYSTEM_DIR", "CLANG_CONFIG_FILE_USER_DIR", + // The direct runtime build does not consume the conventional Make/CMake + // variables, but the lazily-built QuickJS prerequisite does. Their values + // can name mutable tools, headers, libraries, and toolchain files, so a + // stable spelling is not a safe persistent-cache identity. + "CC", + "CFLAGS", + "CPPFLAGS", + "LDFLAGS", + "AR", + "RANLIB", + "CMAKE_TOOLCHAIN_FILE", ] as const; /** These variables only redirect link-time inputs. Runtime objects remain @@ -316,6 +335,8 @@ export class CcCompileError extends Error { } export function runtimeSrcDir(): string { + const testRoot = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"]; + if (testRoot !== undefined) return resolve(testRoot); const require = createRequire(import.meta.url); return join(dirname(require.resolve("@scriptc/runtime/package.json")), "src"); } @@ -454,12 +475,60 @@ export function vendorCacheTargetFlavor( return driver.target === null ? `native-${hostPlatform}-${hostArch}` : driver.target; } +/** Vendor prerequisites live outside the content-addressed build root, so + * their directory name must carry the same environment and resolved compiler + * identity as the artifacts that consume them. The short digest keeps the + * package-local path readable while preventing build-order contamination. */ +export function vendorCacheBuildIdentity( + environmentFingerprint: string, + compilerIdentity: string, +): string { + return createHash("sha256") + .update("vendor-toolchain-v1\0") + .update(environmentFingerprint) + .update("\0") + .update(compilerIdentity) + .digest("hex") + .slice(0, 20); +} + +const vendorCompilerIdentityFallbacks = new Map(); + +async function currentVendorCacheBuildIdentity( + driver: Pick, + environmentFingerprint: string, +): Promise { + // Native vendor recipes additionally use bare clang/ar and CMake; cross + // recipes use the zig driver for compilation and `zig ar`. Include every + // executable that can affect the package-local prerequisite, not just the + // final program's driver. + const commands = [ + driver.argv[0] ?? "clang", + ...(driver.target === null ? ["clang", "cc", "ar", "ranlib", "cmake"] : []), + ].filter((command, index, all) => all.indexOf(command) === index); + const identities = await Promise.all( + commands.map(async (command) => { + const spellingKey = `${environmentFingerprint}\0${command}`; + const resolved = await resolvedToolIdentity(command); + if (resolved !== null) vendorCompilerIdentityFallbacks.set(spellingKey, resolved); + return [ + command, + resolved ?? + vendorCompilerIdentityFallbacks.get(spellingKey) ?? + `\0${command}`, + ].join("\0"); + }), + ); + return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f")); +} + function engineArchivePath( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): string { - const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`; + const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; return join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-${flavor}`, "libqjs.a"); } @@ -493,7 +562,7 @@ async function installArtifact( } /** The engine archive for one flavor, built lazily on the first --dynamic - * compile (~10s) and cached under vendor/.cache/--/ — unlike + * compile (~10s) and cached under vendor/.cache/---/ — unlike * the runtime's own sources (recompiled every build, ~100ms), the engine is * far too big to rebuild per compile. The plain flavor is ALWAYS MinSizeRel * regardless of the program's optimization level: measured binary delta is @@ -513,11 +582,12 @@ async function installArtifact( async function ensureEngineArchive( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): Promise { - const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`; + const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const vendor = vendorEngineDir(); - const archive = engineArchivePath(sanitize, driver, cacheRoot); + const archive = engineArchivePath(sanitize, driver, buildIdentity, cacheRoot); const cacheDir = dirname(archive); if (await fileExists(archive)) return archive; if (driver.target !== null) return buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir); @@ -638,19 +708,20 @@ const LRE_SOURCES = ["libregexp.c", "libunicode.c"]; function lreObjectPaths( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): string[] { const flavor = (sanitize ? "asan" : "plain") + (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + - `-${vendorCacheTargetFlavor(driver)}`; + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const cacheDir = join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-lre-${flavor}`); return LRE_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o"))); } /** The libregexp objects for one flavor, compiled lazily on the first * regex-using static build (~1s) and cached like the engine archive — - * vendor/.cache/-lre-/*.o — with the same atomic-rename + * vendor/.cache/-lre---/*.o — with the same atomic-rename * publish (parallel first builds race safely; losers discard their work). * Plain is -Os (size class matters: the regex fence pins regex-using * binaries well under the engine class); asan matches the final link so @@ -662,6 +733,7 @@ function lreObjectPaths( async function ensureLreObjects( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): Promise { // The flavor keys the DRIVER as well as the target: a zig-cc-built object @@ -671,9 +743,9 @@ async function ensureLreObjects( const flavor = (sanitize ? "asan" : "plain") + (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + - `-${vendorCacheTargetFlavor(driver)}`; + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const vendor = vendorEngineDir(); - const objects = lreObjectPaths(sanitize, driver, cacheRoot); + const objects = lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot); const cacheDir = dirname(objects[0]!); if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects; @@ -721,19 +793,20 @@ const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infbac function zlibObjectPaths( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): string[] { const flavor = (sanitize ? "asan" : "plain") + (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + - `-${vendorCacheTargetFlavor(driver)}`; + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const cacheDir = join(cacheRoot, `zlib-${ZLIB_VERSION}-${flavor}`); return ZLIB_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o"))); } /** The zlib objects for one flavor, compiled lazily on the first zlib-using * CROSS build (~1s) and cached like the lre objects — - * vendor/.cache/zlib--/*.o — with the same atomic-rename + * vendor/.cache/zlib----/*.o — with the same atomic-rename * publish (parallel first builds race safely; losers discard their work). * Plain is -Os, asan matches the final link so the sanitized lane * instruments the codec too. The flavor keys the driver and target exactly @@ -743,14 +816,15 @@ function zlibObjectPaths( async function ensureZlibObjects( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): Promise { const flavor = (sanitize ? "asan" : "plain") + (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") + - `-${vendorCacheTargetFlavor(driver)}`; + `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const vendor = vendorZlibDir(); - const objects = zlibObjectPaths(sanitize, driver, cacheRoot); + const objects = zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot); const cacheDir = dirname(objects[0]!); if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects; @@ -802,14 +876,15 @@ const CURL_STUB_SYMBOLS = [ function curlStubDirPath( driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): string { - return join(cacheRoot, `curl-stub-${driver.target}`); + return join(cacheRoot, `curl-stub-${driver.target}-${buildIdentity}`); } /** The libcurl import stub for one CROSS target, generated lazily on the * first fetch-using cross build and cached like the zlib objects — - * vendor/.cache/curl-stub-/libcurl.so, atomic-rename publish. The + * vendor/.cache/curl-stub--/libcurl.so, atomic-rename publish. The * stub is empty definitions of CURL_STUB_SYMBOLS compiled `-shared` with * `-soname libcurl.so.4`: linking against it satisfies scr_fetch_curl.c's * references and records a plain `DT_NEEDED libcurl.so.4`, which the @@ -820,9 +895,10 @@ function curlStubDirPath( * and asan links against plain shared libraries freely. */ async function ensureCurlStub( driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): Promise { - const cacheDir = curlStubDirPath(driver, cacheRoot); + const cacheDir = curlStubDirPath(driver, buildIdentity, cacheRoot); const lib = join(cacheDir, "libcurl.so"); if (await fileExists(lib)) return cacheDir; @@ -860,15 +936,16 @@ function vendorTlsDir(): string { function tlsArchivePath( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): string { - const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`; + const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; return join(cacheRoot, `mbedtls-${MBEDTLS_VERSION}-${flavor}`, "libmbedtls.a"); } /** The mbedTLS archive for one flavor, compiled lazily on the first * TLS-using build (~15s over ~110 TUs, parallelized) and cached like the - * engine archive — vendor/.cache/mbedtls---/libmbedtls.a — + * engine archive — vendor/.cache/mbedtls----/libmbedtls.a — * with the same atomic-rename publish (parallel first builds race safely; * losers discard their work). Plain is -Os (the TLS stack is link-gated * but still wants its size class small); asan matches the final link so @@ -884,13 +961,14 @@ function tlsArchivePath( async function ensureTlsArchive( sanitize: boolean, driver: CcDriver, + buildIdentity: string, cacheRoot: string = vendorBuildCacheRoot(), ): Promise { - const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`; + const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`; const compileArgv = driver.target !== null ? driver.argv : ["clang"]; const arArgv = driver.target !== null ? [...driver.argv.slice(0, 1), "ar"] : ["ar"]; const vendor = vendorTlsDir(); - const archive = tlsArchivePath(sanitize, driver, cacheRoot); + const archive = tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot); const cacheDir = dirname(archive); if (await fileExists(archive)) return archive; @@ -1031,6 +1109,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise } } const toolchainEnv = toolchainEnvironmentFingerprint(); + const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv); let cachedArchive: string | null = null; let compilerVersion = ""; let runtimeHash = ""; @@ -1047,7 +1126,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise runtimeHash = fingerprint; cachedProgramBytes = programBytes; const key = createHash("sha256") - .update("lib-v3\0") + // v4 invalidates archives that may have been assembled from an + // unverified or cross-environment vendor/runtime object cache. + .update("lib-v4\0") .update(cacheTargetIdentity(driver)).update("\0") .update(toolchainEnv).update("\0") .update(opts.cacheIdentity!).update("\0") @@ -1097,10 +1178,10 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot(); try { const lreObjects = regex - ? await ensureLreObjects(sanitize, driver, vendorCacheRoot) + ? await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) : []; const zlibObjects = opts.zlib - ? await ensureZlibObjects(sanitize, driver, vendorCacheRoot) + ? await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) : []; const buildDir = await mkdtemp(join(tmpdir(), "scriptc-lib-")); try { @@ -1147,6 +1228,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise cachedProgramBytes === null ? undefined : opts.cPath, ); let runtimeObjects: string[] | null = null; + let cacheInputsStable = true; if (root !== null && compilerVersion !== "" && runtimeHash !== "") { try { const sourcePaths = sources.map((f) => join(rtDir, f)); @@ -1155,11 +1237,13 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise driver.argv, cflags, sourcePaths, - `lib-obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`, + `lib-obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`, + async () => (await runtimeFingerprint(rtDir)) === runtimeHash, ); const staged = await stageRuntimeObjects(cached, join(buildDir, "cached-runtime")); runtimeObjects = sourcePaths.map((path) => staged.get(path)!); - } catch { + } catch (err) { + if (err instanceof CacheInputsChangedError) cacheInputsStable = false; runtimeObjects = null; } } @@ -1191,7 +1275,11 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ...objects, ]); if (archiveOutput !== opts.outPath) await installArtifact(archiveOutput, opts.outPath); - if (cachedArchive !== null && root !== null) { + const runtimeStillMatchesKey = + cacheInputsStable && + runtimeHash !== "" && + (await runtimeFingerprint(rtDir).catch(() => null)) === runtimeHash; + if (cachedArchive !== null && root !== null && runtimeStillMatchesKey) { try { const cacheDir = dirname(cachedArchive); await mkdir(cacheDir, { recursive: true }); @@ -1367,7 +1455,7 @@ async function resolvedToolIdentity( return null; } -async function ccVersionOnce( +export async function ccVersionOnce( argv: string[], environmentFingerprint: string = toolchainEnvironmentFingerprint(), ): Promise { @@ -1384,11 +1472,22 @@ async function ccVersionOnce( const key = `${spellingKey}\0${executableIdentity ?? ""}`; let memo = ccVersionMemos.get(key); if (memo === undefined) { - // cwd: the version probe must not touch the caller's directory — - // `zig cc --version` (zig 0.16) drops an empty a.o wherever it runs. - memo = execFileAsync(argv[0] ?? "clang", [...argv.slice(1), "--version"], { cwd: tmpdir() }).then( - (r) => `${executableIdentity ?? ""}\0${`${r.stdout}\n${r.stderr}`.trim()}`, - ); + // `zig cc --version` (zig 0.16) drops an empty a.o in its cwd. A private + // probe directory avoids both caller pollution and the cross-process / + // cross-user collision a fixed tmpdir()/a.o would create on Linux. + memo = (async () => { + const probeDir = await mkdtemp(join(tmpdir(), "scriptc-cc-version-")); + try { + const r = await execFileAsync( + argv[0] ?? "clang", + [...argv.slice(1), "--version"], + { cwd: probeDir }, + ); + return `${executableIdentity ?? ""}\0${`${r.stdout}\n${r.stderr}`.trim()}`; + } finally { + await rm(probeDir, { recursive: true, force: true }).catch(() => undefined); + } + })(); ccVersionMemos.set(key, memo); } if (executableIdentity !== null) ccVersionFallbacks.set(spellingKey, memo); @@ -1456,15 +1555,55 @@ export async function runtimeFingerprint(rtDir: string): Promise { return h.digest("hex"); } +class CacheInputsChangedError extends Error { + constructor() { + super("runtime inputs changed while populating the native object cache"); + this.name = "CacheInputsChangedError"; + } +} + +function cacheDigestPath(path: string): string { + return `${path}.sha256`; +} + +async function fileDigest(path: string): Promise { + return createHash("sha256").update(await readFile(path)).digest("hex"); +} + +/** Cache entries are disposable data, not trusted compiler output. Atomic + * rename prevents ordinary partial writes, while the adjacent digest catches + * disk damage, manual truncation, and entries left by an interrupted/older + * publisher before they reach clang, the linker, or `ar`. */ +async function validCachedFile(object: string): Promise { + try { + const digestPath = cacheDigestPath(object); + const expected = (await readFile(digestPath, "utf8")).trim(); + const valid = /^[0-9a-f]{64}$/.test(expected) && (await fileDigest(object)) === expected; + if (valid) { + const now = new Date(); + await Promise.all([ + utimes(object, now, now).catch(() => undefined), + utimes(digestPath, now, now).catch(() => undefined), + ]); + } + return valid; + } catch { + return false; + } +} + /** The cached .o set for one flag flavor, compiled on first need. Concurrent * first builds (parallel test workers on a cold cache) may duplicate work; - * per-file atomic renames make every winner equivalent. */ + * per-file atomic renames make every winner equivalent. Publication is held + * until verifyInputs confirms that the source/header fingerprint used by the + * key still describes the bytes clang just read. */ async function ensureRuntimeObjects( root: string, ccArgv: string[], cflags: string[], sources: string[], keyPrefix: string, + verifyInputs: () => Promise, ): Promise> { const setKey = createHash("sha256") .update(keyPrefix) @@ -1473,15 +1612,20 @@ async function ensureRuntimeObjects( .slice(0, 24); const objDir = join(root, "obj", setKey); const objOf = (src: string): string => join(objDir, `${basename(src, ".c")}.o`); - const present = await Promise.all(sources.map((s) => fileExists(objOf(s)))); + const present = await Promise.all(sources.map((s) => validCachedFile(objOf(s)))); const missing = sources.filter((_, i) => !present[i]); if (missing.length > 0) { // ccache wraps only the default clang driver — multi-word drivers // (`zig cc`) run bare; their object sets are keyed apart anyway. - const useCcache = ccArgv.length === 1 && ccArgv[0] === "clang" && (await ccacheAvailable()); + const useCcache = + process.env["SCRIPTC_TEST_DISABLE_CCACHE"] !== "1" && + ccArgv.length === 1 && + ccArgv[0] === "clang" && + (await ccacheAvailable()); await mkdir(objDir, { recursive: true }); const tmpDir = await mkdtemp(join(root, "obj", "build-")); try { + const compiled = new Map(); // Modest parallelism: a flavor's objects build once, but several cold // workers can race here — keep each build's CPU footprint small. const width = 4; @@ -1491,15 +1635,33 @@ async function ensureRuntimeObjects( const tmpObj = join(tmpDir, `${basename(src, ".c")}.o`); const argv = [...(useCcache ? ["ccache"] : []), ...ccArgv, ...cflags, "-c", src, "-o", tmpObj]; await execFileAsync(argv[0] ?? "clang", argv.slice(1)); - await rename(tmpObj, objOf(src)); + compiled.set(src, { object: tmpObj, digest: await fileDigest(tmpObj) }); }), ); } + // The fingerprint was computed before these subprocesses started. Do + // not place their outputs under that key if a checkout/package update + // changed any runtime source or included header while clang was reading. + if (!(await verifyInputs())) throw new CacheInputsChangedError(); + for (const [src, built] of compiled) { + const destination = objOf(src); + const tmpDigest = `${built.object}.sha256`; + await writeFile(tmpDigest, `${built.digest}\n`); + // Publish data first and its verifier second. A concurrent reader in + // the tiny gap sees a missing/mismatched digest and falls back safely. + await rename(built.object, destination); + await rename(tmpDigest, cacheDigestPath(destination)); + } } finally { await rm(tmpDir, { recursive: true, force: true }); } } - return new Map(sources.map((s) => [s, objOf(s)])); + if (!(await verifyInputs())) throw new CacheInputsChangedError(); + const objects = new Map(sources.map((s) => [s, objOf(s)])); + if (!(await Promise.all([...objects.values()].map(validCachedFile))).every(Boolean)) { + throw new Error("native object cache integrity check failed"); + } + return objects; } /** Give one active link/archive operation private names for its cached @@ -1643,6 +1805,8 @@ export async function compileC(opts: CcOptions): Promise { } } const cachePolicy = toolchainEnvironmentCachePolicy(); + const toolchainEnv = toolchainEnvironmentFingerprint(); + const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv); // Mutable include/SDK/config inputs can change behind a stable environment // spelling. The complete/runtime cache is disabled below; vendor objects // must follow the same rule instead of silently surviving in package-local @@ -1658,34 +1822,48 @@ export async function compileC(opts: CcOptions): Promise { // Every vendor output path is deterministic from pins, flags, driver, and // target. Build the command/key from those paths now, but do not materialize // them until a complete-binary lookup has missed. - const engineArchive = dynamic ? engineArchivePath(sanitize, driver, vendorCacheRoot) : null; - const tlsArchive = tls ? tlsArchivePath(sanitize, driver, vendorCacheRoot) : null; + const engineArchive = dynamic + ? engineArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) + : null; + const tlsArchive = tls + ? tlsArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) + : null; // --dynamic + regex shares the archive's libregexp (its host hooks and // ours would collide; see scr_regex.c) — the standalone objects are for // static builds only. - const lreObjects = regex && !dynamic ? lreObjectPaths(sanitize, driver, vendorCacheRoot) : []; + const lreObjects = regex && !dynamic + ? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) + : []; // Vendored zlib is the CROSS story only — host builds keep the exact // historical `-lz` system link (see CcOptions.zlib). The native fetch's // gzip decoder rides the same objects/link. const zlibObjects = ((opts.zlib ?? false) || nativeFetch) && driver.target !== null - ? zlibObjectPaths(sanitize, driver, vendorCacheRoot) + ? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot) : []; // The libcurl import stub is likewise CROSS-only — host builds keep the // exact historical system `-lcurl` link (see CcOptions.fetch). Curl // reference builds only. const curlStubDir = curlFetch && driver.target !== null - ? curlStubDirPath(driver, vendorCacheRoot) + ? curlStubDirPath(driver, vendorBuildIdentity, vendorCacheRoot) : null; const materializeVendorPrerequisites = async (): Promise => { // Preserve the historical order to avoid multiplying first-build resource // pressure when several large vendor sets are cold simultaneously. - if (dynamic) await ensureEngineArchive(sanitize, driver, vendorCacheRoot); - if (tls) await ensureTlsArchive(sanitize, driver, vendorCacheRoot); - if (regex && !dynamic) await ensureLreObjects(sanitize, driver, vendorCacheRoot); - if (zlibObjects.length > 0) await ensureZlibObjects(sanitize, driver, vendorCacheRoot); - if (curlStubDir !== null) await ensureCurlStub(driver, vendorCacheRoot); + if (dynamic) { + await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot); + } + if (tls) await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot); + if (regex && !dynamic) { + await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot); + } + if (zlibObjects.length > 0) { + await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot); + } + if (curlStubDir !== null) { + await ensureCurlStub(driver, vendorBuildIdentity, vendorCacheRoot); + } }; // rt() maps each runtime source's path on the command line: identity for // the historical single invocation, cached-.o substitution on cache misses. @@ -1925,7 +2103,6 @@ export async function compileC(opts: CcOptions): Promise { return; } - const toolchainEnv = toolchainEnvironmentFingerprint(); const [cv, fingerprint, cBytes] = await Promise.all([ ccVersionOnce(driver.argv, toolchainEnv), runtimeFingerprint(rtDir), @@ -1952,7 +2129,9 @@ export async function compileC(opts: CcOptions): Promise { a === opts.cPath ? "" : a === opts.outPath ? "" : a, ); const key = createHash("sha256") - .update("bin-v3\0") + // v4 invalidates binaries that may have been published while runtime + // inputs changed underneath the former per-file object publisher. + .update("bin-v4\0") .update(cacheTargetIdentity(driver)).update("\0") .update(toolchainEnv).update("\0") .update(opts.cacheIdentity!).update("\0") @@ -2030,16 +2209,19 @@ export async function compileC(opts: CcOptions): Promise { await writeFile(programPath, cBytes); let objects: Map | null = null; + let cacheInputsStable = true; try { const cached = await ensureRuntimeObjects( root, driver.argv, cflags, rtInputs, - `obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`, + `obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`, + async () => (await runtimeFingerprint(rtDir)) === fingerprint, ); objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects")); - } catch { + } catch (err) { + if (err instanceof CacheInputsChangedError) cacheInputsStable = false; objects = null; // cache trouble is never a build failure } @@ -2051,7 +2233,10 @@ export async function compileC(opts: CcOptions): Promise { ); await installArtifact(privateOut, opts.outPath); - if (cachedBin !== null && keyHex !== null) { + cacheInputsStable = + cacheInputsStable && + (await runtimeFingerprint(rtDir).catch(() => null)) === fingerprint; + if (cachedBin !== null && keyHex !== null && cacheInputsStable) { try { await mkdir(binDir, { recursive: true }); const tmp = join(binDir, `.tmp-${keyHex.slice(0, 8)}-${process.pid}-${Math.random().toString(36).slice(2)}`); diff --git a/packages/runtime/vendor/README.md b/packages/runtime/vendor/README.md index 7b08bd09e..19e56a711 100644 --- a/packages/runtime/vendor/README.md +++ b/packages/runtime/vendor/README.md @@ -19,7 +19,7 @@ The QuickJS-ng JavaScript engine, embedded by the opt-in `--dynamic` build mode The tree is a plain snapshot of the upstream commit with directories the library build does not need removed (tests/, docs/, examples/, test262 fixtures, CI config, generator scripts). No vendored file is modified; to update, re-clone upstream at the new commit, delete its .git directory, apply the same trim, and update this file. -The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/-/` next to this file — one flavor per lane (plain, asan). The cache directory is gitignored and safe to delete. +The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/---/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity. The cache directory is gitignored and safe to delete. ## zlib/ @@ -29,7 +29,7 @@ The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile i The CROSS-target arm of node:zlib (see packages/runtime/src/scr_zlib.c): host builds keep the historical system `-lz` link (macOS ships libz), but zig's cross sysroots have no libz, so SCRIPTC_TARGET builds compile this vendored copy per target instead. Only the flat `*.c`/`*.h` at the distribution root are vendored (LICENSE beside them) — contrib, tests, build machinery, and docs are not. No vendored file is modified; the gz* file-I/O TUs are vendored for faithfulness but never compiled (nothing references the gzFile API — see ZLIB_SOURCES in packages/compiler/src/backend/cc.ts). To update, re-fetch the release tarball, copy the same flat set, and bump `ZLIB_VERSION` in cc.ts. -The objects build lazily on the first zlib-using cross compile into `.cache/zlib--/` next to this file — one flavor per driver/target (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation. +The objects build lazily on the first zlib-using cross compile into `.cache/zlib----/` next to this file — one flavor per driver/target and compiler environment/identity (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation. ## curl/ @@ -39,7 +39,7 @@ The objects build lazily on the first zlib-using cross compile into `.cache/zlib HEADERS ONLY — no curl C source is vendored and none is ever compiled. These headers now serve ONLY the RETIRED curl reference implementation of fetch (packages/runtime/src/scr_fetch_curl.c, selected by `SCRIPTC_FETCH_CURL=1`; kept compilable for one release as the native flip's reference — see scr_fetch.c, the default, which rides scr_net/scr_tls/scr_http/zlib and touches nothing here). Under the flag: host builds link the system `-lcurl` (macOS ships libcurl), and linux-gnu CROSS builds compile scr_fetch_curl.c against these headers, then link against a generated STUB `libcurl.so` (soname `libcurl.so.4`, empty definitions of exactly the symbols scr_fetch_curl.c calls — see CURL_STUB_SYMBOLS in packages/compiler/src/backend/cc.ts) so the produced binary records a plain `DT_NEEDED libcurl.so.4` that the TARGET system's real libcurl satisfies at load time, the standard cross-link import-stub technique. The 7.88.1 pin is a floor, not a lock: scr_fetch_curl.c's newest requirement is CURLOPT_PROTOCOLS_STR (7.85.0) and the unversioned symbol references bind to any libcurl.so.4. This whole directory leaves with scr_fetch_curl.c when the reference retires for good. -The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub-/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself. +The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub--/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself. ## mbedtls/ @@ -49,4 +49,4 @@ The stub builds lazily on the first flag-selected fetch cross compile into `.cac The TLS provider behind node:tls/node:https (see the design note atop packages/runtime/src/scr_tls.c for why mbedTLS over SecureTransport/BoringSSL/libtls). Only `include/` and `library/` are vendored (LICENSE beside them) — tests, docs, programs, scripts, and CMake machinery are not. No vendored file is modified and no custom config is applied (the stock `mbedtls_config.h` builds every `library/*.c` standalone with clang); to update, re-fetch the release tarball, copy the same two directories, and bump `MBEDTLS_VERSION` in packages/compiler/src/backend/cc.ts. -The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls---/` next to this file — one flavor per lane (plain, asan) and native platform/architecture or explicit cross target, the libqjs.a pattern. TLS-free binaries never compile or link any of this. +The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls----/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity, the libqjs.a pattern. TLS-free binaries never compile or link any of this. diff --git a/tests/harness/README.md b/tests/harness/README.md index bca19c1f1..af9a0338f 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -24,7 +24,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A - **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips clang entirely; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies. - **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A hit skips clang and `ar`. -- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Compiles route through ccache when installed, silently falling back when not. +- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime fingerprint after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not. - **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another. Escape hatches: `SCRIPTC_NO_CACHE=1` bypasses every cache in both directions (no reads, no writes — the run behaves exactly like the uncached path). An explicitly empty `SCRIPTC_CACHE_DIR` does the same; a non-empty value overrides the production default. Eviction is a size-capped LRU sweep over each cache root (`SCRIPTC_CACHE_MAX_MB`, default 4096), run after the first write and periodically in long-lived processes; reads bump mtimes.