fix: harden native build caches

This commit is contained in:
Chris Tate
2026-08-01 15:39:44 -05:00
parent 89b4121ebe
commit 53e07fc55c
4 changed files with 552 additions and 59 deletions
+311 -3
View File
@@ -1,18 +1,21 @@
import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
import { chmod, cp, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { delimiter, join } from "node:path";
import { afterAll, expect, test } from "vitest";
import {
cacheTargetIdentity,
ccVersionOnce,
compileC,
compileLibArchive,
resolveBuildCacheRoot,
runtimeFingerprint,
runtimeSrcDir,
stageRuntimeObjects,
toolchainEnvironmentCachePolicy,
toolchainEnvironmentFingerprint,
vendorCacheBuildIdentity,
vendorCacheTargetFlavor,
} from "./cc.js";
@@ -84,8 +87,53 @@ test("the toolchain environment joins cache identities", () => {
completeArtifacts: false,
runtimeObjects: false,
});
expect(toolchainEnvironmentCachePolicy({ CFLAGS: "-I/headers" })).toEqual({
completeArtifacts: false,
runtimeObjects: false,
});
expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe(
vendorCacheBuildIdentity(base, "compiler-two"),
);
expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe(
vendorCacheBuildIdentity(
toolchainEnvironmentFingerprint({ MACOSX_DEPLOYMENT_TARGET: "11.0" }),
"compiler-one",
),
);
});
test.skipIf(process.platform === "win32")(
"compiler version probes use and remove a private working directory",
async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-version-probe-"));
scratch.push(dir);
const probeTmp = join(dir, "tmp");
const fakeCompiler = join(dir, "fake-zig");
await mkdir(probeTmp);
await writeFile(
fakeCompiler,
`#!/bin/sh
: > a.o
printf 'fake zig cc version probe\n'
`,
);
await chmod(fakeCompiler, 0o755);
const oldTmpdir = process.env["TMPDIR"];
try {
process.env["TMPDIR"] = probeTmp;
// Zig 0.16 creates cwd/a.o for `zig cc --version`. Model that side
// effect directly so this regression does not depend on Zig being
// installed in every generic test shard.
await ccVersionOnce([fakeCompiler, "cc"], `version-probe-${dir}`);
expect(await readdir(probeTmp)).toEqual([]);
} finally {
if (oldTmpdir === undefined) delete process.env["TMPDIR"];
else process.env["TMPDIR"] = oldTmpdir;
}
},
);
test("the runtime fingerprint includes the textually included Ryū sources", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-fingerprint-"));
scratch.push(dir);
@@ -314,6 +362,61 @@ test("mutable compiler inputs bypass caches when files change in place", async (
}
});
test.skipIf(process.platform !== "darwin")(
"vendor object caches separate deployment-target environments",
async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-vendor-environment-"));
scratch.push(dir);
const cacheRoot = join(dir, "cache");
const vendorCacheRoot = join(dir, "vendor-cache");
const cPath = join(dir, "program.c");
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
const oldVendorCacheDir = process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
const oldDeploymentTarget = process.env["MACOSX_DEPLOYMENT_TARGET"];
try {
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = vendorCacheRoot;
delete process.env["SCRIPTC_NO_CACHE"];
await writeFile(cPath, "int main(void) { return 0; }\n");
process.env["MACOSX_DEPLOYMENT_TARGET"] = "14.0";
await compileC({
cPath,
outPath: join(dir, "newer"),
cacheIdentity: TEST_CACHE_IDENTITY,
regex: true,
});
process.env["MACOSX_DEPLOYMENT_TARGET"] = "11.0";
await compileC({
cPath,
outPath: join(dir, "older"),
cacheIdentity: TEST_CACHE_IDENTITY,
regex: true,
});
const lreCaches = (await readdir(vendorCacheRoot)).filter((name) => name.includes("-lre-"));
expect(lreCaches).toHaveLength(2);
const minVersions = lreCaches.map((cache) =>
execFileSync("vtool", ["-show-build", join(vendorCacheRoot, cache, "libregexp.o")], {
encoding: "utf8",
}).match(/minos\s+(\S+)/)?.[1],
);
expect(new Set(minVersions)).toEqual(new Set(["11.0", "14.0"]));
} finally {
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
if (oldVendorCacheDir === undefined) delete process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
else process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = oldVendorCacheDir;
if (oldDeploymentTarget === undefined) delete process.env["MACOSX_DEPLOYMENT_TARGET"];
else process.env["MACOSX_DEPLOYMENT_TARGET"] = oldDeploymentTarget;
}
},
);
test("complete binary hits precede missing vendor prerequisite materialization", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-cache-vendor-hit-"));
scratch.push(dir);
@@ -493,7 +596,18 @@ done
exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
`,
);
await chmod(wrapper, 0o755);
// ccache availability is memoized across this file. If an earlier test
// found it, keep the controlled compiler wrapper in the path by making
// a cache shim that simply forwards its compiler argv.
const ccacheWrapper = join(binDir, "ccache");
await writeFile(
ccacheWrapper,
`#!/bin/sh
if [ "$1" = "--version" ]; then exit 1; fi
exec "$@"
`,
);
await Promise.all([chmod(wrapper, 0o755), chmod(ccacheWrapper, 0o755)]);
await writeFile(cPath, source("one"));
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
process.env["SCRIPTC_TEST_REAL_CLANG"] = originalClang!;
@@ -584,6 +698,129 @@ exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
},
);
test.skipIf(process.platform === "win32")(
"runtime edits during object compilation cannot poison the old fingerprint",
async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-cache-race-"));
scratch.push(dir);
const cacheRoot = join(dir, "cache");
const fakeRuntime = join(dir, "runtime", "src");
const originalRuntime = runtimeSrcDir();
await Promise.all([
cp(originalRuntime, fakeRuntime, { recursive: true }),
mkdir(join(dir, "runtime", "vendor"), { recursive: true }).then(() =>
cp(join(originalRuntime, "..", "vendor", "ryu"), join(dir, "runtime", "vendor", "ryu"), {
recursive: true,
}),
),
]);
const binDir = join(dir, "bin");
const cPath = join(dir, "program.c");
const raceSource = join(fakeRuntime, "scr_number.c");
const signal = join(dir, "runtime-compile-started");
const release = join(dir, "runtime-compile-release");
const marker = "scriptc-runtime-race-marker-unique";
const originalRuntimeSource = await readFile(raceSource, "utf8");
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
const oldRuntimeDir = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
const oldDisableCcache = process.env["SCRIPTC_TEST_DISABLE_CCACHE"];
const oldPath = process.env["PATH"];
const oldRealClang = process.env["SCRIPTC_TEST_REAL_CLANG"];
const oldRaceSource = process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"];
const oldRaceSignal = process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"];
const oldRaceRelease = process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"];
const realClang = (oldPath ?? "")
.split(delimiter)
.map((entry) => join(entry === "" ? process.cwd() : entry, "clang"))
.find((candidate) => existsSync(candidate));
expect(realClang).toBeDefined();
try {
await mkdir(binDir);
const wrapper = join(binDir, "clang");
await writeFile(
wrapper,
`#!/bin/sh
for arg in "$@"; do
if [ "$arg" = "$SCRIPTC_TEST_RUNTIME_RACE_SOURCE" ]; then
: > "$SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"
while [ ! -e "$SCRIPTC_TEST_RUNTIME_RACE_RELEASE" ]; do sleep 0.01; done
break
fi
done
exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
`,
);
await chmod(wrapper, 0o755);
await writeFile(cPath, "int main(void) { return 0; }\n");
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = fakeRuntime;
process.env["SCRIPTC_TEST_REAL_CLANG"] = realClang!;
process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = raceSource;
process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = signal;
process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = release;
// Keep ccache out of this controlled race: its own object store is not
// the cache under test and can obscure which compiler read happened.
process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = "1";
process.env["PATH"] = `${binDir}${delimiter}/usr/bin${delimiter}/bin`;
delete process.env["SCRIPTC_NO_CACHE"];
const firstOut = join(dir, "first");
const firstBuild = compileC({
cPath,
outPath: firstOut,
cacheIdentity: TEST_CACHE_IDENTITY,
});
const deadline = Date.now() + 10_000;
while (!existsSync(signal)) {
if (Date.now() > deadline) throw new Error("timed out waiting for runtime compile");
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
}
await writeFile(
raceSource,
`${originalRuntimeSource}\nconst char scriptc_runtime_race_marker[] = "${marker}";\n`,
);
await writeFile(release, "go");
await firstBuild;
expect((await readFile(firstOut)).includes(Buffer.from(marker))).toBe(true);
// Restore the fingerprint, then force a program-cache miss. The old
// fingerprint must not have retained the object built from edited bytes.
await writeFile(raceSource, originalRuntimeSource);
await writeFile(cPath, "int main(void) { return 0; } /* second */\n");
const secondOut = join(dir, "second");
await compileC({
cPath,
outPath: secondOut,
cacheIdentity: TEST_CACHE_IDENTITY,
});
expect((await readFile(secondOut)).includes(Buffer.from(marker))).toBe(false);
expect(await readdir(join(cacheRoot, "bin"))).toHaveLength(1);
} finally {
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
if (oldRuntimeDir === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
else process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = oldRuntimeDir;
if (oldDisableCcache === undefined) delete process.env["SCRIPTC_TEST_DISABLE_CCACHE"];
else process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = oldDisableCcache;
if (oldPath === undefined) delete process.env["PATH"];
else process.env["PATH"] = oldPath;
if (oldRealClang === undefined) delete process.env["SCRIPTC_TEST_REAL_CLANG"];
else process.env["SCRIPTC_TEST_REAL_CLANG"] = oldRealClang;
if (oldRaceSource === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"];
else process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = oldRaceSource;
if (oldRaceSignal === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"];
else process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = oldRaceSignal;
if (oldRaceRelease === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"];
else process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = oldRaceRelease;
}
},
);
test("explicit native link inputs relink while runtime objects remain cached", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-link-input-cache-"));
scratch.push(dir);
@@ -704,6 +941,77 @@ test("system libraries relink after an in-place rebuild while runtime objects re
}
});
test("damaged runtime objects are rebuilt before linking or archiving", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-object-integrity-"));
scratch.push(dir);
const cacheRoot = join(dir, "cache");
const cPath = join(dir, "program.c");
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
const corruption = Buffer.from("scriptc-corrupt-object-unique-marker");
const objectSets = async (): Promise<string[]> =>
(await readdir(join(cacheRoot, "obj"), { withFileTypes: true }))
.filter((entry) => entry.isDirectory() && !entry.name.startsWith("build-"))
.map((entry) => entry.name);
const corruptOneObject = async (set: string): Promise<string> => {
const objectDir = join(cacheRoot, "obj", set);
const object = (await readdir(objectDir)).find((name) => name.endsWith(".o"));
expect(object).toBeDefined();
await writeFile(join(objectDir, object!), corruption);
expect((await stat(`${join(objectDir, object!)}.sha256`)).isFile()).toBe(true);
return join(objectDir, object!);
};
try {
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
delete process.env["SCRIPTC_NO_CACHE"];
await writeFile(cPath, '#include <stdio.h>\nint main(void) { puts("one"); return 0; }\n');
await compileC({
cPath,
outPath: join(dir, "one"),
cacheIdentity: TEST_CACHE_IDENTITY,
});
const [executableSet] = await objectSets();
const damagedExecutableObject = await corruptOneObject(executableSet!);
await writeFile(cPath, '#include <stdio.h>\nint main(void) { puts("two"); return 0; }\n');
const repairedOut = join(dir, "two");
await compileC({
cPath,
outPath: repairedOut,
cacheIdentity: TEST_CACHE_IDENTITY,
});
expect(execFileSync(repairedOut, { encoding: "utf8" }).trim()).toBe("two");
expect(await readFile(damagedExecutableObject)).not.toEqual(corruption);
const beforeLibrary = new Set(await objectSets());
await writeFile(cPath, "int scriptc_integrity_probe = 1;\n");
await compileLibArchive({
cPath,
outPath: join(dir, "one.lib.a"),
cacheIdentity: TEST_CACHE_IDENTITY,
});
const librarySet = (await objectSets()).find((set) => !beforeLibrary.has(set));
expect(librarySet).toBeDefined();
await corruptOneObject(librarySet!);
await writeFile(cPath, "int scriptc_integrity_probe = 2;\n");
const repairedArchive = join(dir, "two.lib.a");
await compileLibArchive({
cPath,
outPath: repairedArchive,
cacheIdentity: TEST_CACHE_IDENTITY,
});
expect((await readFile(repairedArchive)).includes(corruption)).toBe(false);
} finally {
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
}
});
test("library archives hit by content, invalidate on edits, and reuse runtime objects", async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-cache-"));
scratch.push(dir);
@@ -731,7 +1039,7 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob
const objectSet = objectSets.find((entry) => entry.isDirectory() && !entry.name.startsWith("build-"));
expect(objectSet).toBeDefined();
const objectDir = join(cacheRoot, "obj", objectSet!.name);
const objectNames = await readdir(objectDir);
const objectNames = (await readdir(objectDir)).filter((name) => name.endsWith(".o"));
expect(objectNames.length).toBeGreaterThan(10);
const pinnedTime = new Date("2000-01-01T00:00:00.000Z");
await Promise.all(objectNames.map((name) => utimes(join(objectDir, name), pinnedTime, pinnedTime)));
+236 -51
View File
@@ -59,6 +59,14 @@ const TOOLCHAIN_ENV_KEYS = [
"CCC_ADD_ARGS",
"CLANG_CONFIG_FILE_SYSTEM_DIR",
"CLANG_CONFIG_FILE_USER_DIR",
"CC",
"CFLAGS",
"CPPFLAGS",
"LDFLAGS",
"AR",
"RANLIB",
"CMAKE_GENERATOR",
"CMAKE_TOOLCHAIN_FILE",
"SOURCE_DATE_EPOCH",
"ZERO_AR_DATE",
"LANG",
@@ -90,6 +98,17 @@ const MUTABLE_COMPILE_ENV_KEYS = [
"CCC_ADD_ARGS",
"CLANG_CONFIG_FILE_SYSTEM_DIR",
"CLANG_CONFIG_FILE_USER_DIR",
// The direct runtime build does not consume the conventional Make/CMake
// variables, but the lazily-built QuickJS prerequisite does. Their values
// can name mutable tools, headers, libraries, and toolchain files, so a
// stable spelling is not a safe persistent-cache identity.
"CC",
"CFLAGS",
"CPPFLAGS",
"LDFLAGS",
"AR",
"RANLIB",
"CMAKE_TOOLCHAIN_FILE",
] as const;
/** These variables only redirect link-time inputs. Runtime objects remain
@@ -316,6 +335,8 @@ export class CcCompileError extends Error {
}
export function runtimeSrcDir(): string {
const testRoot = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
if (testRoot !== undefined) return resolve(testRoot);
const require = createRequire(import.meta.url);
return join(dirname(require.resolve("@scriptc/runtime/package.json")), "src");
}
@@ -454,12 +475,60 @@ export function vendorCacheTargetFlavor(
return driver.target === null ? `native-${hostPlatform}-${hostArch}` : driver.target;
}
/** Vendor prerequisites live outside the content-addressed build root, so
* their directory name must carry the same environment and resolved compiler
* identity as the artifacts that consume them. The short digest keeps the
* package-local path readable while preventing build-order contamination. */
export function vendorCacheBuildIdentity(
environmentFingerprint: string,
compilerIdentity: string,
): string {
return createHash("sha256")
.update("vendor-toolchain-v1\0")
.update(environmentFingerprint)
.update("\0")
.update(compilerIdentity)
.digest("hex")
.slice(0, 20);
}
const vendorCompilerIdentityFallbacks = new Map<string, string>();
async function currentVendorCacheBuildIdentity(
driver: Pick<CcDriver, "argv" | "target">,
environmentFingerprint: string,
): Promise<string> {
// Native vendor recipes additionally use bare clang/ar and CMake; cross
// recipes use the zig driver for compilation and `zig ar`. Include every
// executable that can affect the package-local prerequisite, not just the
// final program's driver.
const commands = [
driver.argv[0] ?? "clang",
...(driver.target === null ? ["clang", "cc", "ar", "ranlib", "cmake"] : []),
].filter((command, index, all) => all.indexOf(command) === index);
const identities = await Promise.all(
commands.map(async (command) => {
const spellingKey = `${environmentFingerprint}\0${command}`;
const resolved = await resolvedToolIdentity(command);
if (resolved !== null) vendorCompilerIdentityFallbacks.set(spellingKey, resolved);
return [
command,
resolved ??
vendorCompilerIdentityFallbacks.get(spellingKey) ??
`<unresolved>\0${command}`,
].join("\0");
}),
);
return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"));
}
function engineArchivePath(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string {
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
return join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-${flavor}`, "libqjs.a");
}
@@ -493,7 +562,7 @@ async function installArtifact(
}
/** The engine archive for one flavor, built lazily on the first --dynamic
* compile (~10s) and cached under vendor/.cache/<commit>-<flavor>-<target>/ — unlike
* compile (~10s) and cached under vendor/.cache/<commit>-<flavor>-<target>-<toolchain>/ — unlike
* the runtime's own sources (recompiled every build, ~100ms), the engine is
* far too big to rebuild per compile. The plain flavor is ALWAYS MinSizeRel
* regardless of the program's optimization level: measured binary delta is
@@ -513,11 +582,12 @@ async function installArtifact(
async function ensureEngineArchive(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string> {
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const vendor = vendorEngineDir();
const archive = engineArchivePath(sanitize, driver, cacheRoot);
const archive = engineArchivePath(sanitize, driver, buildIdentity, cacheRoot);
const cacheDir = dirname(archive);
if (await fileExists(archive)) return archive;
if (driver.target !== null) return buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir);
@@ -638,19 +708,20 @@ const LRE_SOURCES = ["libregexp.c", "libunicode.c"];
function lreObjectPaths(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string[] {
const flavor =
(sanitize ? "asan" : "plain") +
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
`-${vendorCacheTargetFlavor(driver)}`;
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const cacheDir = join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-lre-${flavor}`);
return LRE_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o")));
}
/** The libregexp objects for one flavor, compiled lazily on the first
* regex-using static build (~1s) and cached like the engine archive —
* vendor/.cache/<commit>-lre-<flavor>/*.o — with the same atomic-rename
* vendor/.cache/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
* publish (parallel first builds race safely; losers discard their work).
* Plain is -Os (size class matters: the regex fence pins regex-using
* binaries well under the engine class); asan matches the final link so
@@ -662,6 +733,7 @@ function lreObjectPaths(
async function ensureLreObjects(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string[]> {
// The flavor keys the DRIVER as well as the target: a zig-cc-built object
@@ -671,9 +743,9 @@ async function ensureLreObjects(
const flavor =
(sanitize ? "asan" : "plain") +
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
`-${vendorCacheTargetFlavor(driver)}`;
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const vendor = vendorEngineDir();
const objects = lreObjectPaths(sanitize, driver, cacheRoot);
const objects = lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
const cacheDir = dirname(objects[0]!);
if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects;
@@ -721,19 +793,20 @@ const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infbac
function zlibObjectPaths(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string[] {
const flavor =
(sanitize ? "asan" : "plain") +
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
`-${vendorCacheTargetFlavor(driver)}`;
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const cacheDir = join(cacheRoot, `zlib-${ZLIB_VERSION}-${flavor}`);
return ZLIB_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o")));
}
/** The zlib objects for one flavor, compiled lazily on the first zlib-using
* CROSS build (~1s) and cached like the lre objects —
* vendor/.cache/zlib-<version>-<flavor>/*.o — with the same atomic-rename
* vendor/.cache/zlib-<version>-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
* publish (parallel first builds race safely; losers discard their work).
* Plain is -Os, asan matches the final link so the sanitized lane
* instruments the codec too. The flavor keys the driver and target exactly
@@ -743,14 +816,15 @@ function zlibObjectPaths(
async function ensureZlibObjects(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string[]> {
const flavor =
(sanitize ? "asan" : "plain") +
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
`-${vendorCacheTargetFlavor(driver)}`;
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const vendor = vendorZlibDir();
const objects = zlibObjectPaths(sanitize, driver, cacheRoot);
const objects = zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
const cacheDir = dirname(objects[0]!);
if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects;
@@ -802,14 +876,15 @@ const CURL_STUB_SYMBOLS = [
function curlStubDirPath(
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string {
return join(cacheRoot, `curl-stub-${driver.target}`);
return join(cacheRoot, `curl-stub-${driver.target}-${buildIdentity}`);
}
/** The libcurl import stub for one CROSS target, generated lazily on the
* first fetch-using cross build and cached like the zlib objects —
* vendor/.cache/curl-stub-<target>/libcurl.so, atomic-rename publish. The
* vendor/.cache/curl-stub-<target>-<toolchain>/libcurl.so, atomic-rename publish. The
* stub is empty definitions of CURL_STUB_SYMBOLS compiled `-shared` with
* `-soname libcurl.so.4`: linking against it satisfies scr_fetch_curl.c's
* references and records a plain `DT_NEEDED libcurl.so.4`, which the
@@ -820,9 +895,10 @@ function curlStubDirPath(
* and asan links against plain shared libraries freely. */
async function ensureCurlStub(
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string> {
const cacheDir = curlStubDirPath(driver, cacheRoot);
const cacheDir = curlStubDirPath(driver, buildIdentity, cacheRoot);
const lib = join(cacheDir, "libcurl.so");
if (await fileExists(lib)) return cacheDir;
@@ -860,15 +936,16 @@ function vendorTlsDir(): string {
function tlsArchivePath(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string {
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
return join(cacheRoot, `mbedtls-${MBEDTLS_VERSION}-${flavor}`, "libmbedtls.a");
}
/** The mbedTLS archive for one flavor, compiled lazily on the first
* TLS-using build (~15s over ~110 TUs, parallelized) and cached like the
* engine archive — vendor/.cache/mbedtls-<version>-<flavor>-<target>/libmbedtls.a —
* engine archive — vendor/.cache/mbedtls-<version>-<flavor>-<target>-<toolchain>/libmbedtls.a —
* with the same atomic-rename publish (parallel first builds race safely;
* losers discard their work). Plain is -Os (the TLS stack is link-gated
* but still wants its size class small); asan matches the final link so
@@ -884,13 +961,14 @@ function tlsArchivePath(
async function ensureTlsArchive(
sanitize: boolean,
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string> {
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const compileArgv = driver.target !== null ? driver.argv : ["clang"];
const arArgv = driver.target !== null ? [...driver.argv.slice(0, 1), "ar"] : ["ar"];
const vendor = vendorTlsDir();
const archive = tlsArchivePath(sanitize, driver, cacheRoot);
const archive = tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot);
const cacheDir = dirname(archive);
if (await fileExists(archive)) return archive;
@@ -1031,6 +1109,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
}
}
const toolchainEnv = toolchainEnvironmentFingerprint();
const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv);
let cachedArchive: string | null = null;
let compilerVersion = "";
let runtimeHash = "";
@@ -1047,7 +1126,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
runtimeHash = fingerprint;
cachedProgramBytes = programBytes;
const key = createHash("sha256")
.update("lib-v3\0")
// v4 invalidates archives that may have been assembled from an
// unverified or cross-environment vendor/runtime object cache.
.update("lib-v4\0")
.update(cacheTargetIdentity(driver)).update("\0")
.update(toolchainEnv).update("\0")
.update(opts.cacheIdentity!).update("\0")
@@ -1097,10 +1178,10 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
try {
const lreObjects = regex
? await ensureLreObjects(sanitize, driver, vendorCacheRoot)
? await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: [];
const zlibObjects = opts.zlib
? await ensureZlibObjects(sanitize, driver, vendorCacheRoot)
? await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: [];
const buildDir = await mkdtemp(join(tmpdir(), "scriptc-lib-"));
try {
@@ -1147,6 +1228,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
cachedProgramBytes === null ? undefined : opts.cPath,
);
let runtimeObjects: string[] | null = null;
let cacheInputsStable = true;
if (root !== null && compilerVersion !== "" && runtimeHash !== "") {
try {
const sourcePaths = sources.map((f) => join(rtDir, f));
@@ -1155,11 +1237,13 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
driver.argv,
cflags,
sourcePaths,
`lib-obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`,
`lib-obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`,
async () => (await runtimeFingerprint(rtDir)) === runtimeHash,
);
const staged = await stageRuntimeObjects(cached, join(buildDir, "cached-runtime"));
runtimeObjects = sourcePaths.map((path) => staged.get(path)!);
} catch {
} catch (err) {
if (err instanceof CacheInputsChangedError) cacheInputsStable = false;
runtimeObjects = null;
}
}
@@ -1191,7 +1275,11 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
...objects,
]);
if (archiveOutput !== opts.outPath) await installArtifact(archiveOutput, opts.outPath);
if (cachedArchive !== null && root !== null) {
const runtimeStillMatchesKey =
cacheInputsStable &&
runtimeHash !== "" &&
(await runtimeFingerprint(rtDir).catch(() => null)) === runtimeHash;
if (cachedArchive !== null && root !== null && runtimeStillMatchesKey) {
try {
const cacheDir = dirname(cachedArchive);
await mkdir(cacheDir, { recursive: true });
@@ -1367,7 +1455,7 @@ async function resolvedToolIdentity(
return null;
}
async function ccVersionOnce(
export async function ccVersionOnce(
argv: string[],
environmentFingerprint: string = toolchainEnvironmentFingerprint(),
): Promise<string> {
@@ -1384,11 +1472,22 @@ async function ccVersionOnce(
const key = `${spellingKey}\0${executableIdentity ?? "<unresolved>"}`;
let memo = ccVersionMemos.get(key);
if (memo === undefined) {
// cwd: the version probe must not touch the caller's directory —
// `zig cc --version` (zig 0.16) drops an empty a.o wherever it runs.
memo = execFileAsync(argv[0] ?? "clang", [...argv.slice(1), "--version"], { cwd: tmpdir() }).then(
(r) => `${executableIdentity ?? "<unresolved>"}\0${`${r.stdout}\n${r.stderr}`.trim()}`,
);
// `zig cc --version` (zig 0.16) drops an empty a.o in its cwd. A private
// probe directory avoids both caller pollution and the cross-process /
// cross-user collision a fixed tmpdir()/a.o would create on Linux.
memo = (async () => {
const probeDir = await mkdtemp(join(tmpdir(), "scriptc-cc-version-"));
try {
const r = await execFileAsync(
argv[0] ?? "clang",
[...argv.slice(1), "--version"],
{ cwd: probeDir },
);
return `${executableIdentity ?? "<unresolved>"}\0${`${r.stdout}\n${r.stderr}`.trim()}`;
} finally {
await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
}
})();
ccVersionMemos.set(key, memo);
}
if (executableIdentity !== null) ccVersionFallbacks.set(spellingKey, memo);
@@ -1456,15 +1555,55 @@ export async function runtimeFingerprint(rtDir: string): Promise<string> {
return h.digest("hex");
}
class CacheInputsChangedError extends Error {
constructor() {
super("runtime inputs changed while populating the native object cache");
this.name = "CacheInputsChangedError";
}
}
function cacheDigestPath(path: string): string {
return `${path}.sha256`;
}
async function fileDigest(path: string): Promise<string> {
return createHash("sha256").update(await readFile(path)).digest("hex");
}
/** Cache entries are disposable data, not trusted compiler output. Atomic
* rename prevents ordinary partial writes, while the adjacent digest catches
* disk damage, manual truncation, and entries left by an interrupted/older
* publisher before they reach clang, the linker, or `ar`. */
async function validCachedFile(object: string): Promise<boolean> {
try {
const digestPath = cacheDigestPath(object);
const expected = (await readFile(digestPath, "utf8")).trim();
const valid = /^[0-9a-f]{64}$/.test(expected) && (await fileDigest(object)) === expected;
if (valid) {
const now = new Date();
await Promise.all([
utimes(object, now, now).catch(() => undefined),
utimes(digestPath, now, now).catch(() => undefined),
]);
}
return valid;
} catch {
return false;
}
}
/** The cached .o set for one flag flavor, compiled on first need. Concurrent
* first builds (parallel test workers on a cold cache) may duplicate work;
* per-file atomic renames make every winner equivalent. */
* per-file atomic renames make every winner equivalent. Publication is held
* until verifyInputs confirms that the source/header fingerprint used by the
* key still describes the bytes clang just read. */
async function ensureRuntimeObjects(
root: string,
ccArgv: string[],
cflags: string[],
sources: string[],
keyPrefix: string,
verifyInputs: () => Promise<boolean>,
): Promise<Map<string, string>> {
const setKey = createHash("sha256")
.update(keyPrefix)
@@ -1473,15 +1612,20 @@ async function ensureRuntimeObjects(
.slice(0, 24);
const objDir = join(root, "obj", setKey);
const objOf = (src: string): string => join(objDir, `${basename(src, ".c")}.o`);
const present = await Promise.all(sources.map((s) => fileExists(objOf(s))));
const present = await Promise.all(sources.map((s) => validCachedFile(objOf(s))));
const missing = sources.filter((_, i) => !present[i]);
if (missing.length > 0) {
// ccache wraps only the default clang driver — multi-word drivers
// (`zig cc`) run bare; their object sets are keyed apart anyway.
const useCcache = ccArgv.length === 1 && ccArgv[0] === "clang" && (await ccacheAvailable());
const useCcache =
process.env["SCRIPTC_TEST_DISABLE_CCACHE"] !== "1" &&
ccArgv.length === 1 &&
ccArgv[0] === "clang" &&
(await ccacheAvailable());
await mkdir(objDir, { recursive: true });
const tmpDir = await mkdtemp(join(root, "obj", "build-"));
try {
const compiled = new Map<string, { object: string; digest: string }>();
// Modest parallelism: a flavor's objects build once, but several cold
// workers can race here — keep each build's CPU footprint small.
const width = 4;
@@ -1491,15 +1635,33 @@ async function ensureRuntimeObjects(
const tmpObj = join(tmpDir, `${basename(src, ".c")}.o`);
const argv = [...(useCcache ? ["ccache"] : []), ...ccArgv, ...cflags, "-c", src, "-o", tmpObj];
await execFileAsync(argv[0] ?? "clang", argv.slice(1));
await rename(tmpObj, objOf(src));
compiled.set(src, { object: tmpObj, digest: await fileDigest(tmpObj) });
}),
);
}
// The fingerprint was computed before these subprocesses started. Do
// not place their outputs under that key if a checkout/package update
// changed any runtime source or included header while clang was reading.
if (!(await verifyInputs())) throw new CacheInputsChangedError();
for (const [src, built] of compiled) {
const destination = objOf(src);
const tmpDigest = `${built.object}.sha256`;
await writeFile(tmpDigest, `${built.digest}\n`);
// Publish data first and its verifier second. A concurrent reader in
// the tiny gap sees a missing/mismatched digest and falls back safely.
await rename(built.object, destination);
await rename(tmpDigest, cacheDigestPath(destination));
}
} finally {
await rm(tmpDir, { recursive: true, force: true });
}
}
return new Map(sources.map((s) => [s, objOf(s)]));
if (!(await verifyInputs())) throw new CacheInputsChangedError();
const objects = new Map(sources.map((s) => [s, objOf(s)]));
if (!(await Promise.all([...objects.values()].map(validCachedFile))).every(Boolean)) {
throw new Error("native object cache integrity check failed");
}
return objects;
}
/** Give one active link/archive operation private names for its cached
@@ -1643,6 +1805,8 @@ export async function compileC(opts: CcOptions): Promise<void> {
}
}
const cachePolicy = toolchainEnvironmentCachePolicy();
const toolchainEnv = toolchainEnvironmentFingerprint();
const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv);
// Mutable include/SDK/config inputs can change behind a stable environment
// spelling. The complete/runtime cache is disabled below; vendor objects
// must follow the same rule instead of silently surviving in package-local
@@ -1658,34 +1822,48 @@ export async function compileC(opts: CcOptions): Promise<void> {
// Every vendor output path is deterministic from pins, flags, driver, and
// target. Build the command/key from those paths now, but do not materialize
// them until a complete-binary lookup has missed.
const engineArchive = dynamic ? engineArchivePath(sanitize, driver, vendorCacheRoot) : null;
const tlsArchive = tls ? tlsArchivePath(sanitize, driver, vendorCacheRoot) : null;
const engineArchive = dynamic
? engineArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: null;
const tlsArchive = tls
? tlsArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: null;
// --dynamic + regex shares the archive's libregexp (its host hooks and
// ours would collide; see scr_regex.c) — the standalone objects are for
// static builds only.
const lreObjects = regex && !dynamic ? lreObjectPaths(sanitize, driver, vendorCacheRoot) : [];
const lreObjects = regex && !dynamic
? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: [];
// Vendored zlib is the CROSS story only — host builds keep the exact
// historical `-lz` system link (see CcOptions.zlib). The native fetch's
// gzip decoder rides the same objects/link.
const zlibObjects =
((opts.zlib ?? false) || nativeFetch) && driver.target !== null
? zlibObjectPaths(sanitize, driver, vendorCacheRoot)
? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: [];
// The libcurl import stub is likewise CROSS-only — host builds keep the
// exact historical system `-lcurl` link (see CcOptions.fetch). Curl
// reference builds only.
const curlStubDir =
curlFetch && driver.target !== null
? curlStubDirPath(driver, vendorCacheRoot)
? curlStubDirPath(driver, vendorBuildIdentity, vendorCacheRoot)
: null;
const materializeVendorPrerequisites = async (): Promise<void> => {
// Preserve the historical order to avoid multiplying first-build resource
// pressure when several large vendor sets are cold simultaneously.
if (dynamic) await ensureEngineArchive(sanitize, driver, vendorCacheRoot);
if (tls) await ensureTlsArchive(sanitize, driver, vendorCacheRoot);
if (regex && !dynamic) await ensureLreObjects(sanitize, driver, vendorCacheRoot);
if (zlibObjects.length > 0) await ensureZlibObjects(sanitize, driver, vendorCacheRoot);
if (curlStubDir !== null) await ensureCurlStub(driver, vendorCacheRoot);
if (dynamic) {
await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
}
if (tls) await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
if (regex && !dynamic) {
await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
}
if (zlibObjects.length > 0) {
await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
}
if (curlStubDir !== null) {
await ensureCurlStub(driver, vendorBuildIdentity, vendorCacheRoot);
}
};
// rt() maps each runtime source's path on the command line: identity for
// the historical single invocation, cached-.o substitution on cache misses.
@@ -1925,7 +2103,6 @@ export async function compileC(opts: CcOptions): Promise<void> {
return;
}
const toolchainEnv = toolchainEnvironmentFingerprint();
const [cv, fingerprint, cBytes] = await Promise.all([
ccVersionOnce(driver.argv, toolchainEnv),
runtimeFingerprint(rtDir),
@@ -1952,7 +2129,9 @@ export async function compileC(opts: CcOptions): Promise<void> {
a === opts.cPath ? "<program.c>" : a === opts.outPath ? "<out>" : a,
);
const key = createHash("sha256")
.update("bin-v3\0")
// v4 invalidates binaries that may have been published while runtime
// inputs changed underneath the former per-file object publisher.
.update("bin-v4\0")
.update(cacheTargetIdentity(driver)).update("\0")
.update(toolchainEnv).update("\0")
.update(opts.cacheIdentity!).update("\0")
@@ -2030,16 +2209,19 @@ export async function compileC(opts: CcOptions): Promise<void> {
await writeFile(programPath, cBytes);
let objects: Map<string, string> | null = null;
let cacheInputsStable = true;
try {
const cached = await ensureRuntimeObjects(
root,
driver.argv,
cflags,
rtInputs,
`obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`,
`obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`,
async () => (await runtimeFingerprint(rtDir)) === fingerprint,
);
objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects"));
} catch {
} catch (err) {
if (err instanceof CacheInputsChangedError) cacheInputsStable = false;
objects = null; // cache trouble is never a build failure
}
@@ -2051,7 +2233,10 @@ export async function compileC(opts: CcOptions): Promise<void> {
);
await installArtifact(privateOut, opts.outPath);
if (cachedBin !== null && keyHex !== null) {
cacheInputsStable =
cacheInputsStable &&
(await runtimeFingerprint(rtDir).catch(() => null)) === fingerprint;
if (cachedBin !== null && keyHex !== null && cacheInputsStable) {
try {
await mkdir(binDir, { recursive: true });
const tmp = join(binDir, `.tmp-${keyHex.slice(0, 8)}-${process.pid}-${Math.random().toString(36).slice(2)}`);
+4 -4
View File
@@ -19,7 +19,7 @@ The QuickJS-ng JavaScript engine, embedded by the opt-in `--dynamic` build mode
The tree is a plain snapshot of the upstream commit with directories the library build does not need removed (tests/, docs/, examples/, test262 fixtures, CI config, generator scripts). No vendored file is modified; to update, re-clone upstream at the new commit, delete its .git directory, apply the same trim, and update this file.
The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/<commit>-<flavor>/` next to this file — one flavor per lane (plain, asan). The cache directory is gitignored and safe to delete.
The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/<commit>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity. The cache directory is gitignored and safe to delete.
## zlib/
@@ -29,7 +29,7 @@ The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile i
The CROSS-target arm of node:zlib (see packages/runtime/src/scr_zlib.c): host builds keep the historical system `-lz` link (macOS ships libz), but zig's cross sysroots have no libz, so SCRIPTC_TARGET builds compile this vendored copy per target instead. Only the flat `*.c`/`*.h` at the distribution root are vendored (LICENSE beside them) — contrib, tests, build machinery, and docs are not. No vendored file is modified; the gz* file-I/O TUs are vendored for faithfulness but never compiled (nothing references the gzFile API — see ZLIB_SOURCES in packages/compiler/src/backend/cc.ts). To update, re-fetch the release tarball, copy the same flat set, and bump `ZLIB_VERSION` in cc.ts.
The objects build lazily on the first zlib-using cross compile into `.cache/zlib-<version>-<flavor>/` next to this file — one flavor per driver/target (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation.
The objects build lazily on the first zlib-using cross compile into `.cache/zlib-<version>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per driver/target and compiler environment/identity (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation.
## curl/
@@ -39,7 +39,7 @@ The objects build lazily on the first zlib-using cross compile into `.cache/zlib
HEADERS ONLY — no curl C source is vendored and none is ever compiled. These headers now serve ONLY the RETIRED curl reference implementation of fetch (packages/runtime/src/scr_fetch_curl.c, selected by `SCRIPTC_FETCH_CURL=1`; kept compilable for one release as the native flip's reference — see scr_fetch.c, the default, which rides scr_net/scr_tls/scr_http/zlib and touches nothing here). Under the flag: host builds link the system `-lcurl` (macOS ships libcurl), and linux-gnu CROSS builds compile scr_fetch_curl.c against these headers, then link against a generated STUB `libcurl.so` (soname `libcurl.so.4`, empty definitions of exactly the symbols scr_fetch_curl.c calls — see CURL_STUB_SYMBOLS in packages/compiler/src/backend/cc.ts) so the produced binary records a plain `DT_NEEDED libcurl.so.4` that the TARGET system's real libcurl satisfies at load time, the standard cross-link import-stub technique. The 7.88.1 pin is a floor, not a lock: scr_fetch_curl.c's newest requirement is CURLOPT_PROTOCOLS_STR (7.85.0) and the unversioned symbol references bind to any libcurl.so.4. This whole directory leaves with scr_fetch_curl.c when the reference retires for good.
The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub-<flavor>/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself.
The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub-<target>-<toolchain>/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself.
## mbedtls/
@@ -49,4 +49,4 @@ The stub builds lazily on the first flag-selected fetch cross compile into `.cac
The TLS provider behind node:tls/node:https (see the design note atop packages/runtime/src/scr_tls.c for why mbedTLS over SecureTransport/BoringSSL/libtls). Only `include/` and `library/` are vendored (LICENSE beside them) — tests, docs, programs, scripts, and CMake machinery are not. No vendored file is modified and no custom config is applied (the stock `mbedtls_config.h` builds every `library/*.c` standalone with clang); to update, re-fetch the release tarball, copy the same two directories, and bump `MBEDTLS_VERSION` in packages/compiler/src/backend/cc.ts.
The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls-<version>-<flavor>-<target>/` next to this file — one flavor per lane (plain, asan) and native platform/architecture or explicit cross target, the libqjs.a pattern. TLS-free binaries never compile or link any of this.
The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls-<version>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity, the libqjs.a pattern. TLS-free binaries never compile or link any of this.
+1 -1
View File
@@ -24,7 +24,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A
- **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips clang entirely; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies.
- **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A hit skips clang and `ar`.
- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Compiles route through ccache when installed, silently falling back when not.
- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime fingerprint after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not.
- **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another.
Escape hatches: `SCRIPTC_NO_CACHE=1` bypasses every cache in both directions (no reads, no writes — the run behaves exactly like the uncached path). An explicitly empty `SCRIPTC_CACHE_DIR` does the same; a non-empty value overrides the production default. Eviction is a size-capped LRU sweep over each cache root (`SCRIPTC_CACHE_MAX_MB`, default 4096), run after the first write and periodically in long-lived processes; reads bump mtimes.