mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-24 06:14:01 +08:00
fix: harden native build caches
This commit is contained in:
@@ -1,18 +1,21 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
|
||||
import { chmod, cp, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { afterAll, expect, test } from "vitest";
|
||||
import {
|
||||
cacheTargetIdentity,
|
||||
ccVersionOnce,
|
||||
compileC,
|
||||
compileLibArchive,
|
||||
resolveBuildCacheRoot,
|
||||
runtimeFingerprint,
|
||||
runtimeSrcDir,
|
||||
stageRuntimeObjects,
|
||||
toolchainEnvironmentCachePolicy,
|
||||
toolchainEnvironmentFingerprint,
|
||||
vendorCacheBuildIdentity,
|
||||
vendorCacheTargetFlavor,
|
||||
} from "./cc.js";
|
||||
|
||||
@@ -84,8 +87,53 @@ test("the toolchain environment joins cache identities", () => {
|
||||
completeArtifacts: false,
|
||||
runtimeObjects: false,
|
||||
});
|
||||
expect(toolchainEnvironmentCachePolicy({ CFLAGS: "-I/headers" })).toEqual({
|
||||
completeArtifacts: false,
|
||||
runtimeObjects: false,
|
||||
});
|
||||
|
||||
expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe(
|
||||
vendorCacheBuildIdentity(base, "compiler-two"),
|
||||
);
|
||||
expect(vendorCacheBuildIdentity(base, "compiler-one")).not.toBe(
|
||||
vendorCacheBuildIdentity(
|
||||
toolchainEnvironmentFingerprint({ MACOSX_DEPLOYMENT_TARGET: "11.0" }),
|
||||
"compiler-one",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === "win32")(
|
||||
"compiler version probes use and remove a private working directory",
|
||||
async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-version-probe-"));
|
||||
scratch.push(dir);
|
||||
const probeTmp = join(dir, "tmp");
|
||||
const fakeCompiler = join(dir, "fake-zig");
|
||||
await mkdir(probeTmp);
|
||||
await writeFile(
|
||||
fakeCompiler,
|
||||
`#!/bin/sh
|
||||
: > a.o
|
||||
printf 'fake zig cc version probe\n'
|
||||
`,
|
||||
);
|
||||
await chmod(fakeCompiler, 0o755);
|
||||
const oldTmpdir = process.env["TMPDIR"];
|
||||
try {
|
||||
process.env["TMPDIR"] = probeTmp;
|
||||
// Zig 0.16 creates cwd/a.o for `zig cc --version`. Model that side
|
||||
// effect directly so this regression does not depend on Zig being
|
||||
// installed in every generic test shard.
|
||||
await ccVersionOnce([fakeCompiler, "cc"], `version-probe-${dir}`);
|
||||
expect(await readdir(probeTmp)).toEqual([]);
|
||||
} finally {
|
||||
if (oldTmpdir === undefined) delete process.env["TMPDIR"];
|
||||
else process.env["TMPDIR"] = oldTmpdir;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test("the runtime fingerprint includes the textually included Ryū sources", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-fingerprint-"));
|
||||
scratch.push(dir);
|
||||
@@ -314,6 +362,61 @@ test("mutable compiler inputs bypass caches when files change in place", async (
|
||||
}
|
||||
});
|
||||
|
||||
test.skipIf(process.platform !== "darwin")(
|
||||
"vendor object caches separate deployment-target environments",
|
||||
async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-vendor-environment-"));
|
||||
scratch.push(dir);
|
||||
const cacheRoot = join(dir, "cache");
|
||||
const vendorCacheRoot = join(dir, "vendor-cache");
|
||||
const cPath = join(dir, "program.c");
|
||||
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
|
||||
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
|
||||
const oldVendorCacheDir = process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
|
||||
const oldDeploymentTarget = process.env["MACOSX_DEPLOYMENT_TARGET"];
|
||||
|
||||
try {
|
||||
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
|
||||
process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = vendorCacheRoot;
|
||||
delete process.env["SCRIPTC_NO_CACHE"];
|
||||
await writeFile(cPath, "int main(void) { return 0; }\n");
|
||||
|
||||
process.env["MACOSX_DEPLOYMENT_TARGET"] = "14.0";
|
||||
await compileC({
|
||||
cPath,
|
||||
outPath: join(dir, "newer"),
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
regex: true,
|
||||
});
|
||||
process.env["MACOSX_DEPLOYMENT_TARGET"] = "11.0";
|
||||
await compileC({
|
||||
cPath,
|
||||
outPath: join(dir, "older"),
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
regex: true,
|
||||
});
|
||||
|
||||
const lreCaches = (await readdir(vendorCacheRoot)).filter((name) => name.includes("-lre-"));
|
||||
expect(lreCaches).toHaveLength(2);
|
||||
const minVersions = lreCaches.map((cache) =>
|
||||
execFileSync("vtool", ["-show-build", join(vendorCacheRoot, cache, "libregexp.o")], {
|
||||
encoding: "utf8",
|
||||
}).match(/minos\s+(\S+)/)?.[1],
|
||||
);
|
||||
expect(new Set(minVersions)).toEqual(new Set(["11.0", "14.0"]));
|
||||
} finally {
|
||||
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
|
||||
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
|
||||
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
|
||||
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
|
||||
if (oldVendorCacheDir === undefined) delete process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"];
|
||||
else process.env["SCRIPTC_TEST_VENDOR_CACHE_DIR"] = oldVendorCacheDir;
|
||||
if (oldDeploymentTarget === undefined) delete process.env["MACOSX_DEPLOYMENT_TARGET"];
|
||||
else process.env["MACOSX_DEPLOYMENT_TARGET"] = oldDeploymentTarget;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test("complete binary hits precede missing vendor prerequisite materialization", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-cache-vendor-hit-"));
|
||||
scratch.push(dir);
|
||||
@@ -493,7 +596,18 @@ done
|
||||
exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
|
||||
`,
|
||||
);
|
||||
await chmod(wrapper, 0o755);
|
||||
// ccache availability is memoized across this file. If an earlier test
|
||||
// found it, keep the controlled compiler wrapper in the path by making
|
||||
// a cache shim that simply forwards its compiler argv.
|
||||
const ccacheWrapper = join(binDir, "ccache");
|
||||
await writeFile(
|
||||
ccacheWrapper,
|
||||
`#!/bin/sh
|
||||
if [ "$1" = "--version" ]; then exit 1; fi
|
||||
exec "$@"
|
||||
`,
|
||||
);
|
||||
await Promise.all([chmod(wrapper, 0o755), chmod(ccacheWrapper, 0o755)]);
|
||||
await writeFile(cPath, source("one"));
|
||||
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
|
||||
process.env["SCRIPTC_TEST_REAL_CLANG"] = originalClang!;
|
||||
@@ -584,6 +698,129 @@ exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
|
||||
},
|
||||
);
|
||||
|
||||
test.skipIf(process.platform === "win32")(
|
||||
"runtime edits during object compilation cannot poison the old fingerprint",
|
||||
async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-runtime-cache-race-"));
|
||||
scratch.push(dir);
|
||||
const cacheRoot = join(dir, "cache");
|
||||
const fakeRuntime = join(dir, "runtime", "src");
|
||||
const originalRuntime = runtimeSrcDir();
|
||||
await Promise.all([
|
||||
cp(originalRuntime, fakeRuntime, { recursive: true }),
|
||||
mkdir(join(dir, "runtime", "vendor"), { recursive: true }).then(() =>
|
||||
cp(join(originalRuntime, "..", "vendor", "ryu"), join(dir, "runtime", "vendor", "ryu"), {
|
||||
recursive: true,
|
||||
}),
|
||||
),
|
||||
]);
|
||||
|
||||
const binDir = join(dir, "bin");
|
||||
const cPath = join(dir, "program.c");
|
||||
const raceSource = join(fakeRuntime, "scr_number.c");
|
||||
const signal = join(dir, "runtime-compile-started");
|
||||
const release = join(dir, "runtime-compile-release");
|
||||
const marker = "scriptc-runtime-race-marker-unique";
|
||||
const originalRuntimeSource = await readFile(raceSource, "utf8");
|
||||
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
|
||||
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
|
||||
const oldRuntimeDir = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
|
||||
const oldDisableCcache = process.env["SCRIPTC_TEST_DISABLE_CCACHE"];
|
||||
const oldPath = process.env["PATH"];
|
||||
const oldRealClang = process.env["SCRIPTC_TEST_REAL_CLANG"];
|
||||
const oldRaceSource = process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"];
|
||||
const oldRaceSignal = process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"];
|
||||
const oldRaceRelease = process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"];
|
||||
const realClang = (oldPath ?? "")
|
||||
.split(delimiter)
|
||||
.map((entry) => join(entry === "" ? process.cwd() : entry, "clang"))
|
||||
.find((candidate) => existsSync(candidate));
|
||||
expect(realClang).toBeDefined();
|
||||
|
||||
try {
|
||||
await mkdir(binDir);
|
||||
const wrapper = join(binDir, "clang");
|
||||
await writeFile(
|
||||
wrapper,
|
||||
`#!/bin/sh
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" = "$SCRIPTC_TEST_RUNTIME_RACE_SOURCE" ]; then
|
||||
: > "$SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"
|
||||
while [ ! -e "$SCRIPTC_TEST_RUNTIME_RACE_RELEASE" ]; do sleep 0.01; done
|
||||
break
|
||||
fi
|
||||
done
|
||||
exec "$SCRIPTC_TEST_REAL_CLANG" "$@"
|
||||
`,
|
||||
);
|
||||
await chmod(wrapper, 0o755);
|
||||
await writeFile(cPath, "int main(void) { return 0; }\n");
|
||||
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
|
||||
process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = fakeRuntime;
|
||||
process.env["SCRIPTC_TEST_REAL_CLANG"] = realClang!;
|
||||
process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = raceSource;
|
||||
process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = signal;
|
||||
process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = release;
|
||||
// Keep ccache out of this controlled race: its own object store is not
|
||||
// the cache under test and can obscure which compiler read happened.
|
||||
process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = "1";
|
||||
process.env["PATH"] = `${binDir}${delimiter}/usr/bin${delimiter}/bin`;
|
||||
delete process.env["SCRIPTC_NO_CACHE"];
|
||||
|
||||
const firstOut = join(dir, "first");
|
||||
const firstBuild = compileC({
|
||||
cPath,
|
||||
outPath: firstOut,
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (!existsSync(signal)) {
|
||||
if (Date.now() > deadline) throw new Error("timed out waiting for runtime compile");
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
await writeFile(
|
||||
raceSource,
|
||||
`${originalRuntimeSource}\nconst char scriptc_runtime_race_marker[] = "${marker}";\n`,
|
||||
);
|
||||
await writeFile(release, "go");
|
||||
await firstBuild;
|
||||
expect((await readFile(firstOut)).includes(Buffer.from(marker))).toBe(true);
|
||||
|
||||
// Restore the fingerprint, then force a program-cache miss. The old
|
||||
// fingerprint must not have retained the object built from edited bytes.
|
||||
await writeFile(raceSource, originalRuntimeSource);
|
||||
await writeFile(cPath, "int main(void) { return 0; } /* second */\n");
|
||||
const secondOut = join(dir, "second");
|
||||
await compileC({
|
||||
cPath,
|
||||
outPath: secondOut,
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
expect((await readFile(secondOut)).includes(Buffer.from(marker))).toBe(false);
|
||||
expect(await readdir(join(cacheRoot, "bin"))).toHaveLength(1);
|
||||
} finally {
|
||||
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
|
||||
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
|
||||
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
|
||||
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
|
||||
if (oldRuntimeDir === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
|
||||
else process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"] = oldRuntimeDir;
|
||||
if (oldDisableCcache === undefined) delete process.env["SCRIPTC_TEST_DISABLE_CCACHE"];
|
||||
else process.env["SCRIPTC_TEST_DISABLE_CCACHE"] = oldDisableCcache;
|
||||
if (oldPath === undefined) delete process.env["PATH"];
|
||||
else process.env["PATH"] = oldPath;
|
||||
if (oldRealClang === undefined) delete process.env["SCRIPTC_TEST_REAL_CLANG"];
|
||||
else process.env["SCRIPTC_TEST_REAL_CLANG"] = oldRealClang;
|
||||
if (oldRaceSource === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"];
|
||||
else process.env["SCRIPTC_TEST_RUNTIME_RACE_SOURCE"] = oldRaceSource;
|
||||
if (oldRaceSignal === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"];
|
||||
else process.env["SCRIPTC_TEST_RUNTIME_RACE_SIGNAL"] = oldRaceSignal;
|
||||
if (oldRaceRelease === undefined) delete process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"];
|
||||
else process.env["SCRIPTC_TEST_RUNTIME_RACE_RELEASE"] = oldRaceRelease;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test("explicit native link inputs relink while runtime objects remain cached", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-link-input-cache-"));
|
||||
scratch.push(dir);
|
||||
@@ -704,6 +941,77 @@ test("system libraries relink after an in-place rebuild while runtime objects re
|
||||
}
|
||||
});
|
||||
|
||||
test("damaged runtime objects are rebuilt before linking or archiving", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-object-integrity-"));
|
||||
scratch.push(dir);
|
||||
const cacheRoot = join(dir, "cache");
|
||||
const cPath = join(dir, "program.c");
|
||||
const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
|
||||
const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
|
||||
const corruption = Buffer.from("scriptc-corrupt-object-unique-marker");
|
||||
|
||||
const objectSets = async (): Promise<string[]> =>
|
||||
(await readdir(join(cacheRoot, "obj"), { withFileTypes: true }))
|
||||
.filter((entry) => entry.isDirectory() && !entry.name.startsWith("build-"))
|
||||
.map((entry) => entry.name);
|
||||
const corruptOneObject = async (set: string): Promise<string> => {
|
||||
const objectDir = join(cacheRoot, "obj", set);
|
||||
const object = (await readdir(objectDir)).find((name) => name.endsWith(".o"));
|
||||
expect(object).toBeDefined();
|
||||
await writeFile(join(objectDir, object!), corruption);
|
||||
expect((await stat(`${join(objectDir, object!)}.sha256`)).isFile()).toBe(true);
|
||||
return join(objectDir, object!);
|
||||
};
|
||||
|
||||
try {
|
||||
process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
|
||||
delete process.env["SCRIPTC_NO_CACHE"];
|
||||
await writeFile(cPath, '#include <stdio.h>\nint main(void) { puts("one"); return 0; }\n');
|
||||
await compileC({
|
||||
cPath,
|
||||
outPath: join(dir, "one"),
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
const [executableSet] = await objectSets();
|
||||
const damagedExecutableObject = await corruptOneObject(executableSet!);
|
||||
|
||||
await writeFile(cPath, '#include <stdio.h>\nint main(void) { puts("two"); return 0; }\n');
|
||||
const repairedOut = join(dir, "two");
|
||||
await compileC({
|
||||
cPath,
|
||||
outPath: repairedOut,
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
expect(execFileSync(repairedOut, { encoding: "utf8" }).trim()).toBe("two");
|
||||
expect(await readFile(damagedExecutableObject)).not.toEqual(corruption);
|
||||
|
||||
const beforeLibrary = new Set(await objectSets());
|
||||
await writeFile(cPath, "int scriptc_integrity_probe = 1;\n");
|
||||
await compileLibArchive({
|
||||
cPath,
|
||||
outPath: join(dir, "one.lib.a"),
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
const librarySet = (await objectSets()).find((set) => !beforeLibrary.has(set));
|
||||
expect(librarySet).toBeDefined();
|
||||
await corruptOneObject(librarySet!);
|
||||
|
||||
await writeFile(cPath, "int scriptc_integrity_probe = 2;\n");
|
||||
const repairedArchive = join(dir, "two.lib.a");
|
||||
await compileLibArchive({
|
||||
cPath,
|
||||
outPath: repairedArchive,
|
||||
cacheIdentity: TEST_CACHE_IDENTITY,
|
||||
});
|
||||
expect((await readFile(repairedArchive)).includes(corruption)).toBe(false);
|
||||
} finally {
|
||||
if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
|
||||
else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
|
||||
if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
|
||||
else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
|
||||
}
|
||||
});
|
||||
|
||||
test("library archives hit by content, invalidate on edits, and reuse runtime objects", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-cache-"));
|
||||
scratch.push(dir);
|
||||
@@ -731,7 +1039,7 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob
|
||||
const objectSet = objectSets.find((entry) => entry.isDirectory() && !entry.name.startsWith("build-"));
|
||||
expect(objectSet).toBeDefined();
|
||||
const objectDir = join(cacheRoot, "obj", objectSet!.name);
|
||||
const objectNames = await readdir(objectDir);
|
||||
const objectNames = (await readdir(objectDir)).filter((name) => name.endsWith(".o"));
|
||||
expect(objectNames.length).toBeGreaterThan(10);
|
||||
const pinnedTime = new Date("2000-01-01T00:00:00.000Z");
|
||||
await Promise.all(objectNames.map((name) => utimes(join(objectDir, name), pinnedTime, pinnedTime)));
|
||||
|
||||
@@ -59,6 +59,14 @@ const TOOLCHAIN_ENV_KEYS = [
|
||||
"CCC_ADD_ARGS",
|
||||
"CLANG_CONFIG_FILE_SYSTEM_DIR",
|
||||
"CLANG_CONFIG_FILE_USER_DIR",
|
||||
"CC",
|
||||
"CFLAGS",
|
||||
"CPPFLAGS",
|
||||
"LDFLAGS",
|
||||
"AR",
|
||||
"RANLIB",
|
||||
"CMAKE_GENERATOR",
|
||||
"CMAKE_TOOLCHAIN_FILE",
|
||||
"SOURCE_DATE_EPOCH",
|
||||
"ZERO_AR_DATE",
|
||||
"LANG",
|
||||
@@ -90,6 +98,17 @@ const MUTABLE_COMPILE_ENV_KEYS = [
|
||||
"CCC_ADD_ARGS",
|
||||
"CLANG_CONFIG_FILE_SYSTEM_DIR",
|
||||
"CLANG_CONFIG_FILE_USER_DIR",
|
||||
// The direct runtime build does not consume the conventional Make/CMake
|
||||
// variables, but the lazily-built QuickJS prerequisite does. Their values
|
||||
// can name mutable tools, headers, libraries, and toolchain files, so a
|
||||
// stable spelling is not a safe persistent-cache identity.
|
||||
"CC",
|
||||
"CFLAGS",
|
||||
"CPPFLAGS",
|
||||
"LDFLAGS",
|
||||
"AR",
|
||||
"RANLIB",
|
||||
"CMAKE_TOOLCHAIN_FILE",
|
||||
] as const;
|
||||
|
||||
/** These variables only redirect link-time inputs. Runtime objects remain
|
||||
@@ -316,6 +335,8 @@ export class CcCompileError extends Error {
|
||||
}
|
||||
|
||||
export function runtimeSrcDir(): string {
|
||||
const testRoot = process.env["SCRIPTC_TEST_RUNTIME_SRC_DIR"];
|
||||
if (testRoot !== undefined) return resolve(testRoot);
|
||||
const require = createRequire(import.meta.url);
|
||||
return join(dirname(require.resolve("@scriptc/runtime/package.json")), "src");
|
||||
}
|
||||
@@ -454,12 +475,60 @@ export function vendorCacheTargetFlavor(
|
||||
return driver.target === null ? `native-${hostPlatform}-${hostArch}` : driver.target;
|
||||
}
|
||||
|
||||
/** Vendor prerequisites live outside the content-addressed build root, so
|
||||
* their directory name must carry the same environment and resolved compiler
|
||||
* identity as the artifacts that consume them. The short digest keeps the
|
||||
* package-local path readable while preventing build-order contamination. */
|
||||
export function vendorCacheBuildIdentity(
|
||||
environmentFingerprint: string,
|
||||
compilerIdentity: string,
|
||||
): string {
|
||||
return createHash("sha256")
|
||||
.update("vendor-toolchain-v1\0")
|
||||
.update(environmentFingerprint)
|
||||
.update("\0")
|
||||
.update(compilerIdentity)
|
||||
.digest("hex")
|
||||
.slice(0, 20);
|
||||
}
|
||||
|
||||
const vendorCompilerIdentityFallbacks = new Map<string, string>();
|
||||
|
||||
async function currentVendorCacheBuildIdentity(
|
||||
driver: Pick<CcDriver, "argv" | "target">,
|
||||
environmentFingerprint: string,
|
||||
): Promise<string> {
|
||||
// Native vendor recipes additionally use bare clang/ar and CMake; cross
|
||||
// recipes use the zig driver for compilation and `zig ar`. Include every
|
||||
// executable that can affect the package-local prerequisite, not just the
|
||||
// final program's driver.
|
||||
const commands = [
|
||||
driver.argv[0] ?? "clang",
|
||||
...(driver.target === null ? ["clang", "cc", "ar", "ranlib", "cmake"] : []),
|
||||
].filter((command, index, all) => all.indexOf(command) === index);
|
||||
const identities = await Promise.all(
|
||||
commands.map(async (command) => {
|
||||
const spellingKey = `${environmentFingerprint}\0${command}`;
|
||||
const resolved = await resolvedToolIdentity(command);
|
||||
if (resolved !== null) vendorCompilerIdentityFallbacks.set(spellingKey, resolved);
|
||||
return [
|
||||
command,
|
||||
resolved ??
|
||||
vendorCompilerIdentityFallbacks.get(spellingKey) ??
|
||||
`<unresolved>\0${command}`,
|
||||
].join("\0");
|
||||
}),
|
||||
);
|
||||
return vendorCacheBuildIdentity(environmentFingerprint, identities.join("\x1f"));
|
||||
}
|
||||
|
||||
function engineArchivePath(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): string {
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
return join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-${flavor}`, "libqjs.a");
|
||||
}
|
||||
|
||||
@@ -493,7 +562,7 @@ async function installArtifact(
|
||||
}
|
||||
|
||||
/** The engine archive for one flavor, built lazily on the first --dynamic
|
||||
* compile (~10s) and cached under vendor/.cache/<commit>-<flavor>-<target>/ — unlike
|
||||
* compile (~10s) and cached under vendor/.cache/<commit>-<flavor>-<target>-<toolchain>/ — unlike
|
||||
* the runtime's own sources (recompiled every build, ~100ms), the engine is
|
||||
* far too big to rebuild per compile. The plain flavor is ALWAYS MinSizeRel
|
||||
* regardless of the program's optimization level: measured binary delta is
|
||||
@@ -513,11 +582,12 @@ async function installArtifact(
|
||||
async function ensureEngineArchive(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): Promise<string> {
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const vendor = vendorEngineDir();
|
||||
const archive = engineArchivePath(sanitize, driver, cacheRoot);
|
||||
const archive = engineArchivePath(sanitize, driver, buildIdentity, cacheRoot);
|
||||
const cacheDir = dirname(archive);
|
||||
if (await fileExists(archive)) return archive;
|
||||
if (driver.target !== null) return buildEngineArchiveCross(sanitize, driver, cacheRoot, cacheDir);
|
||||
@@ -638,19 +708,20 @@ const LRE_SOURCES = ["libregexp.c", "libunicode.c"];
|
||||
function lreObjectPaths(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): string[] {
|
||||
const flavor =
|
||||
(sanitize ? "asan" : "plain") +
|
||||
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
|
||||
`-${vendorCacheTargetFlavor(driver)}`;
|
||||
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const cacheDir = join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-lre-${flavor}`);
|
||||
return LRE_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o")));
|
||||
}
|
||||
|
||||
/** The libregexp objects for one flavor, compiled lazily on the first
|
||||
* regex-using static build (~1s) and cached like the engine archive —
|
||||
* vendor/.cache/<commit>-lre-<flavor>/*.o — with the same atomic-rename
|
||||
* vendor/.cache/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
|
||||
* publish (parallel first builds race safely; losers discard their work).
|
||||
* Plain is -Os (size class matters: the regex fence pins regex-using
|
||||
* binaries well under the engine class); asan matches the final link so
|
||||
@@ -662,6 +733,7 @@ function lreObjectPaths(
|
||||
async function ensureLreObjects(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): Promise<string[]> {
|
||||
// The flavor keys the DRIVER as well as the target: a zig-cc-built object
|
||||
@@ -671,9 +743,9 @@ async function ensureLreObjects(
|
||||
const flavor =
|
||||
(sanitize ? "asan" : "plain") +
|
||||
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
|
||||
`-${vendorCacheTargetFlavor(driver)}`;
|
||||
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const vendor = vendorEngineDir();
|
||||
const objects = lreObjectPaths(sanitize, driver, cacheRoot);
|
||||
const objects = lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
|
||||
const cacheDir = dirname(objects[0]!);
|
||||
if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects;
|
||||
|
||||
@@ -721,19 +793,20 @@ const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infbac
|
||||
function zlibObjectPaths(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): string[] {
|
||||
const flavor =
|
||||
(sanitize ? "asan" : "plain") +
|
||||
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
|
||||
`-${vendorCacheTargetFlavor(driver)}`;
|
||||
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const cacheDir = join(cacheRoot, `zlib-${ZLIB_VERSION}-${flavor}`);
|
||||
return ZLIB_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o")));
|
||||
}
|
||||
|
||||
/** The zlib objects for one flavor, compiled lazily on the first zlib-using
|
||||
* CROSS build (~1s) and cached like the lre objects —
|
||||
* vendor/.cache/zlib-<version>-<flavor>/*.o — with the same atomic-rename
|
||||
* vendor/.cache/zlib-<version>-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
|
||||
* publish (parallel first builds race safely; losers discard their work).
|
||||
* Plain is -Os, asan matches the final link so the sanitized lane
|
||||
* instruments the codec too. The flavor keys the driver and target exactly
|
||||
@@ -743,14 +816,15 @@ function zlibObjectPaths(
|
||||
async function ensureZlibObjects(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): Promise<string[]> {
|
||||
const flavor =
|
||||
(sanitize ? "asan" : "plain") +
|
||||
(driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
|
||||
`-${vendorCacheTargetFlavor(driver)}`;
|
||||
`-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const vendor = vendorZlibDir();
|
||||
const objects = zlibObjectPaths(sanitize, driver, cacheRoot);
|
||||
const objects = zlibObjectPaths(sanitize, driver, buildIdentity, cacheRoot);
|
||||
const cacheDir = dirname(objects[0]!);
|
||||
if ((await Promise.all(objects.map(fileExists))).every(Boolean)) return objects;
|
||||
|
||||
@@ -802,14 +876,15 @@ const CURL_STUB_SYMBOLS = [
|
||||
|
||||
function curlStubDirPath(
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): string {
|
||||
return join(cacheRoot, `curl-stub-${driver.target}`);
|
||||
return join(cacheRoot, `curl-stub-${driver.target}-${buildIdentity}`);
|
||||
}
|
||||
|
||||
/** The libcurl import stub for one CROSS target, generated lazily on the
|
||||
* first fetch-using cross build and cached like the zlib objects —
|
||||
* vendor/.cache/curl-stub-<target>/libcurl.so, atomic-rename publish. The
|
||||
* vendor/.cache/curl-stub-<target>-<toolchain>/libcurl.so, atomic-rename publish. The
|
||||
* stub is empty definitions of CURL_STUB_SYMBOLS compiled `-shared` with
|
||||
* `-soname libcurl.so.4`: linking against it satisfies scr_fetch_curl.c's
|
||||
* references and records a plain `DT_NEEDED libcurl.so.4`, which the
|
||||
@@ -820,9 +895,10 @@ function curlStubDirPath(
|
||||
* and asan links against plain shared libraries freely. */
|
||||
async function ensureCurlStub(
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): Promise<string> {
|
||||
const cacheDir = curlStubDirPath(driver, cacheRoot);
|
||||
const cacheDir = curlStubDirPath(driver, buildIdentity, cacheRoot);
|
||||
const lib = join(cacheDir, "libcurl.so");
|
||||
if (await fileExists(lib)) return cacheDir;
|
||||
|
||||
@@ -860,15 +936,16 @@ function vendorTlsDir(): string {
|
||||
function tlsArchivePath(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): string {
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
return join(cacheRoot, `mbedtls-${MBEDTLS_VERSION}-${flavor}`, "libmbedtls.a");
|
||||
}
|
||||
|
||||
/** The mbedTLS archive for one flavor, compiled lazily on the first
|
||||
* TLS-using build (~15s over ~110 TUs, parallelized) and cached like the
|
||||
* engine archive — vendor/.cache/mbedtls-<version>-<flavor>-<target>/libmbedtls.a —
|
||||
* engine archive — vendor/.cache/mbedtls-<version>-<flavor>-<target>-<toolchain>/libmbedtls.a —
|
||||
* with the same atomic-rename publish (parallel first builds race safely;
|
||||
* losers discard their work). Plain is -Os (the TLS stack is link-gated
|
||||
* but still wants its size class small); asan matches the final link so
|
||||
@@ -884,13 +961,14 @@ function tlsArchivePath(
|
||||
async function ensureTlsArchive(
|
||||
sanitize: boolean,
|
||||
driver: CcDriver,
|
||||
buildIdentity: string,
|
||||
cacheRoot: string = vendorBuildCacheRoot(),
|
||||
): Promise<string> {
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}`;
|
||||
const flavor = `${sanitize ? "asan" : "plain"}-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
|
||||
const compileArgv = driver.target !== null ? driver.argv : ["clang"];
|
||||
const arArgv = driver.target !== null ? [...driver.argv.slice(0, 1), "ar"] : ["ar"];
|
||||
const vendor = vendorTlsDir();
|
||||
const archive = tlsArchivePath(sanitize, driver, cacheRoot);
|
||||
const archive = tlsArchivePath(sanitize, driver, buildIdentity, cacheRoot);
|
||||
const cacheDir = dirname(archive);
|
||||
if (await fileExists(archive)) return archive;
|
||||
|
||||
@@ -1031,6 +1109,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
}
|
||||
}
|
||||
const toolchainEnv = toolchainEnvironmentFingerprint();
|
||||
const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv);
|
||||
let cachedArchive: string | null = null;
|
||||
let compilerVersion = "";
|
||||
let runtimeHash = "";
|
||||
@@ -1047,7 +1126,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
runtimeHash = fingerprint;
|
||||
cachedProgramBytes = programBytes;
|
||||
const key = createHash("sha256")
|
||||
.update("lib-v3\0")
|
||||
// v4 invalidates archives that may have been assembled from an
|
||||
// unverified or cross-environment vendor/runtime object cache.
|
||||
.update("lib-v4\0")
|
||||
.update(cacheTargetIdentity(driver)).update("\0")
|
||||
.update(toolchainEnv).update("\0")
|
||||
.update(opts.cacheIdentity!).update("\0")
|
||||
@@ -1097,10 +1178,10 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
const vendorCacheRoot = transientVendorRoot ?? vendorBuildCacheRoot();
|
||||
try {
|
||||
const lreObjects = regex
|
||||
? await ensureLreObjects(sanitize, driver, vendorCacheRoot)
|
||||
? await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: [];
|
||||
const zlibObjects = opts.zlib
|
||||
? await ensureZlibObjects(sanitize, driver, vendorCacheRoot)
|
||||
? await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: [];
|
||||
const buildDir = await mkdtemp(join(tmpdir(), "scriptc-lib-"));
|
||||
try {
|
||||
@@ -1147,6 +1228,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
cachedProgramBytes === null ? undefined : opts.cPath,
|
||||
);
|
||||
let runtimeObjects: string[] | null = null;
|
||||
let cacheInputsStable = true;
|
||||
if (root !== null && compilerVersion !== "" && runtimeHash !== "") {
|
||||
try {
|
||||
const sourcePaths = sources.map((f) => join(rtDir, f));
|
||||
@@ -1155,11 +1237,13 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
driver.argv,
|
||||
cflags,
|
||||
sourcePaths,
|
||||
`lib-obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`,
|
||||
`lib-obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${driver.argv.join(" ")}\0${compilerVersion}\0${runtimeHash}\0`,
|
||||
async () => (await runtimeFingerprint(rtDir)) === runtimeHash,
|
||||
);
|
||||
const staged = await stageRuntimeObjects(cached, join(buildDir, "cached-runtime"));
|
||||
runtimeObjects = sourcePaths.map((path) => staged.get(path)!);
|
||||
} catch {
|
||||
} catch (err) {
|
||||
if (err instanceof CacheInputsChangedError) cacheInputsStable = false;
|
||||
runtimeObjects = null;
|
||||
}
|
||||
}
|
||||
@@ -1191,7 +1275,11 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
|
||||
...objects,
|
||||
]);
|
||||
if (archiveOutput !== opts.outPath) await installArtifact(archiveOutput, opts.outPath);
|
||||
if (cachedArchive !== null && root !== null) {
|
||||
const runtimeStillMatchesKey =
|
||||
cacheInputsStable &&
|
||||
runtimeHash !== "" &&
|
||||
(await runtimeFingerprint(rtDir).catch(() => null)) === runtimeHash;
|
||||
if (cachedArchive !== null && root !== null && runtimeStillMatchesKey) {
|
||||
try {
|
||||
const cacheDir = dirname(cachedArchive);
|
||||
await mkdir(cacheDir, { recursive: true });
|
||||
@@ -1367,7 +1455,7 @@ async function resolvedToolIdentity(
|
||||
return null;
|
||||
}
|
||||
|
||||
async function ccVersionOnce(
|
||||
export async function ccVersionOnce(
|
||||
argv: string[],
|
||||
environmentFingerprint: string = toolchainEnvironmentFingerprint(),
|
||||
): Promise<string> {
|
||||
@@ -1384,11 +1472,22 @@ async function ccVersionOnce(
|
||||
const key = `${spellingKey}\0${executableIdentity ?? "<unresolved>"}`;
|
||||
let memo = ccVersionMemos.get(key);
|
||||
if (memo === undefined) {
|
||||
// cwd: the version probe must not touch the caller's directory —
|
||||
// `zig cc --version` (zig 0.16) drops an empty a.o wherever it runs.
|
||||
memo = execFileAsync(argv[0] ?? "clang", [...argv.slice(1), "--version"], { cwd: tmpdir() }).then(
|
||||
(r) => `${executableIdentity ?? "<unresolved>"}\0${`${r.stdout}\n${r.stderr}`.trim()}`,
|
||||
);
|
||||
// `zig cc --version` (zig 0.16) drops an empty a.o in its cwd. A private
|
||||
// probe directory avoids both caller pollution and the cross-process /
|
||||
// cross-user collision a fixed tmpdir()/a.o would create on Linux.
|
||||
memo = (async () => {
|
||||
const probeDir = await mkdtemp(join(tmpdir(), "scriptc-cc-version-"));
|
||||
try {
|
||||
const r = await execFileAsync(
|
||||
argv[0] ?? "clang",
|
||||
[...argv.slice(1), "--version"],
|
||||
{ cwd: probeDir },
|
||||
);
|
||||
return `${executableIdentity ?? "<unresolved>"}\0${`${r.stdout}\n${r.stderr}`.trim()}`;
|
||||
} finally {
|
||||
await rm(probeDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
}
|
||||
})();
|
||||
ccVersionMemos.set(key, memo);
|
||||
}
|
||||
if (executableIdentity !== null) ccVersionFallbacks.set(spellingKey, memo);
|
||||
@@ -1456,15 +1555,55 @@ export async function runtimeFingerprint(rtDir: string): Promise<string> {
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
class CacheInputsChangedError extends Error {
|
||||
constructor() {
|
||||
super("runtime inputs changed while populating the native object cache");
|
||||
this.name = "CacheInputsChangedError";
|
||||
}
|
||||
}
|
||||
|
||||
function cacheDigestPath(path: string): string {
|
||||
return `${path}.sha256`;
|
||||
}
|
||||
|
||||
async function fileDigest(path: string): Promise<string> {
|
||||
return createHash("sha256").update(await readFile(path)).digest("hex");
|
||||
}
|
||||
|
||||
/** Cache entries are disposable data, not trusted compiler output. Atomic
|
||||
* rename prevents ordinary partial writes, while the adjacent digest catches
|
||||
* disk damage, manual truncation, and entries left by an interrupted/older
|
||||
* publisher before they reach clang, the linker, or `ar`. */
|
||||
async function validCachedFile(object: string): Promise<boolean> {
|
||||
try {
|
||||
const digestPath = cacheDigestPath(object);
|
||||
const expected = (await readFile(digestPath, "utf8")).trim();
|
||||
const valid = /^[0-9a-f]{64}$/.test(expected) && (await fileDigest(object)) === expected;
|
||||
if (valid) {
|
||||
const now = new Date();
|
||||
await Promise.all([
|
||||
utimes(object, now, now).catch(() => undefined),
|
||||
utimes(digestPath, now, now).catch(() => undefined),
|
||||
]);
|
||||
}
|
||||
return valid;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The cached .o set for one flag flavor, compiled on first need. Concurrent
|
||||
* first builds (parallel test workers on a cold cache) may duplicate work;
|
||||
* per-file atomic renames make every winner equivalent. */
|
||||
* per-file atomic renames make every winner equivalent. Publication is held
|
||||
* until verifyInputs confirms that the source/header fingerprint used by the
|
||||
* key still describes the bytes clang just read. */
|
||||
async function ensureRuntimeObjects(
|
||||
root: string,
|
||||
ccArgv: string[],
|
||||
cflags: string[],
|
||||
sources: string[],
|
||||
keyPrefix: string,
|
||||
verifyInputs: () => Promise<boolean>,
|
||||
): Promise<Map<string, string>> {
|
||||
const setKey = createHash("sha256")
|
||||
.update(keyPrefix)
|
||||
@@ -1473,15 +1612,20 @@ async function ensureRuntimeObjects(
|
||||
.slice(0, 24);
|
||||
const objDir = join(root, "obj", setKey);
|
||||
const objOf = (src: string): string => join(objDir, `${basename(src, ".c")}.o`);
|
||||
const present = await Promise.all(sources.map((s) => fileExists(objOf(s))));
|
||||
const present = await Promise.all(sources.map((s) => validCachedFile(objOf(s))));
|
||||
const missing = sources.filter((_, i) => !present[i]);
|
||||
if (missing.length > 0) {
|
||||
// ccache wraps only the default clang driver — multi-word drivers
|
||||
// (`zig cc`) run bare; their object sets are keyed apart anyway.
|
||||
const useCcache = ccArgv.length === 1 && ccArgv[0] === "clang" && (await ccacheAvailable());
|
||||
const useCcache =
|
||||
process.env["SCRIPTC_TEST_DISABLE_CCACHE"] !== "1" &&
|
||||
ccArgv.length === 1 &&
|
||||
ccArgv[0] === "clang" &&
|
||||
(await ccacheAvailable());
|
||||
await mkdir(objDir, { recursive: true });
|
||||
const tmpDir = await mkdtemp(join(root, "obj", "build-"));
|
||||
try {
|
||||
const compiled = new Map<string, { object: string; digest: string }>();
|
||||
// Modest parallelism: a flavor's objects build once, but several cold
|
||||
// workers can race here — keep each build's CPU footprint small.
|
||||
const width = 4;
|
||||
@@ -1491,15 +1635,33 @@ async function ensureRuntimeObjects(
|
||||
const tmpObj = join(tmpDir, `${basename(src, ".c")}.o`);
|
||||
const argv = [...(useCcache ? ["ccache"] : []), ...ccArgv, ...cflags, "-c", src, "-o", tmpObj];
|
||||
await execFileAsync(argv[0] ?? "clang", argv.slice(1));
|
||||
await rename(tmpObj, objOf(src));
|
||||
compiled.set(src, { object: tmpObj, digest: await fileDigest(tmpObj) });
|
||||
}),
|
||||
);
|
||||
}
|
||||
// The fingerprint was computed before these subprocesses started. Do
|
||||
// not place their outputs under that key if a checkout/package update
|
||||
// changed any runtime source or included header while clang was reading.
|
||||
if (!(await verifyInputs())) throw new CacheInputsChangedError();
|
||||
for (const [src, built] of compiled) {
|
||||
const destination = objOf(src);
|
||||
const tmpDigest = `${built.object}.sha256`;
|
||||
await writeFile(tmpDigest, `${built.digest}\n`);
|
||||
// Publish data first and its verifier second. A concurrent reader in
|
||||
// the tiny gap sees a missing/mismatched digest and falls back safely.
|
||||
await rename(built.object, destination);
|
||||
await rename(tmpDigest, cacheDigestPath(destination));
|
||||
}
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
return new Map(sources.map((s) => [s, objOf(s)]));
|
||||
if (!(await verifyInputs())) throw new CacheInputsChangedError();
|
||||
const objects = new Map(sources.map((s) => [s, objOf(s)]));
|
||||
if (!(await Promise.all([...objects.values()].map(validCachedFile))).every(Boolean)) {
|
||||
throw new Error("native object cache integrity check failed");
|
||||
}
|
||||
return objects;
|
||||
}
|
||||
|
||||
/** Give one active link/archive operation private names for its cached
|
||||
@@ -1643,6 +1805,8 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
const cachePolicy = toolchainEnvironmentCachePolicy();
|
||||
const toolchainEnv = toolchainEnvironmentFingerprint();
|
||||
const vendorBuildIdentity = await currentVendorCacheBuildIdentity(driver, toolchainEnv);
|
||||
// Mutable include/SDK/config inputs can change behind a stable environment
|
||||
// spelling. The complete/runtime cache is disabled below; vendor objects
|
||||
// must follow the same rule instead of silently surviving in package-local
|
||||
@@ -1658,34 +1822,48 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
// Every vendor output path is deterministic from pins, flags, driver, and
|
||||
// target. Build the command/key from those paths now, but do not materialize
|
||||
// them until a complete-binary lookup has missed.
|
||||
const engineArchive = dynamic ? engineArchivePath(sanitize, driver, vendorCacheRoot) : null;
|
||||
const tlsArchive = tls ? tlsArchivePath(sanitize, driver, vendorCacheRoot) : null;
|
||||
const engineArchive = dynamic
|
||||
? engineArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: null;
|
||||
const tlsArchive = tls
|
||||
? tlsArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: null;
|
||||
// --dynamic + regex shares the archive's libregexp (its host hooks and
|
||||
// ours would collide; see scr_regex.c) — the standalone objects are for
|
||||
// static builds only.
|
||||
const lreObjects = regex && !dynamic ? lreObjectPaths(sanitize, driver, vendorCacheRoot) : [];
|
||||
const lreObjects = regex && !dynamic
|
||||
? lreObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: [];
|
||||
// Vendored zlib is the CROSS story only — host builds keep the exact
|
||||
// historical `-lz` system link (see CcOptions.zlib). The native fetch's
|
||||
// gzip decoder rides the same objects/link.
|
||||
const zlibObjects =
|
||||
((opts.zlib ?? false) || nativeFetch) && driver.target !== null
|
||||
? zlibObjectPaths(sanitize, driver, vendorCacheRoot)
|
||||
? zlibObjectPaths(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: [];
|
||||
// The libcurl import stub is likewise CROSS-only — host builds keep the
|
||||
// exact historical system `-lcurl` link (see CcOptions.fetch). Curl
|
||||
// reference builds only.
|
||||
const curlStubDir =
|
||||
curlFetch && driver.target !== null
|
||||
? curlStubDirPath(driver, vendorCacheRoot)
|
||||
? curlStubDirPath(driver, vendorBuildIdentity, vendorCacheRoot)
|
||||
: null;
|
||||
const materializeVendorPrerequisites = async (): Promise<void> => {
|
||||
// Preserve the historical order to avoid multiplying first-build resource
|
||||
// pressure when several large vendor sets are cold simultaneously.
|
||||
if (dynamic) await ensureEngineArchive(sanitize, driver, vendorCacheRoot);
|
||||
if (tls) await ensureTlsArchive(sanitize, driver, vendorCacheRoot);
|
||||
if (regex && !dynamic) await ensureLreObjects(sanitize, driver, vendorCacheRoot);
|
||||
if (zlibObjects.length > 0) await ensureZlibObjects(sanitize, driver, vendorCacheRoot);
|
||||
if (curlStubDir !== null) await ensureCurlStub(driver, vendorCacheRoot);
|
||||
if (dynamic) {
|
||||
await ensureEngineArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
|
||||
}
|
||||
if (tls) await ensureTlsArchive(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
|
||||
if (regex && !dynamic) {
|
||||
await ensureLreObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
|
||||
}
|
||||
if (zlibObjects.length > 0) {
|
||||
await ensureZlibObjects(sanitize, driver, vendorBuildIdentity, vendorCacheRoot);
|
||||
}
|
||||
if (curlStubDir !== null) {
|
||||
await ensureCurlStub(driver, vendorBuildIdentity, vendorCacheRoot);
|
||||
}
|
||||
};
|
||||
// rt() maps each runtime source's path on the command line: identity for
|
||||
// the historical single invocation, cached-.o substitution on cache misses.
|
||||
@@ -1925,7 +2103,6 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
const toolchainEnv = toolchainEnvironmentFingerprint();
|
||||
const [cv, fingerprint, cBytes] = await Promise.all([
|
||||
ccVersionOnce(driver.argv, toolchainEnv),
|
||||
runtimeFingerprint(rtDir),
|
||||
@@ -1952,7 +2129,9 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
a === opts.cPath ? "<program.c>" : a === opts.outPath ? "<out>" : a,
|
||||
);
|
||||
const key = createHash("sha256")
|
||||
.update("bin-v3\0")
|
||||
// v4 invalidates binaries that may have been published while runtime
|
||||
// inputs changed underneath the former per-file object publisher.
|
||||
.update("bin-v4\0")
|
||||
.update(cacheTargetIdentity(driver)).update("\0")
|
||||
.update(toolchainEnv).update("\0")
|
||||
.update(opts.cacheIdentity!).update("\0")
|
||||
@@ -2030,16 +2209,19 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
await writeFile(programPath, cBytes);
|
||||
|
||||
let objects: Map<string, string> | null = null;
|
||||
let cacheInputsStable = true;
|
||||
try {
|
||||
const cached = await ensureRuntimeObjects(
|
||||
root,
|
||||
driver.argv,
|
||||
cflags,
|
||||
rtInputs,
|
||||
`obj-v2\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`,
|
||||
`obj-v3\0${cacheTargetIdentity(driver)}\0${toolchainEnv}\0${ccName}\0${cv}\0${fingerprint}\0`,
|
||||
async () => (await runtimeFingerprint(rtDir)) === fingerprint,
|
||||
);
|
||||
objects = await stageRuntimeObjects(cached, join(buildDir, "runtime-objects"));
|
||||
} catch {
|
||||
} catch (err) {
|
||||
if (err instanceof CacheInputsChangedError) cacheInputsStable = false;
|
||||
objects = null; // cache trouble is never a build failure
|
||||
}
|
||||
|
||||
@@ -2051,7 +2233,10 @@ export async function compileC(opts: CcOptions): Promise<void> {
|
||||
);
|
||||
await installArtifact(privateOut, opts.outPath);
|
||||
|
||||
if (cachedBin !== null && keyHex !== null) {
|
||||
cacheInputsStable =
|
||||
cacheInputsStable &&
|
||||
(await runtimeFingerprint(rtDir).catch(() => null)) === fingerprint;
|
||||
if (cachedBin !== null && keyHex !== null && cacheInputsStable) {
|
||||
try {
|
||||
await mkdir(binDir, { recursive: true });
|
||||
const tmp = join(binDir, `.tmp-${keyHex.slice(0, 8)}-${process.pid}-${Math.random().toString(36).slice(2)}`);
|
||||
|
||||
Vendored
+4
-4
@@ -19,7 +19,7 @@ The QuickJS-ng JavaScript engine, embedded by the opt-in `--dynamic` build mode
|
||||
|
||||
The tree is a plain snapshot of the upstream commit with directories the library build does not need removed (tests/, docs/, examples/, test262 fixtures, CI config, generator scripts). No vendored file is modified; to update, re-clone upstream at the new commit, delete its .git directory, apply the same trim, and update this file.
|
||||
|
||||
The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/<commit>-<flavor>/` next to this file — one flavor per lane (plain, asan). The cache directory is gitignored and safe to delete.
|
||||
The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile into `.cache/<commit>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity. The cache directory is gitignored and safe to delete.
|
||||
|
||||
## zlib/
|
||||
|
||||
@@ -29,7 +29,7 @@ The engine archive (libqjs.a) is built lazily on the first `--dynamic` compile i
|
||||
|
||||
The CROSS-target arm of node:zlib (see packages/runtime/src/scr_zlib.c): host builds keep the historical system `-lz` link (macOS ships libz), but zig's cross sysroots have no libz, so SCRIPTC_TARGET builds compile this vendored copy per target instead. Only the flat `*.c`/`*.h` at the distribution root are vendored (LICENSE beside them) — contrib, tests, build machinery, and docs are not. No vendored file is modified; the gz* file-I/O TUs are vendored for faithfulness but never compiled (nothing references the gzFile API — see ZLIB_SOURCES in packages/compiler/src/backend/cc.ts). To update, re-fetch the release tarball, copy the same flat set, and bump `ZLIB_VERSION` in cc.ts.
|
||||
|
||||
The objects build lazily on the first zlib-using cross compile into `.cache/zlib-<version>-<flavor>/` next to this file — one flavor per driver/target (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation.
|
||||
The objects build lazily on the first zlib-using cross compile into `.cache/zlib-<version>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per driver/target and compiler environment/identity (the lre-objects pattern). Zlib-free binaries never compile any of this; compressed OUTPUT bytes are zlib-version-dependent, which is why the corpus only compares round-trips and fixed-blob inflation.
|
||||
|
||||
## curl/
|
||||
|
||||
@@ -39,7 +39,7 @@ The objects build lazily on the first zlib-using cross compile into `.cache/zlib
|
||||
|
||||
HEADERS ONLY — no curl C source is vendored and none is ever compiled. These headers now serve ONLY the RETIRED curl reference implementation of fetch (packages/runtime/src/scr_fetch_curl.c, selected by `SCRIPTC_FETCH_CURL=1`; kept compilable for one release as the native flip's reference — see scr_fetch.c, the default, which rides scr_net/scr_tls/scr_http/zlib and touches nothing here). Under the flag: host builds link the system `-lcurl` (macOS ships libcurl), and linux-gnu CROSS builds compile scr_fetch_curl.c against these headers, then link against a generated STUB `libcurl.so` (soname `libcurl.so.4`, empty definitions of exactly the symbols scr_fetch_curl.c calls — see CURL_STUB_SYMBOLS in packages/compiler/src/backend/cc.ts) so the produced binary records a plain `DT_NEEDED libcurl.so.4` that the TARGET system's real libcurl satisfies at load time, the standard cross-link import-stub technique. The 7.88.1 pin is a floor, not a lock: scr_fetch_curl.c's newest requirement is CURLOPT_PROTOCOLS_STR (7.85.0) and the unversioned symbol references bind to any libcurl.so.4. This whole directory leaves with scr_fetch_curl.c when the reference retires for good.
|
||||
|
||||
The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub-<flavor>/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself.
|
||||
The stub builds lazily on the first flag-selected fetch cross compile into `.cache/curl-stub-<target>-<toolchain>/` next to this file (the zlib-objects pattern). Default builds never see any of this, and no build ever compiles curl itself.
|
||||
|
||||
## mbedtls/
|
||||
|
||||
@@ -49,4 +49,4 @@ The stub builds lazily on the first flag-selected fetch cross compile into `.cac
|
||||
|
||||
The TLS provider behind node:tls/node:https (see the design note atop packages/runtime/src/scr_tls.c for why mbedTLS over SecureTransport/BoringSSL/libtls). Only `include/` and `library/` are vendored (LICENSE beside them) — tests, docs, programs, scripts, and CMake machinery are not. No vendored file is modified and no custom config is applied (the stock `mbedtls_config.h` builds every `library/*.c` standalone with clang); to update, re-fetch the release tarball, copy the same two directories, and bump `MBEDTLS_VERSION` in packages/compiler/src/backend/cc.ts.
|
||||
|
||||
The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls-<version>-<flavor>-<target>/` next to this file — one flavor per lane (plain, asan) and native platform/architecture or explicit cross target, the libqjs.a pattern. TLS-free binaries never compile or link any of this.
|
||||
The archive (libmbedtls.a) is built lazily on the first TLS-using compile into `.cache/mbedtls-<version>-<flavor>-<target>-<toolchain>/` next to this file — one flavor per lane (plain, asan), native platform/architecture or explicit cross target, and compiler environment/identity, the libqjs.a pattern. TLS-free binaries never compile or link any of this.
|
||||
|
||||
@@ -24,7 +24,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A
|
||||
|
||||
- **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips clang entirely; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies.
|
||||
- **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A hit skips clang and `ar`.
|
||||
- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Compiles route through ccache when installed, silently falling back when not.
|
||||
- **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime fingerprint after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not.
|
||||
- **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another.
|
||||
|
||||
Escape hatches: `SCRIPTC_NO_CACHE=1` bypasses every cache in both directions (no reads, no writes — the run behaves exactly like the uncached path). An explicitly empty `SCRIPTC_CACHE_DIR` does the same; a non-empty value overrides the production default. Eviction is a size-capped LRU sweep over each cache root (`SCRIPTC_CACHE_MAX_MB`, default 4096), run after the first write and periodically in long-lived processes; reads bump mtimes.
|
||||
|
||||
Reference in New Issue
Block a user