mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-14 04:15:51 +08:00
Malte UblandClaude Opus 5.5
678ddf8a4e
A file the island decision put in the set is no longer compiled natively: it is keyed by its absolute path, its declarations type the program as foreign, its diagnostics do not fail the program the build does not compile, and every import or require of it lowers to an island load. `--node-island=<path>` therefore takes effect for a user module, not only for a package. The recovery stays one round: when the lowering refuses a user JavaScript module, the reload adds every such module at once and each one is reported once as an SC8001 note naming the module and its reason. The build reports the islands it ran (packages and modules) with its result, and an executable cache hit reports them again, so a runner can tell an island result from a native one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
979 lines
36 KiB
TypeScript
979 lines
36 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import {
|
|
chmod,
|
|
copyFile,
|
|
mkdir,
|
|
readFile,
|
|
rename,
|
|
rm,
|
|
stat,
|
|
utimes,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { basename, dirname, join, resolve } from "node:path";
|
|
import type { ScrDiagnostic } from "../diagnostics/diagnostic.js";
|
|
import type { NativeProfileMatch } from "../backend/native-profile.js";
|
|
import type { FrontendInputSnapshot } from "../frontend/input-tracker.js";
|
|
import { frontendInputsStillMatch, validFrontendInputSnapshot } from "../frontend/input-tracker.js";
|
|
import { compilerReleaseVersion } from "../library/sidecar.js";
|
|
import { nativeArtifactDependenciesStillMatch } from "../backend/native/artifact-stamps.js";
|
|
import { type NativeArtifactDependency } from "../backend/native/contracts.js";
|
|
import { optimizationKeyParts } from "../backend/optimization.js";
|
|
import { type TargetCpu, cpuDispatchKeyParts, targetCpuKeyParts } from "../backend/target-cpu.js";
|
|
import { privateSiblingPath } from "../backend/build-cache.js";
|
|
import type { CompilerImplementationDependency } from "../library/compiler-self-identity.js";
|
|
import {
|
|
installDarwinDebugSymbols,
|
|
needsDarwinDebugSymbols,
|
|
readDarwinDebugSymbols,
|
|
} from "../backend/debug-symbols.js";
|
|
import {
|
|
compilerImplementationDependenciesStillMatch,
|
|
compilerImplementationRoot,
|
|
} from "../library/compiler-self-identity.js";
|
|
import {
|
|
cacheKey as sharedCacheKey,
|
|
digest,
|
|
frontendOutputExclusions,
|
|
installBytes,
|
|
outputPaths,
|
|
readCachedFile,
|
|
stampIntegrity as sharedStampIntegrity,
|
|
stampPath as sharedStampPath,
|
|
validNativeFeatures as validSharedNativeFeatures,
|
|
} from "../library/cache-primitives.js";
|
|
|
|
interface CachedExecutableFile {
|
|
name: string;
|
|
digest: string;
|
|
}
|
|
|
|
export interface EarlyExecutableNativeFeatures {
|
|
backend: "llvm";
|
|
/** Omitted is the historical release posture. */
|
|
optimization?: "dev" | "speed";
|
|
workers?: boolean;
|
|
youngNursery?: boolean;
|
|
/** `--large-pages` links scr_text_pages.c. */
|
|
largePages?: boolean;
|
|
regex: boolean;
|
|
copying: boolean;
|
|
textDecoderLegacy: boolean;
|
|
fileHandle: boolean;
|
|
fetch: boolean;
|
|
eventTarget?: boolean;
|
|
wasm?: boolean;
|
|
warningReplace?: boolean;
|
|
stdioExtended?: boolean;
|
|
/** Error-taking write completion callbacks: SCR_STREAM_WRITE_ERRORS. */
|
|
streamWriteErrors?: boolean;
|
|
/** The process output write slot: SCR_STDIO_WRITE_SLOT. */
|
|
stdioWriteSlot?: boolean;
|
|
nodeIsland?: boolean;
|
|
moduleNamespace?: boolean;
|
|
moduleNamespaceDyn?: boolean;
|
|
primitiveWrapper?: boolean;
|
|
zlib: boolean;
|
|
brotli?: boolean;
|
|
assert: boolean;
|
|
inspect: boolean;
|
|
dynInvoke: boolean;
|
|
dc: boolean;
|
|
dynAsync: boolean;
|
|
dynTier?: boolean;
|
|
streamDyn?: boolean;
|
|
handleProps?: boolean;
|
|
childLifecycle?: boolean;
|
|
declAdopt?: boolean;
|
|
typeOrigins?: boolean;
|
|
dynForAwait?: boolean;
|
|
eventsOnceSignal?: boolean;
|
|
mime?: boolean;
|
|
webStreams?: boolean;
|
|
abortSignal?: boolean;
|
|
utilTiming?: boolean;
|
|
dynBuffer?: boolean;
|
|
builtinValues?: boolean;
|
|
computedRequire?: boolean;
|
|
events: boolean;
|
|
emitter: boolean;
|
|
symbol: boolean;
|
|
bigint: boolean;
|
|
/** BigIntStats member reads: scr_stats_bigint.c. Absent (false) otherwise. */
|
|
bigstats?: boolean;
|
|
searchParams: boolean;
|
|
qs: boolean;
|
|
utilPromisify: boolean;
|
|
dynBuiltinClass: boolean;
|
|
dynBuiltinStream: boolean;
|
|
parseArgs: boolean;
|
|
stream: boolean;
|
|
net: boolean;
|
|
http: boolean;
|
|
http2: boolean;
|
|
dgram: boolean;
|
|
dnsPromises: boolean;
|
|
dgramDyn: boolean;
|
|
textCodecDyn: boolean;
|
|
watch: boolean;
|
|
watchFile: boolean;
|
|
foreignFfi: boolean;
|
|
nodeTest: boolean;
|
|
tls: boolean;
|
|
cipher: boolean;
|
|
ecdh: boolean;
|
|
dh: boolean;
|
|
pki: boolean;
|
|
ed25519: boolean;
|
|
x509: boolean;
|
|
perfLoop: boolean;
|
|
tlsCa: boolean;
|
|
webcrypto: boolean;
|
|
/** Intl over ICU (scr_intl.c, the ICU archive and its data file). */
|
|
intl?: boolean;
|
|
}
|
|
|
|
interface EarlyExecutableCacheStamp {
|
|
version: 1;
|
|
key: string;
|
|
frontend: FrontendInputSnapshot;
|
|
files: {
|
|
translationUnit: CachedExecutableFile;
|
|
ir: CachedExecutableFile | null;
|
|
executable: CachedExecutableFile | null;
|
|
debugSymbols?: CachedExecutableFile;
|
|
};
|
|
nativeDependencies: NativeArtifactDependency[] | null;
|
|
native: EarlyExecutableNativeFeatures;
|
|
/** The build's performance warnings (SC61xx), which a cache hit repeats. */
|
|
performance?: ScrDiagnostic[];
|
|
/** The Node islands the build ran, which a cache hit reports again. */
|
|
islands?: { packages: string[]; modules: string[] };
|
|
/** Programs that call into Node.js: the libnode pack identity the build
|
|
* recorded (EarlyExecutableCacheOptions.libnodeIdentity). */
|
|
libnode?: string;
|
|
/** How a --profile-use build's profile matched, which a cache hit reports. */
|
|
profileMatch?: NativeProfileMatch;
|
|
integrity: string;
|
|
}
|
|
|
|
export interface EarlyExecutableCacheOptions {
|
|
entryPath: string;
|
|
outDir: string;
|
|
outPath: string;
|
|
emitIr: boolean;
|
|
sanitize: boolean;
|
|
backend: "llvm";
|
|
/** Omitted is the historical release posture and preserves v1 keys. */
|
|
optimization?: "dev" | "speed";
|
|
/** Omitted retains the unstripped executable's historical cache key. */
|
|
strip?: true;
|
|
/** Omitted unless --static is set. */
|
|
staticLink?: true;
|
|
/** Omitted for the default Windows console subsystem. */
|
|
windowsSubsystem?: "gui";
|
|
/** Omitted unless --detached-exit is set. */
|
|
detachedExit?: true;
|
|
/** The identity of the libnode pack a program that calls into Node.js
|
|
* records (node-islands/libnode.ts libnodeCacheKey), or undefined when the
|
|
* build names none. Not part of the key: an entry of a program that calls
|
|
* into Node records it, and a hit requires the same identity, so the pack
|
|
* settings never change the key of a program that does not. */
|
|
libnodeIdentity?: () => string | undefined;
|
|
/** Omitted unless --large-pages is set. */
|
|
largePages?: true;
|
|
/** Omitted unless the build links without huge text (CompileOptions.hugeText). */
|
|
noHugeText?: true;
|
|
/** Omitted unless --exceptions=rare is set. */
|
|
exceptions?: "rare";
|
|
/** Omitted for the x86-64 baseline (`--target-cpu`). */
|
|
targetCpu?: TargetCpu;
|
|
/** Omitted for builds without runtime CPU dispatch. */
|
|
cpuDispatch?: true;
|
|
/** Omitted without --profile-generate/--profile-use; otherwise the
|
|
* pattern, or the digest of the profile's bytes. */
|
|
profile?: string;
|
|
/** Omitted without a function layout; otherwise the digest of its bytes. */
|
|
functionLayout?: string;
|
|
/** Omitted for the default tsgo checker; with SCRIPTC_CHECKER=tsc-ts, the
|
|
* tsc-ts executable's file identity. */
|
|
checker?: string;
|
|
/** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */
|
|
youngNursery?: true;
|
|
/** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */
|
|
heapCensus?: true;
|
|
npmStatic: readonly string[] | "auto" | null;
|
|
/** Omitted for the default Node island policy; else nodeIslandOptionsKey. */
|
|
nodeIslands?: string;
|
|
/** Omitted unless shape inference's overlay compiles the packages
|
|
* (frontend/shape-inference). */
|
|
shapeOverlay?: true;
|
|
/** Raw manifest identity: path and bytes. Native archives remain under the
|
|
* stricter native cache's independent dependency validation. */
|
|
ffiProfile: { path: string; bytes: Uint8Array } | null;
|
|
target: string;
|
|
compiler: string[];
|
|
nativeEnvironment: string;
|
|
nodeVersion: string;
|
|
implementation: string;
|
|
implementationDependencies: CompilerImplementationDependency[];
|
|
}
|
|
|
|
export type EarlyExecutableRouteOptions = Omit<
|
|
EarlyExecutableCacheOptions,
|
|
"implementation" | "implementationDependencies" | "nativeEnvironment"
|
|
> & {
|
|
/** Discovery waits until a complete payload and its frontend proof match. */
|
|
nativeEnvironment: string | (() => Promise<string>);
|
|
};
|
|
|
|
interface EarlyExecutableRouteStamp {
|
|
version: 2;
|
|
key: string;
|
|
implementation: string;
|
|
nativeEnvironment: string;
|
|
integrity: string;
|
|
}
|
|
|
|
interface EarlyExecutableImplementationProof {
|
|
version: 1;
|
|
implementation: string;
|
|
dependencies: CompilerImplementationDependency[];
|
|
integrity: string;
|
|
}
|
|
|
|
export interface EarlyExecutableCacheHit {
|
|
llvmPath: string;
|
|
irPath?: string;
|
|
/** When the executable was not restored: private copies of the cached LLVM text and IR beside
|
|
* llvmPath and irPath, which the build compiles from and installs at those names when it is done
|
|
* (index.ts BuildSources). A restored executable needs no compile, and its files are installed at
|
|
* llvmPath and irPath. */
|
|
llvmSourcePath?: string;
|
|
irSourcePath?: string;
|
|
/** llvmPath (irPath) already holds the cached bytes: the build leaves it as it is. */
|
|
llvmPathCurrent?: true;
|
|
irPathCurrent?: true;
|
|
/** True when the final executable was restored after native dependencies
|
|
* validated. False restores only frontend artifacts and must call compileC. */
|
|
executableRestored: boolean;
|
|
native: EarlyExecutableNativeFeatures;
|
|
frontend: FrontendInputSnapshot;
|
|
/** The performance warnings the build reported (SC61xx), repeated on a hit. */
|
|
performance?: ScrDiagnostic[];
|
|
/** The Node islands the build ran, repeated on a hit. */
|
|
islands?: { packages: string[]; modules: string[] };
|
|
/** How a --profile-use build's profile matched, reported again on a hit. */
|
|
profileMatch?: NativeProfileMatch;
|
|
}
|
|
|
|
export interface EarlyExecutableCachePublish extends EarlyExecutableCacheHit {
|
|
nativeDependencies?: NativeArtifactDependency[];
|
|
}
|
|
|
|
function isIslands(value: unknown): value is { packages: string[]; modules: string[] } {
|
|
if (typeof value !== "object" || value === null) return false;
|
|
const islands = value as Record<string, unknown>;
|
|
return (
|
|
Array.isArray(islands.packages) &&
|
|
Array.isArray(islands.modules) &&
|
|
[...islands.packages, ...islands.modules].every((item) => typeof item === "string")
|
|
);
|
|
}
|
|
|
|
function isProfileMatch(value: unknown): value is NativeProfileMatch {
|
|
if (typeof value !== "object" || value === null) return false;
|
|
const match = value as Record<string, unknown>;
|
|
return ["annotated", "mismatched", "missing", "recovered", "unknownPartitions"].every((key) => {
|
|
const count = match[key];
|
|
return typeof count === "number" && Number.isInteger(count) && count >= 0;
|
|
});
|
|
}
|
|
|
|
const BOOLEAN_NATIVE_KEYS = [
|
|
"regex",
|
|
"copying",
|
|
"textDecoderLegacy",
|
|
"fileHandle",
|
|
"fetch",
|
|
"zlib",
|
|
"assert",
|
|
"inspect",
|
|
"dynInvoke",
|
|
"dc",
|
|
"dynAsync",
|
|
"events",
|
|
"emitter",
|
|
"symbol",
|
|
"bigint",
|
|
"searchParams",
|
|
"qs",
|
|
"utilPromisify",
|
|
"dynBuiltinClass",
|
|
"dynBuiltinStream",
|
|
"parseArgs",
|
|
"stream",
|
|
"net",
|
|
"http",
|
|
"http2",
|
|
"dgram",
|
|
"dnsPromises",
|
|
"dgramDyn",
|
|
"textCodecDyn",
|
|
"watch",
|
|
"watchFile",
|
|
"foreignFfi",
|
|
"nodeTest",
|
|
"tls",
|
|
"tlsCa",
|
|
"cipher",
|
|
"ecdh",
|
|
"dh",
|
|
"pki",
|
|
"ed25519",
|
|
"x509",
|
|
"webcrypto",
|
|
"perfLoop",
|
|
] as const satisfies readonly (keyof EarlyExecutableNativeFeatures)[];
|
|
|
|
function validNativeFeatures(value: unknown): value is EarlyExecutableNativeFeatures {
|
|
return validSharedNativeFeatures<EarlyExecutableNativeFeatures>(
|
|
value,
|
|
BOOLEAN_NATIVE_KEYS,
|
|
(native) =>
|
|
(native.optimization === undefined ||
|
|
native.optimization === "dev" ||
|
|
native.optimization === "speed") &&
|
|
(native.workers === undefined || typeof native.workers === "boolean") &&
|
|
(native.youngNursery === undefined || typeof native.youngNursery === "boolean"),
|
|
);
|
|
}
|
|
|
|
function cacheKey(options: EarlyExecutableCacheOptions): string {
|
|
const ffiParts: (string | Uint8Array)[] =
|
|
options.ffiProfile === null
|
|
? ["<ffi-off>"]
|
|
: [resolve(options.ffiProfile.path), options.ffiProfile.bytes];
|
|
return sharedCacheKey("early-executable-v1", [
|
|
resolve(options.entryPath),
|
|
resolve(options.outDir),
|
|
resolve(options.outPath),
|
|
options.emitIr ? "emit-ir" : "no-ir",
|
|
options.sanitize ? "sanitize" : "plain",
|
|
"static",
|
|
options.backend,
|
|
...optimizationKeyParts(options.optimization),
|
|
...(options.strip ? ["strip"] : []),
|
|
...(options.staticLink ? ["static-link"] : []),
|
|
...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []),
|
|
...(options.detachedExit ? ["detached-exit"] : []),
|
|
...(options.largePages ? ["large-pages"] : []),
|
|
...(options.noHugeText ? ["no-huge-text"] : []),
|
|
...(options.exceptions === "rare" ? ["exceptions-rare"] : []),
|
|
...targetCpuKeyParts(options.targetCpu),
|
|
...cpuDispatchKeyParts(options.cpuDispatch),
|
|
...(options.profile !== undefined ? [options.profile] : []),
|
|
...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []),
|
|
...(options.checker !== undefined ? [`checker:${options.checker}`] : []),
|
|
...(options.youngNursery ? ["young-nursery"] : []),
|
|
...(options.nodeIslands !== undefined ? [`node-islands:${options.nodeIslands}`] : []),
|
|
...(options.heapCensus ? ["heap-census"] : []),
|
|
options.npmStatic === null
|
|
? "<npm-static-off>"
|
|
: options.npmStatic === "auto"
|
|
? "<npm-static-auto>"
|
|
: JSON.stringify(options.npmStatic),
|
|
options.target,
|
|
options.compiler.join("\x1f"),
|
|
options.nativeEnvironment,
|
|
options.nodeVersion,
|
|
options.implementation,
|
|
...ffiParts,
|
|
]);
|
|
}
|
|
|
|
function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment">): string {
|
|
const hash = createHash("sha256")
|
|
.update("early-executable-route-v2\0")
|
|
.update(compilerReleaseVersion())
|
|
.update("\0")
|
|
.update(compilerImplementationRoot())
|
|
.update("\0")
|
|
.update(resolve(options.entryPath))
|
|
.update("\0")
|
|
.update(resolve(options.outDir))
|
|
.update("\0")
|
|
.update(resolve(options.outPath))
|
|
.update("\0")
|
|
.update(options.emitIr ? "emit-ir" : "no-ir")
|
|
.update("\0")
|
|
.update(options.sanitize ? "sanitize" : "plain")
|
|
.update("\0")
|
|
.update("static")
|
|
.update("\0")
|
|
.update(options.backend)
|
|
.update("\0");
|
|
for (const part of optimizationKeyParts(options.optimization)) hash.update(`${part}\0`);
|
|
if (options.strip) hash.update("strip\0");
|
|
if (options.staticLink) hash.update("static-link\0");
|
|
if (options.windowsSubsystem === "gui") hash.update("windows-subsystem-gui\0");
|
|
if (options.detachedExit) hash.update("detached-exit\0");
|
|
if (options.largePages) hash.update("large-pages\0");
|
|
if (options.noHugeText) hash.update("no-huge-text\0");
|
|
if (options.exceptions === "rare") hash.update("exceptions-rare\0");
|
|
for (const part of targetCpuKeyParts(options.targetCpu)) hash.update(`${part}\0`);
|
|
for (const part of cpuDispatchKeyParts(options.cpuDispatch)) hash.update(`${part}\0`);
|
|
if (options.profile !== undefined) hash.update(`${options.profile}\0`);
|
|
if (options.functionLayout !== undefined) hash.update(`layout:${options.functionLayout}\0`);
|
|
if (options.checker !== undefined) hash.update(`checker:${options.checker}\0`);
|
|
if (options.youngNursery) hash.update("young-nursery\0");
|
|
if (options.nodeIslands !== undefined) hash.update(`node-islands:${options.nodeIslands}\0`);
|
|
if (options.heapCensus) hash.update("heap-census\0");
|
|
hash
|
|
.update(
|
|
options.npmStatic === null
|
|
? "<npm-static-off>"
|
|
: options.npmStatic === "auto"
|
|
? "<npm-static-auto>"
|
|
: JSON.stringify(options.npmStatic),
|
|
)
|
|
.update("\0")
|
|
.update(options.target)
|
|
.update("\0")
|
|
.update(options.compiler.join("\x1f"))
|
|
.update("\0")
|
|
.update(options.nodeVersion)
|
|
.update("\0");
|
|
if (options.ffiProfile === null) {
|
|
hash.update("<ffi-off>");
|
|
} else {
|
|
hash.update(resolve(options.ffiProfile.path)).update("\0").update(options.ffiProfile.bytes);
|
|
}
|
|
return hash.digest("hex");
|
|
}
|
|
|
|
function routePath(
|
|
root: string,
|
|
options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment">,
|
|
): string {
|
|
return join(root, "early-exe-route", routeKey(options));
|
|
}
|
|
|
|
function nativeBuildMarkerPath(
|
|
root: string,
|
|
options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment">,
|
|
): string {
|
|
return join(root, "early-exe-native-build", routeKey(options));
|
|
}
|
|
|
|
/** Records that a build of the invocation `options` emitted its native program object, so the
|
|
* native helper's partition cache holds its partitions. Unlike the route, the record does not
|
|
* depend on the build publishing its executable. */
|
|
export async function recordEarlyExecutableNativeBuild(
|
|
root: string | null,
|
|
options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment">,
|
|
): Promise<void> {
|
|
if (root === null) return;
|
|
const marker = nativeBuildMarkerPath(root, options);
|
|
await mkdir(dirname(marker), { recursive: true, mode: 0o700 });
|
|
await writeFile(marker, "", { mode: 0o600 });
|
|
}
|
|
|
|
/** Whether an earlier build of the invocation `options` recorded its native program object
|
|
* (`recordEarlyExecutableNativeBuild`). False when there is no cache. */
|
|
export async function hasEarlyExecutableNativeBuild(
|
|
root: string | null,
|
|
options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment">,
|
|
): Promise<boolean> {
|
|
if (root === null) return false;
|
|
return stat(nativeBuildMarkerPath(root, options)).then(
|
|
(info) => info.isFile(),
|
|
() => false,
|
|
);
|
|
}
|
|
|
|
function routeIntegrity(stamp: Omit<EarlyExecutableRouteStamp, "integrity">): string {
|
|
return createHash("sha256")
|
|
.update("early-executable-route-stamp-v2\0")
|
|
.update(JSON.stringify(stamp))
|
|
.digest("hex");
|
|
}
|
|
|
|
function implementationProofPath(root: string, implementation: string): string {
|
|
const install = createHash("sha256")
|
|
.update("early-executable-install-v1\0")
|
|
.update(compilerImplementationRoot())
|
|
.digest("hex");
|
|
return join(root, "early-exe-implementation", install, implementation);
|
|
}
|
|
|
|
function implementationProofIntegrity(
|
|
proof: Omit<EarlyExecutableImplementationProof, "integrity">,
|
|
): string {
|
|
return createHash("sha256")
|
|
.update("early-executable-implementation-proof-v1\0")
|
|
.update(JSON.stringify(proof))
|
|
.digest("hex");
|
|
}
|
|
|
|
async function installCacheMetadata(source: string, destination: string): Promise<void> {
|
|
await rename(source, destination).catch(async () => {
|
|
// Windows does not replace an existing destination with rename(). Route
|
|
// and implementation-proof files are republished on every full compile,
|
|
// so use the same replacement fallback as the payload cache below.
|
|
await rm(destination, { force: true });
|
|
await rename(source, destination);
|
|
});
|
|
}
|
|
|
|
function stampPath(root: string, options: EarlyExecutableCacheOptions): string {
|
|
return sharedStampPath(root, "early-exe", cacheKey(options));
|
|
}
|
|
|
|
function stampIntegrity(stamp: Omit<EarlyExecutableCacheStamp, "integrity">): string {
|
|
return sharedStampIntegrity("early-executable-stamp-v1", stamp);
|
|
}
|
|
|
|
function executableFrontendOutputExclusions(
|
|
options: EarlyExecutableCacheOptions,
|
|
backend: "llvm",
|
|
): ReturnType<typeof frontendOutputExclusions> {
|
|
return frontendOutputExclusions(options, backend, "", [
|
|
options.outPath,
|
|
`${options.outPath}.dSYM`,
|
|
]);
|
|
}
|
|
|
|
/** Ordinary source/configuration edits are cheap misses before driver
|
|
* discovery. Replay declaration-file bytes and resolution metadata in the
|
|
* final complete proof instead of twice. */
|
|
function routedFrontendCandidate(frontend: FrontendInputSnapshot): FrontendInputSnapshot {
|
|
return {
|
|
...frontend,
|
|
probes: frontend.probes.filter(
|
|
(probe) => probe.op === "file" && !/\.d\.(?:ts|mts|cts)$/.test(probe.path),
|
|
),
|
|
};
|
|
}
|
|
|
|
async function fileMatches(
|
|
destination: string,
|
|
expectedDigest: string,
|
|
expectedMode?: number,
|
|
): Promise<boolean> {
|
|
try {
|
|
if (expectedMode !== undefined) {
|
|
const info = await stat(destination);
|
|
if (!info.isFile() || (info.mode & 0o777) !== expectedMode) return false;
|
|
}
|
|
return digest(await readFile(destination)) === expectedDigest;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function installExecutable(bytes: Uint8Array, destination: string): Promise<void> {
|
|
await mkdir(dirname(destination), { recursive: true });
|
|
const tmp = join(
|
|
dirname(destination),
|
|
`.scriptc-exe-bin-${process.pid}-${Math.random().toString(36).slice(2)}`,
|
|
);
|
|
try {
|
|
await writeFile(tmp, bytes, { mode: 0o600 });
|
|
await chmod(tmp, 0o777 & ~process.umask());
|
|
await rename(tmp, destination);
|
|
} finally {
|
|
await rm(tmp, { force: true }).catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
export async function readEarlyExecutableCache(
|
|
root: string | null,
|
|
options: EarlyExecutableCacheOptions,
|
|
): Promise<EarlyExecutableCacheHit | null> {
|
|
return readExecutableCache(root, options);
|
|
}
|
|
|
|
async function readExecutableCache(
|
|
root: string | null,
|
|
options: EarlyExecutableCacheOptions,
|
|
validateRoute?: () => Promise<boolean>,
|
|
): Promise<EarlyExecutableCacheHit | null> {
|
|
if (root === null) return null;
|
|
const path = stampPath(root, options);
|
|
try {
|
|
const stamp = JSON.parse(await readFile(path, "utf8")) as EarlyExecutableCacheStamp;
|
|
const { integrity, ...unsigned } = stamp;
|
|
if (
|
|
stamp.version !== 1 ||
|
|
stamp.key !== cacheKey(options) ||
|
|
!validFrontendInputSnapshot(stamp.frontend) ||
|
|
!validNativeFeatures(stamp.native) ||
|
|
stamp.files?.translationUnit?.name !== "program.tu" ||
|
|
!/^[0-9a-f]{64}$/.test(stamp.files.translationUnit.digest) ||
|
|
(stamp.files.ir !== null &&
|
|
(stamp.files.ir?.name !== "program.ir.json" ||
|
|
!/^[0-9a-f]{64}$/.test(stamp.files.ir.digest))) ||
|
|
(stamp.files.executable !== null &&
|
|
(stamp.files.executable?.name !== "program.bin" ||
|
|
!/^[0-9a-f]{64}$/.test(stamp.files.executable.digest))) ||
|
|
(stamp.files.debugSymbols !== undefined &&
|
|
(stamp.files.executable === null ||
|
|
stamp.files.debugSymbols.name !== "program.dsym" ||
|
|
!/^[0-9a-f]{64}$/.test(stamp.files.debugSymbols.digest))) ||
|
|
(stamp.files.executable !== null &&
|
|
needsDarwinDebugSymbols(
|
|
options.target.split(":")[1] ?? "",
|
|
options.optimization,
|
|
options.strip,
|
|
) !==
|
|
(stamp.files.debugSymbols !== undefined)) ||
|
|
(stamp.files.executable === null) !== (stamp.nativeDependencies === null) ||
|
|
(stamp.nativeDependencies !== null && !Array.isArray(stamp.nativeDependencies)) ||
|
|
(stamp.files.ir !== null) !== options.emitIr ||
|
|
stampIntegrity(unsigned) !== integrity ||
|
|
(stamp.native.nodeIsland === true &&
|
|
(stamp.libnode ?? "") !== (options.libnodeIdentity?.() ?? "")) ||
|
|
!frontendInputsStillMatch(
|
|
validateRoute === undefined ? stamp.frontend : routedFrontendCandidate(stamp.frontend),
|
|
executableFrontendOutputExclusions(options, stamp.native.backend),
|
|
)
|
|
)
|
|
return null;
|
|
|
|
// Startup may defer driver discovery until the source/configuration proof
|
|
// matches. A frontend-only entry cannot serve that route, so it should
|
|
// fall through without probing a driver the full compiler will probe too.
|
|
if (validateRoute !== undefined && stamp.files.executable === null) return null;
|
|
const directory = dirname(path);
|
|
const [translationUnit, ir, executable, debugSymbols] = await Promise.all([
|
|
readCachedFile(
|
|
join(directory, stamp.files.translationUnit.name),
|
|
stamp.files.translationUnit.digest,
|
|
),
|
|
stamp.files.ir === null
|
|
? Promise.resolve(null)
|
|
: readCachedFile(join(directory, stamp.files.ir.name), stamp.files.ir.digest),
|
|
stamp.files.executable === null
|
|
? Promise.resolve(null)
|
|
: readCachedFile(
|
|
join(directory, stamp.files.executable.name),
|
|
stamp.files.executable.digest,
|
|
),
|
|
stamp.files.debugSymbols === undefined
|
|
? Promise.resolve(null)
|
|
: readCachedFile(
|
|
join(directory, stamp.files.debugSymbols.name),
|
|
stamp.files.debugSymbols.digest,
|
|
),
|
|
]);
|
|
if (
|
|
translationUnit === null ||
|
|
(stamp.files.ir !== null && ir === null) ||
|
|
(stamp.files.executable !== null && executable === null) ||
|
|
(stamp.files.debugSymbols !== undefined && debugSymbols === null)
|
|
)
|
|
return null;
|
|
|
|
if (validateRoute !== undefined) {
|
|
if (!(await validateRoute())) return null;
|
|
// The compiler proof is checked AFTER deferred discovery by the routed
|
|
// reader. Recheck source/configuration bytes here too before restoring.
|
|
if (
|
|
!frontendInputsStillMatch(
|
|
stamp.frontend,
|
|
executableFrontendOutputExclusions(options, stamp.native.backend),
|
|
)
|
|
)
|
|
return null;
|
|
}
|
|
// Validate the native proof before restoring frontend artifacts: replacing
|
|
// a TU can update its output directory metadata, which is itself part of
|
|
// compileC's same-output dependency snapshot.
|
|
let executableRestored =
|
|
executable !== null &&
|
|
stamp.nativeDependencies !== null &&
|
|
(await nativeArtifactDependenciesStillMatch(stamp.nativeDependencies)) &&
|
|
// Recheck after hashing every dependency: a concurrent tool/runtime
|
|
// update must not win the window immediately before installation.
|
|
(await nativeArtifactDependenciesStillMatch(stamp.nativeDependencies));
|
|
if (validateRoute !== undefined && !executableRestored) return null;
|
|
const paths = outputPaths(options, stamp.native.backend);
|
|
if (executableRestored) {
|
|
try {
|
|
if (debugSymbols !== null) await installDarwinDebugSymbols(debugSymbols, options.outPath);
|
|
const expectedMode = 0o777 & ~process.umask();
|
|
if (!(await fileMatches(options.outPath, stamp.files.executable!.digest, expectedMode))) {
|
|
await installExecutable(executable!, options.outPath);
|
|
}
|
|
if (debugSymbols === null && options.target.split(":")[1] === "darwin") {
|
|
await rm(`${options.outPath}.dSYM`, { recursive: true, force: true });
|
|
}
|
|
} catch {
|
|
executableRestored = false;
|
|
}
|
|
}
|
|
// A build that compiles from the cached translation unit compiles its own copy: a concurrent
|
|
// build of another program with the same entry name may replace the shared names meanwhile.
|
|
let llvmSourcePath: string | undefined;
|
|
let irSourcePath: string | undefined;
|
|
let llvmPathCurrent = false;
|
|
let irPathCurrent = false;
|
|
if (executableRestored) {
|
|
if (!(await fileMatches(paths.llvmPath, stamp.files.translationUnit.digest)))
|
|
await installBytes(translationUnit, paths.llvmPath);
|
|
if (
|
|
ir !== null &&
|
|
stamp.files.ir !== null &&
|
|
!(await fileMatches(paths.irPath, stamp.files.ir.digest))
|
|
)
|
|
await installBytes(ir, paths.irPath);
|
|
} else {
|
|
// The extension keeps clang reading the copy as LLVM IR rather than as a linker input.
|
|
llvmSourcePath = `${privateSiblingPath(paths.llvmPath, "llvm")}.ll`;
|
|
await installBytes(translationUnit, llvmSourcePath);
|
|
llvmPathCurrent = await fileMatches(paths.llvmPath, stamp.files.translationUnit.digest);
|
|
if (ir !== null && stamp.files.ir !== null) {
|
|
irSourcePath = privateSiblingPath(paths.irPath, "ir");
|
|
await installBytes(ir, irSourcePath);
|
|
irPathCurrent = await fileMatches(paths.irPath, stamp.files.ir.digest);
|
|
}
|
|
}
|
|
const now = new Date();
|
|
await Promise.all(
|
|
[
|
|
path,
|
|
join(directory, stamp.files.translationUnit.name),
|
|
...(stamp.files.ir === null ? [] : [join(directory, stamp.files.ir.name)]),
|
|
...(stamp.files.executable === null ? [] : [join(directory, stamp.files.executable.name)]),
|
|
...(stamp.files.debugSymbols === undefined
|
|
? []
|
|
: [join(directory, stamp.files.debugSymbols.name)]),
|
|
].map((cachePath) => utimes(cachePath, now, now).catch(() => undefined)),
|
|
);
|
|
return {
|
|
llvmPath: paths.llvmPath,
|
|
native: stamp.native,
|
|
executableRestored,
|
|
frontend: stamp.frontend,
|
|
...(Array.isArray(stamp.performance) && stamp.performance.length > 0
|
|
? { performance: stamp.performance }
|
|
: {}),
|
|
...(isIslands(stamp.islands) ? { islands: stamp.islands } : {}),
|
|
...(isProfileMatch(stamp.profileMatch) ? { profileMatch: stamp.profileMatch } : {}),
|
|
...(ir === null ? {} : { irPath: paths.irPath }),
|
|
...(llvmSourcePath === undefined ? {} : { llvmSourcePath }),
|
|
...(irSourcePath === undefined ? {} : { irSourcePath }),
|
|
...(llvmPathCurrent ? { llvmPathCurrent: true as const } : {}),
|
|
...(irPathCurrent ? { irPathCurrent: true as const } : {}),
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Follow the exact-invocation route without hashing/importing the complete
|
|
* compiler package. Source/configuration misses skip deferred driver discovery;
|
|
* a hit still proves the current native environment before restoring output.
|
|
* The route is only an index: the full payload key retains that environment. */
|
|
export async function readRoutedExecutableCache(
|
|
root: string | null,
|
|
options: EarlyExecutableRouteOptions,
|
|
): Promise<EarlyExecutableCacheHit | null> {
|
|
if (root === null) return null;
|
|
try {
|
|
const routeFile = routePath(root, options);
|
|
const route = JSON.parse(await readFile(routeFile, "utf8")) as EarlyExecutableRouteStamp;
|
|
const { integrity, ...unsigned } = route;
|
|
if (
|
|
route.version !== 2 ||
|
|
!/^[0-9a-f]{64}$/.test(route.key) ||
|
|
!/^[0-9a-f]{64}$/.test(route.implementation) ||
|
|
typeof route.nativeEnvironment !== "string" ||
|
|
(typeof options.nativeEnvironment === "string" &&
|
|
options.nativeEnvironment !== route.nativeEnvironment) ||
|
|
routeIntegrity(unsigned) !== integrity
|
|
)
|
|
return null;
|
|
const proofFile = implementationProofPath(root, route.implementation);
|
|
const proof = JSON.parse(
|
|
await readFile(proofFile, "utf8"),
|
|
) as EarlyExecutableImplementationProof;
|
|
const { integrity: proofIntegrity, ...proofUnsigned } = proof;
|
|
if (
|
|
proof.version !== 1 ||
|
|
proof.implementation !== route.implementation ||
|
|
!Array.isArray(proof.dependencies) ||
|
|
implementationProofIntegrity(proofUnsigned) !== proofIntegrity
|
|
)
|
|
return null;
|
|
const complete: EarlyExecutableCacheOptions = {
|
|
...options,
|
|
nativeEnvironment: route.nativeEnvironment,
|
|
implementation: route.implementation,
|
|
implementationDependencies: proof.dependencies,
|
|
};
|
|
if (cacheKey(complete) !== route.key) return null;
|
|
const hit = await readExecutableCache(root, complete, async () => {
|
|
const environment =
|
|
typeof options.nativeEnvironment === "function"
|
|
? await options.nativeEnvironment()
|
|
: options.nativeEnvironment;
|
|
return (
|
|
environment === route.nativeEnvironment &&
|
|
(await compilerImplementationDependenciesStillMatch(proof.dependencies))
|
|
);
|
|
});
|
|
if (hit?.executableRestored !== true) return null;
|
|
const now = new Date();
|
|
await Promise.all(
|
|
[routeFile, proofFile].map((cachePath) => utimes(cachePath, now, now).catch(() => undefined)),
|
|
);
|
|
return hit;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/** Publish the lightweight lookup metadata for an already-validated payload.
|
|
* Full-compiler cache hits call this as a repair path: route/proof files can be
|
|
* evicted independently of the larger executable entry, and one fallback
|
|
* lookup should make later CLI invocations lightweight again. */
|
|
export async function publishEarlyExecutableRoute(
|
|
root: string | null,
|
|
options: EarlyExecutableCacheOptions,
|
|
): Promise<void> {
|
|
if (root === null) return;
|
|
const proofDestination = implementationProofPath(root, options.implementation);
|
|
const proofUnsigned: Omit<EarlyExecutableImplementationProof, "integrity"> = {
|
|
version: 1,
|
|
implementation: options.implementation,
|
|
dependencies: options.implementationDependencies,
|
|
};
|
|
const proof: EarlyExecutableImplementationProof = {
|
|
...proofUnsigned,
|
|
integrity: implementationProofIntegrity(proofUnsigned),
|
|
};
|
|
await mkdir(dirname(proofDestination), { recursive: true, mode: 0o700 });
|
|
const proofTmp = `${proofDestination}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`;
|
|
try {
|
|
await writeFile(proofTmp, `${JSON.stringify(proof)}\n`, { mode: 0o600 });
|
|
await installCacheMetadata(proofTmp, proofDestination);
|
|
} finally {
|
|
await rm(proofTmp, { force: true }).catch(() => undefined);
|
|
}
|
|
|
|
const routeOptions: EarlyExecutableRouteOptions = options;
|
|
const routeDestination = routePath(root, routeOptions);
|
|
const routeUnsigned: Omit<EarlyExecutableRouteStamp, "integrity"> = {
|
|
version: 2,
|
|
key: cacheKey(options),
|
|
implementation: options.implementation,
|
|
nativeEnvironment: options.nativeEnvironment,
|
|
};
|
|
const route: EarlyExecutableRouteStamp = {
|
|
...routeUnsigned,
|
|
integrity: routeIntegrity(routeUnsigned),
|
|
};
|
|
await mkdir(dirname(routeDestination), { recursive: true, mode: 0o700 });
|
|
const routeTmp = `${routeDestination}.tmp-${process.pid}-${Math.random().toString(36).slice(2)}`;
|
|
try {
|
|
await writeFile(routeTmp, `${JSON.stringify(route)}\n`, { mode: 0o600 });
|
|
await installCacheMetadata(routeTmp, routeDestination);
|
|
} finally {
|
|
await rm(routeTmp, { force: true }).catch(() => undefined);
|
|
}
|
|
}
|
|
|
|
export async function publishEarlyExecutableCache(
|
|
root: string | null,
|
|
options: EarlyExecutableCacheOptions,
|
|
result: EarlyExecutableCachePublish,
|
|
): Promise<void> {
|
|
if (root === null || !result.frontend.stable) return;
|
|
const destination = dirname(stampPath(root, options));
|
|
const parent = dirname(destination);
|
|
const stage = join(
|
|
parent,
|
|
`.tmp-${basename(destination).slice(0, 12)}-${process.pid}-${Math.random().toString(36).slice(2)}`,
|
|
);
|
|
try {
|
|
await mkdir(stage, { recursive: true, mode: 0o700 });
|
|
const publishFile = async (source: string, name: string): Promise<CachedExecutableFile> => {
|
|
const target = join(stage, name);
|
|
await copyFile(source, target);
|
|
await chmod(target, 0o600);
|
|
return { name, digest: digest(await readFile(target)) };
|
|
};
|
|
const publishExecutable =
|
|
result.nativeDependencies === undefined || !result.executableRestored
|
|
? Promise.resolve(null)
|
|
: publishFile(options.outPath, "program.bin");
|
|
const [translationUnit, ir, executable] = await Promise.all([
|
|
publishFile(result.llvmPath, "program.tu"),
|
|
result.irPath === undefined
|
|
? Promise.resolve(null)
|
|
: publishFile(result.irPath, "program.ir.json"),
|
|
publishExecutable,
|
|
]);
|
|
let debugSymbols: CachedExecutableFile | undefined;
|
|
if (
|
|
executable !== null &&
|
|
needsDarwinDebugSymbols(
|
|
options.target.split(":")[1] ?? "",
|
|
options.optimization,
|
|
options.strip,
|
|
)
|
|
) {
|
|
const bytes = await readDarwinDebugSymbols(options.outPath);
|
|
await writeFile(join(stage, "program.dsym"), bytes, { mode: 0o600 });
|
|
debugSymbols = { name: "program.dsym", digest: digest(bytes) };
|
|
}
|
|
if (
|
|
!frontendInputsStillMatch(
|
|
result.frontend,
|
|
executableFrontendOutputExclusions(options, result.native.backend),
|
|
)
|
|
)
|
|
return;
|
|
const unsigned: Omit<EarlyExecutableCacheStamp, "integrity"> = {
|
|
version: 1,
|
|
key: cacheKey(options),
|
|
frontend: result.frontend,
|
|
files: {
|
|
translationUnit,
|
|
ir,
|
|
executable,
|
|
...(debugSymbols === undefined ? {} : { debugSymbols }),
|
|
},
|
|
nativeDependencies: executable === null ? null : result.nativeDependencies!,
|
|
native: result.native,
|
|
...(result.performance === undefined || result.performance.length === 0
|
|
? {}
|
|
: { performance: result.performance }),
|
|
...(result.islands === undefined ? {} : { islands: result.islands }),
|
|
...(result.native.nodeIsland === true ? { libnode: options.libnodeIdentity?.() ?? "" } : {}),
|
|
...(result.profileMatch === undefined ? {} : { profileMatch: result.profileMatch }),
|
|
};
|
|
const stamp: EarlyExecutableCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) };
|
|
await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 });
|
|
await mkdir(destination, { recursive: true, mode: 0o700 });
|
|
const install = async (name: string): Promise<void> => {
|
|
const source = join(stage, name);
|
|
const target = join(destination, name);
|
|
await rename(source, target).catch(async () => {
|
|
await rm(target, { force: true });
|
|
await rename(source, target);
|
|
});
|
|
};
|
|
await install(translationUnit.name);
|
|
if (ir !== null) await install(ir.name);
|
|
if (executable !== null) await install(executable.name);
|
|
if (debugSymbols !== undefined) await install(debugSymbols.name);
|
|
await install("stamp.json");
|
|
await publishEarlyExecutableRoute(root, options);
|
|
} finally {
|
|
await rm(stage, { recursive: true, force: true }).catch(() => undefined);
|
|
}
|
|
}
|