refactor: share early cache primitives (#228)

- Extract common cache hashing, paths, payload I/O, and feature validation.
- Preserve library and executable stamp schemas and on-disk identities.

packages/compiler/src/executable/early-cache.ts

@@ -7,6 +7,17 @@ import { compilerReleaseVersion } from "../library/sidecar.js";
 import { nativeArtifactDependenciesStillMatch, type NativeArtifactDependency } from "../backend/cc.js";
 import type { CompilerImplementationDependency } from "../library/implementation-identity.js";
 import { compilerImplementationDependenciesStillMatch, compilerImplementationRoot } from "../library/implementation-identity.js";
+import {
+  cacheKey as sharedCacheKey,
+  digest,
+  frontendOutputExclusions,
+  installBytes,
+  outputPaths,
+  readCachedFile,
+  stampIntegrity as sharedStampIntegrity,
+  stampPath as sharedStampPath,
+  validNativeFeatures as validSharedNativeFeatures,
+} from "../library/cache-primitives.js";
 
 interface CachedExecutableFile {
   name: string;
@@ -151,49 +162,40 @@ const BOOLEAN_NATIVE_KEYS = [
 ] as const satisfies readonly (keyof EarlyExecutableNativeFeatures)[];
 
 function validNativeFeatures(value: unknown): value is EarlyExecutableNativeFeatures {
-  if (value === null || typeof value !== "object") return false;
-  const native = value as Partial<EarlyExecutableNativeFeatures>;
-  return (native.backend === "c" || native.backend === "llvm") &&
-    (native.optimization === undefined || native.optimization === "dev") &&
-    (native.llvmRefusal === undefined || typeof native.llvmRefusal === "string") &&
-    BOOLEAN_NATIVE_KEYS.every((key) => typeof native[key] === "boolean");
-}
-
-function digest(bytes: Uint8Array): string {
-  return createHash("sha256").update(bytes).digest("hex");
+  return validSharedNativeFeatures<EarlyExecutableNativeFeatures>(
+    value,
+    BOOLEAN_NATIVE_KEYS,
+    (native) =>
+      (native.optimization === undefined || native.optimization === "dev") &&
+      (native.llvmRefusal === undefined || typeof native.llvmRefusal === "string"),
+  );
 }
 
 function cacheKey(options: EarlyExecutableCacheOptions): string {
-  const hash = createHash("sha256")
-    .update("early-executable-v1\0")
-    .update(compilerReleaseVersion()).update("\0")
-    .update(resolve(options.entryPath)).update("\0")
-    .update(resolve(options.outDir)).update("\0")
-    .update(resolve(options.outPath)).update("\0")
-    .update(options.emitIr ? "emit-ir" : "no-ir").update("\0")
-    .update(options.sanitize ? "sanitize" : "plain").update("\0")
-    .update(options.dynamic ? "dynamic" : "static").update("\0")
-    .update(options.backend).update("\0");
-  if (options.optimization === "dev") hash.update("optimization-dev\0");
-  hash
-    .update(options.npmStatic === null
+  const ffiParts: (string | Uint8Array)[] = options.ffiProfile === null
+    ? ["<ffi-off>"]
+    : [resolve(options.ffiProfile.path), options.ffiProfile.bytes];
+  return sharedCacheKey("early-executable-v1", [
+    resolve(options.entryPath),
+    resolve(options.outDir),
+    resolve(options.outPath),
+    options.emitIr ? "emit-ir" : "no-ir",
+    options.sanitize ? "sanitize" : "plain",
+    options.dynamic ? "dynamic" : "static",
+    options.backend,
+    ...(options.optimization === "dev" ? ["optimization-dev"] : []),
+    options.npmStatic === null
       ? "<npm-static-off>"
       : options.npmStatic === "auto"
         ? "<npm-static-auto>"
-        : JSON.stringify(options.npmStatic)).update("\0")
-    .update(options.target).update("\0")
-    .update(options.compiler.join("\x1f")).update("\0")
-    .update(options.nativeEnvironment).update("\0")
-    .update(options.nodeVersion).update("\0")
-    .update(options.implementation).update("\0");
-  if (options.ffiProfile === null) {
-    hash.update("<ffi-off>");
-  } else {
-    hash
-      .update(resolve(options.ffiProfile.path)).update("\0")
-      .update(options.ffiProfile.bytes);
-  }
-  return hash.digest("hex");
+        : JSON.stringify(options.npmStatic),
+    options.target,
+    options.compiler.join("\x1f"),
+    options.nativeEnvironment,
+    options.nodeVersion,
+    options.implementation,
+    ...ffiParts,
+  ]);
 }
 
 function routeKey(options: EarlyExecutableRouteOptions): string {
@@ -268,72 +270,18 @@ async function installCacheMetadata(source: string, destination: string): Promis
 }
 
 function stampPath(root: string, options: EarlyExecutableCacheOptions): string {
-  return join(root, "early-exe", cacheKey(options), "stamp.json");
+  return sharedStampPath(root, "early-exe", cacheKey(options));
 }
 
 function stampIntegrity(stamp: Omit<EarlyExecutableCacheStamp, "integrity">): string {
-  return createHash("sha256")
-    .update("early-executable-stamp-v1\0")
-    .update(JSON.stringify(stamp))
-    .digest("hex");
+  return sharedStampIntegrity("early-executable-stamp-v1", stamp);
 }
 
-function outputPaths(options: EarlyExecutableCacheOptions, backend: "c" | "llvm"): {
-  cPath: string;
-  staleCPath: string;
-  irPath: string;
-} {
-  const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "");
-  return {
-    cPath: join(options.outDir, `${stem}.${backend === "llvm" ? "ll" : "c"}`),
-    staleCPath: join(options.outDir, `${stem}.${backend === "llvm" ? "c" : "ll"}`),
-    irPath: join(options.outDir, `${stem}.ir.json`),
-  };
-}
-
-function frontendOutputExclusions(
+function executableFrontendOutputExclusions(
   options: EarlyExecutableCacheOptions,
   backend: "c" | "llvm",
-): { outputPaths: string[]; outputDirectories: Set<string> } {
-  const paths = outputPaths(options, backend);
-  const outputArtifacts = [
-    paths.cPath,
-    paths.staleCPath,
-    ...(options.emitIr ? [paths.irPath] : []),
-    options.outPath,
-  ].map((path) => resolve(path));
-  const outputDirectories = new Set<string>();
-  for (const artifact of outputArtifacts) {
-    for (let directory = dirname(artifact); ; directory = dirname(directory)) {
-      outputDirectories.add(directory);
-      if (dirname(directory) === directory) break;
-    }
-  }
-  return { outputPaths: outputArtifacts, outputDirectories };
-}
-
-async function readCachedFile(path: string, expected: string): Promise<Buffer | null> {
-  try {
-    const bytes = await readFile(path);
-    return digest(bytes) === expected ? bytes : null;
-  } catch {
-    return null;
-  }
-}
-
-async function installBytes(bytes: Uint8Array, destination: string): Promise<void> {
-  await mkdir(dirname(destination), { recursive: true });
-  const tmp = join(
-    dirname(destination),
-    `.scriptc-exe-hit-${process.pid}-${Math.random().toString(36).slice(2)}`,
-  );
-  try {
-    await writeFile(tmp, bytes, { mode: 0o600 });
-    await chmod(tmp, 0o666 & ~process.umask());
-    await rename(tmp, destination);
-  } finally {
-    await rm(tmp, { force: true }).catch(() => undefined);
-  }
+): ReturnType<typeof frontendOutputExclusions> {
+  return frontendOutputExclusions(options, backend, "", [options.outPath]);
 }
 
 async function fileMatches(
@@ -395,7 +343,7 @@ export async function readEarlyExecutableCache(
       stampIntegrity(unsigned) !== integrity ||
       !frontendInputsStillMatch(
         stamp.frontend,
-        frontendOutputExclusions(options, stamp.native.backend),
+        executableFrontendOutputExclusions(options, stamp.native.backend),
       )
     ) return null;
 
@@ -598,7 +546,7 @@ export async function publishEarlyExecutableCache(
     ]);
     if (!frontendInputsStillMatch(
       result.frontend,
-      frontendOutputExclusions(options, result.native.backend),
+      executableFrontendOutputExclusions(options, result.native.backend),
     )) return;
     const unsigned: Omit<EarlyExecutableCacheStamp, "integrity"> = {
       version: 1,

packages/compiler/src/library/cache-primitives.ts

@@ -0,0 +1,120 @@
+import { createHash } from "node:crypto";
+import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import type { FrontendInputExclusions } from "../frontend/input-tracker.js";
+import { compilerReleaseVersion } from "./sidecar.js";
+
+export type CacheBackend = "c" | "llvm";
+
+interface CacheOutputOptions {
+  entryPath: string;
+  outDir: string;
+  emitIr: boolean;
+}
+
+export interface CacheOutputPaths {
+  cPath: string;
+  staleCPath: string;
+  irPath: string;
+}
+
+export function digest(bytes: Uint8Array): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+export function cacheKey(
+  namespace: string,
+  parts: readonly (string | Uint8Array)[],
+): string {
+  const hash = createHash("sha256")
+    .update(namespace).update("\0")
+    .update(compilerReleaseVersion()).update("\0");
+  for (const [index, part] of parts.entries()) {
+    hash.update(part);
+    if (index + 1 < parts.length) hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+export function stampPath(root: string, namespace: string, key: string): string {
+  return join(root, namespace, key, "stamp.json");
+}
+
+export function stampIntegrity(namespace: string, stamp: object): string {
+  return createHash("sha256")
+    .update(namespace).update("\0")
+    .update(JSON.stringify(stamp))
+    .digest("hex");
+}
+
+export function outputPaths(
+  options: CacheOutputOptions,
+  backend: CacheBackend,
+  stemSuffix = "",
+): CacheOutputPaths {
+  const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "") + stemSuffix;
+  return {
+    cPath: join(options.outDir, `${stem}.${backend === "llvm" ? "ll" : "c"}`),
+    staleCPath: join(options.outDir, `${stem}.${backend === "llvm" ? "c" : "ll"}`),
+    irPath: join(options.outDir, `${stem}.ir.json`),
+  };
+}
+
+export function frontendOutputExclusions(
+  options: CacheOutputOptions,
+  backend: CacheBackend,
+  stemSuffix: string,
+  additionalPaths: readonly string[],
+): FrontendInputExclusions {
+  const paths = outputPaths(options, backend, stemSuffix);
+  const outputArtifacts = [
+    paths.cPath,
+    paths.staleCPath,
+    ...(options.emitIr ? [paths.irPath] : []),
+    ...additionalPaths,
+  ].map((path) => resolve(path));
+  const outputDirectories = new Set<string>();
+  for (const artifact of outputArtifacts) {
+    for (let directory = dirname(artifact); ; directory = dirname(directory)) {
+      outputDirectories.add(directory);
+      if (dirname(directory) === directory) break;
+    }
+  }
+  return { outputPaths: outputArtifacts, outputDirectories };
+}
+
+export async function readCachedFile(path: string, expected: string): Promise<Buffer | null> {
+  try {
+    const bytes = await readFile(path);
+    return digest(bytes) === expected ? bytes : null;
+  } catch {
+    return null;
+  }
+}
+
+export async function installBytes(bytes: Uint8Array, destination: string): Promise<void> {
+  await mkdir(dirname(destination), { recursive: true });
+  const tmp = join(
+    dirname(destination),
+    `.scriptc-early-hit-${process.pid}-${Math.random().toString(36).slice(2)}`,
+  );
+  try {
+    await writeFile(tmp, bytes, { mode: 0o600 });
+    await chmod(tmp, 0o666 & ~process.umask());
+    await rename(tmp, destination);
+  } finally {
+    await rm(tmp, { force: true }).catch(() => undefined);
+  }
+}
+
+export function validNativeFeatures<T extends { backend: CacheBackend }>(
+  value: unknown,
+  booleanKeys: readonly (keyof T)[],
+  validAdditional: (native: Partial<T>) => boolean = () => true,
+): value is T {
+  if (value === null || typeof value !== "object") return false;
+  const native = value as Partial<T>;
+  return (native.backend === "c" || native.backend === "llvm") &&
+    booleanKeys.every((key) => typeof native[key] === "boolean") &&
+    validAdditional(native);
+}

packages/compiler/src/library/early-cache.ts

@@ -1,15 +1,24 @@
-import { createHash } from "node:crypto";
 import { chmod, copyFile, mkdir, readFile, rename, rm, utimes, writeFile } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
 import { promisify } from "node:util";
 import { deserialize as deserializeV8, serialize as serializeV8 } from "node:v8";
 import { gzip, gunzip } from "node:zlib";
-import { compilerReleaseVersion } from "./sidecar.js";
 import type { IrModule } from "../ir/nodes.js";
 import { IR_VERSION } from "../ir/serialize.js";
-import { frontendInputsSemanticallyMatch, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputExclusions, type FrontendInputSnapshot } from "../frontend/input-tracker.js";
+import { frontendInputsSemanticallyMatch, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot } from "../frontend/input-tracker.js";
 import { rebaseSourceLocations, semanticallyEqualSource, sourceLineRebaseIsIdentity } from "./semantic-source.js";
 import { compilerImplementationIdentity } from "./implementation-identity.js";
+import {
+  cacheKey as sharedCacheKey,
+  digest,
+  frontendOutputExclusions as sharedFrontendOutputExclusions,
+  installBytes,
+  outputPaths as sharedOutputPaths,
+  readCachedFile,
+  stampIntegrity as sharedStampIntegrity,
+  stampPath as sharedStampPath,
+  validNativeFeatures as validSharedNativeFeatures,
+} from "./cache-primitives.js";
 
 const gzipAsync = promisify(gzip);
 const gunzipAsync = promisify(gunzip);
@@ -102,44 +111,40 @@ export interface SemanticLibraryCacheHit {
   changedSources: string[];
 }
 
-function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeatures {
-  if (value === null || typeof value !== "object") return false;
-  const native = value as Partial<EarlyLibraryNativeFeatures>;
-  return (native.backend === "c" || native.backend === "llvm") &&
-    [
-      native.regex,
-      native.assert,
-      native.inspect,
-      native.symbol,
-      native.searchParams,
-      native.emitter,
-      native.zlib,
-      native.copying,
-      native.textDecoderLegacy,
-    ].every((flag) => typeof flag === "boolean") &&
-    (native.buildId === undefined || /^[0-9a-f]{16}$/.test(native.buildId));
-}
+const BOOLEAN_NATIVE_KEYS = [
+  "regex",
+  "assert",
+  "inspect",
+  "symbol",
+  "searchParams",
+  "emitter",
+  "zlib",
+  "copying",
+  "textDecoderLegacy",
+] as const satisfies readonly (keyof EarlyLibraryNativeFeatures)[];
 
-function digest(bytes: Uint8Array): string {
-  return createHash("sha256").update(bytes).digest("hex");
+function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeatures {
+  return validSharedNativeFeatures<EarlyLibraryNativeFeatures>(
+    value,
+    BOOLEAN_NATIVE_KEYS,
+    (native) => native.buildId === undefined || /^[0-9a-f]{16}$/.test(native.buildId),
+  );
 }
 
 function cacheKey(options: EarlyLibraryCacheOptions): string {
-  return createHash("sha256")
-    .update("early-library-v2\0")
-    .update(compilerReleaseVersion()).update("\0")
-    .update(resolve(options.profilePath)).update("\0")
-    .update(options.profileBytes).update("\0")
-    .update(resolve(options.entryPath)).update("\0")
-    .update(resolve(options.outDir)).update("\0")
-    .update(options.outPath === undefined ? "<default>" : resolve(options.outPath)).update("\0")
-    .update(options.emitIr ? "emit-ir" : "no-ir").update("\0")
-    .update(options.sanitize ? "sanitize" : "plain").update("\0")
-    .update(options.target).update("\0")
-    .update(options.compiler.join("\x1f")).update("\0")
-    .update(options.nodeVersion).update("\0")
-    .update(options.implementation)
-    .digest("hex");
+  return sharedCacheKey("early-library-v2", [
+    resolve(options.profilePath),
+    options.profileBytes,
+    resolve(options.entryPath),
+    resolve(options.outDir),
+    options.outPath === undefined ? "<default>" : resolve(options.outPath),
+    options.emitIr ? "emit-ir" : "no-ir",
+    options.sanitize ? "sanitize" : "plain",
+    options.target,
+    options.compiler.join("\x1f"),
+    options.nodeVersion,
+    options.implementation,
+  ]);
 }
 
 /**
@@ -154,27 +159,15 @@ export async function libraryFrontendImplementationFingerprint(): Promise<string
 }
 
 function stampPath(root: string, options: EarlyLibraryCacheOptions): string {
-  return join(root, "early-lib", cacheKey(options), "stamp.json");
+  return sharedStampPath(root, "early-lib", cacheKey(options));
 }
 
 function stampIntegrity(stamp: Omit<EarlyLibraryCacheStamp, "integrity">): string {
-  return createHash("sha256")
-    .update("early-library-stamp-v2\0")
-    .update(JSON.stringify(stamp))
-    .digest("hex");
+  return sharedStampIntegrity("early-library-stamp-v2", stamp);
 }
 
-function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm"): {
-  cPath: string;
-  staleCPath: string;
-  irPath: string;
-} {
-  const stem = basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "");
-  return {
-    cPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "ll" : "c"}`),
-    staleCPath: join(options.outDir, `${stem}.lib.${backend === "llvm" ? "c" : "ll"}`),
-    irPath: join(options.outDir, `${stem}.lib.ir.json`),
-  };
+function outputPaths(options: EarlyLibraryCacheOptions, backend: "c" | "llvm") {
+  return sharedOutputPaths(options, backend, ".lib");
 }
 
 function sidecarOutputPath(options: EarlyLibraryCacheOptions, configured: string | null): string {
@@ -193,47 +186,11 @@ function frontendOutputExclusions(
   options: EarlyLibraryCacheOptions,
   backend: "c" | "llvm",
   sidecarPath: string | undefined,
-): FrontendInputExclusions {
-  const paths = outputPaths(options, backend);
-  const outputArtifacts = [
-    paths.cPath,
-    paths.staleCPath,
-    ...(options.emitIr ? [paths.irPath] : []),
+): ReturnType<typeof sharedFrontendOutputExclusions> {
+  return sharedFrontendOutputExclusions(options, backend, ".lib", [
     archiveOutputPath(options),
     ...(sidecarPath === undefined ? [] : [sidecarPath]),
-  ].map((path) => resolve(path));
-  const outputDirectories = new Set<string>();
-  for (const artifact of outputArtifacts) {
-    for (let directory = dirname(artifact); ; directory = dirname(directory)) {
-      outputDirectories.add(directory);
-      if (dirname(directory) === directory) break;
-    }
-  }
-  return {
-    outputPaths: outputArtifacts,
-    outputDirectories,
-  };
-}
-
-async function readCachedFile(path: string, expected: string): Promise<Buffer | null> {
-  try {
-    const bytes = await readFile(path);
-    return digest(bytes) === expected ? bytes : null;
-  } catch {
-    return null;
-  }
-}
-
-async function installBytes(bytes: Uint8Array, destination: string): Promise<void> {
-  await mkdir(dirname(destination), { recursive: true });
-  const tmp = join(dirname(destination), `.scriptc-early-hit-${process.pid}-${Math.random().toString(36).slice(2)}`);
-  try {
-    await writeFile(tmp, bytes, { mode: 0o600 });
-    await chmod(tmp, 0o666 & ~process.umask());
-    await rename(tmp, destination);
-  } finally {
-    await rm(tmp, { force: true }).catch(() => undefined);
-  }
+  ]);
 }
 
 export async function readEarlyLibraryCache(