Verify runtime pack artifacts once per file identity, and store pack selections

Every native exe build that the executable cache does not restore hashed
each selected runtime artifact before staging it, and every exe build
parsed and validated the whole runtime-pack.json manifest to select its
artifacts.

- Staging still copies each artifact. It skips the SHA-256 of an artifact
  when the file's identity (device, inode, size, mtime, ctime) is the one
  recorded the last time that path matched the same size and digest, and
  the identity does not change while the file is read. Other artifacts are
  hashed as before, and their identities are recorded after they match.
- A pack's selection for a target, compiler version, feature set, flavor
  and mode is stored, keyed by the file identities of package.json and
  runtime-pack.json. A stored selection skips parsing and validating the
  manifest; the manifest is parsed only when a caller needs it, through
  runtimePackManifest.
- The native compiler keeps both in its cache, in the runtime-pack family,
  which prune covers.
This commit is contained in:
Chris Tate
2026-10-24 09:20:05 -05:00
parent e579f62556
commit 48cbf3a89e
8 changed files with 432 additions and 29 deletions
@@ -1,12 +1,26 @@
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, renameSync, statSync, utimesSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, expect, test } from "vitest";
import { afterEach, expect, test, vi } from "vitest";
import { nativeCodegenDiag } from "../diagnostics/diagnostic.js";
import { compilerReleaseVersion } from "../library/sidecar.js";
import type { NativeLinkFeatures } from "./native-link-info.js";
import { RuntimePackError } from "./runtime-pack-core.js";
import { selectNativeRuntimePack } from "./runtime-pack-native.js";
import { WASM32_WASI_TARGET } from "./targets.js";
import { RuntimePackError, type RuntimePackManifest } from "./runtime-pack-core.js";
import {
runtimePackManifest,
selectNativeRuntimePack,
stageNativeRuntimeSelection,
type NativeRuntimeSelection,
type RuntimePackStore,
} from "./runtime-pack-native.js";
import { MACOS_ARM64_TARGET, WASM32_WASI_TARGET } from "./targets.js";
vi.mock("node:crypto", async (original) => {
const crypto = await original<typeof import("node:crypto")>();
return { ...crypto, createHash: vi.fn(crypto.createHash) };
});
const dirs: string[] = [];
afterEach(async () =>
@@ -45,3 +59,136 @@ test("a missing native helper keeps the reinstall hint", () => {
"reinstall scriptc with optional dependencies enabled for this host",
);
});
/** A runtime pack with one runtime object and a license, and a store that
* keeps entries in memory. */
async function packFixture() {
const root = await mkdtemp(join(tmpdir(), "scriptc-runtime-pack-store-"));
dirs.push(root);
const target = MACOS_ARM64_TARGET;
const version = compilerReleaseVersion();
writeFileSync(
join(root, "package.json"),
JSON.stringify({ name: target.runtimePackPackage, version }),
);
mkdirSync(join(root, "artifacts"));
const object = join(root, "artifacts", "base.o");
writeFileSync(object, "base object");
writeFileSync(join(root, "license.txt"), "license");
const unit = {
source: "scr_base.c",
predicate: true,
variants: [
{
id: "default",
when: {},
defines: [],
path: "artifacts/base.o",
sha256: createHash("sha256").update("base object").digest("hex"),
size: Buffer.byteLength("base object"),
},
],
};
const manifest: RuntimePackManifest = {
schema: "scriptc.runtime-pack.v1",
format: 1,
package: target.runtimePackPackage,
version,
target: {
name: target.name,
llvm_triple: target.llvmTriple,
architecture: target.architecture,
object_format: target.objectFormat,
minimum_os: target.minimumOs,
},
runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" },
compiler: { command: "clang", identity: "fixture clang", target: target.llvmTriple },
macros: {
executable: [],
excluded: ["SCR_LIB", "SCR_THREAD_INSTANCES", "SCR_RC_AUDIT"],
sanitizer: "external-toolchain-required",
},
flavors: {
release: { optimization: "-O2", runtime_units: [unit] },
dev: { optimization: "-O0", runtime_units: [unit] },
},
archives: [],
system_libraries: [{ name: "System", predicate: true }],
licenses: [{ path: "license.txt", license: "fixture" }],
};
const manifestText = JSON.stringify(manifest);
writeFileSync(join(root, "runtime-pack.json"), manifestText);
const entries = new Map<string, string>();
const store: RuntimePackStore = {
read: (key) => entries.get(key) ?? null,
write: (key, text) => void entries.set(key, text),
};
const select = () =>
selectNativeRuntimePack(
root,
target,
version,
{} as NativeLinkFeatures,
"release",
"executable",
store,
);
let stages = 0;
const stage = (selection: NativeRuntimeSelection) => {
const directory = join(root, `stage-${stages++}`);
mkdirSync(directory);
return stageNativeRuntimeSelection(selection, directory, store);
};
return { root, object, manifestText, entries, select, stage };
}
test("a stored selection answers while the pack's files keep their identities", async () => {
const f = await packFixture();
const first = f.select();
expect(first.selected.systemLibraries).toEqual(["System"]);
expect(f.entries.size).toBe(1);
// The next selection reads the stored answer instead of the manifest.
const [key, text] = [...f.entries][0]!;
const stored = JSON.parse(text);
stored.selected.systemLibraries = ["Stored"];
f.entries.set(key, JSON.stringify(stored));
const second = f.select();
expect(second.selected.systemLibraries).toEqual(["Stored"]);
expect(second.licenses).toEqual(first.licenses);
expect(second.manifest).toBeUndefined();
expect(runtimePackManifest(second).flavors.release).toEqual(first.manifest!.flavors.release);
// Writing the manifest again, with the same text, gives it a new identity.
writeFileSync(join(f.root, "runtime-pack.json"), f.manifestText);
expect(f.select().selected.systemLibraries).toEqual(["System"]);
});
test.each(["replaced", "rewritten in place"])(
"staging hashes an artifact again when it is %s, even at the same size and mtime",
async (change) => {
const f = await packFixture();
const selection = f.select();
f.stage(selection);
const before = statSync(f.object);
const changed = change === "replaced" ? f.object + ".new" : f.object;
writeFileSync(changed, "x".repeat(before.size));
utimesSync(changed, before.atime, before.mtime);
if (changed !== f.object) renameSync(changed, f.object);
expect(() => f.stage(selection)).toThrow(
"runtime pack artifact hash mismatch: artifacts/base.o",
);
},
);
test("staging does not hash an unchanged artifact again", async () => {
const f = await packFixture();
const selection = f.select();
const hashes = () => vi.mocked(createHash).mock.calls.length;
let start = hashes();
f.stage(selection);
const firstStage = hashes() - start;
start = hashes();
const staged = f.stage(selection);
// Only the store key is hashed; the staged copy has the verified bytes.
expect(hashes() - start).toBe(firstStage - 1);
expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
});
@@ -1,6 +1,8 @@
/** Synchronous runtime-pack host for the statically compiled driver. Each
* selected artifact is hashed and copied from the same read into private
* storage. This host has no persistent cache or mutable installed link inputs. */
* selected artifact is verified and copied from the same read into private
* storage. With a store, a pack's selections and the identities of the
* artifacts it verified are kept between builds: an artifact whose file
* identity is unchanged since it was hashed is not hashed again. */
import type { NativeOptimization } from "./optimization.js";
import { createHash } from "node:crypto";
import {
@@ -10,6 +12,7 @@ import {
mkdirSync,
openSync,
readFileSync,
statSync,
writeSync,
} from "node:fs";
import { dirname, join } from "node:path";
@@ -34,13 +37,63 @@ export interface NativeRuntimePack {
export interface NativeRuntimeSelection {
root: string;
manifest: RuntimePackManifest;
/** The parsed manifest; a selection read from a store has none until
* runtimePackManifest parses it. */
manifest?: RuntimePackManifest;
licenses: RuntimePackManifest["licenses"];
packageText: string;
manifestText: string;
selected: RuntimePackArtifacts;
flavor: NativeOptimization;
}
/** Text entries kept between builds (the native compiler's cache). */
export interface RuntimePackStore {
read(key: string): string | null;
write(key: string, text: string): void;
}
interface FileIdentity {
dev: number;
ino: number;
size: number;
mtime: number;
ctime: number;
}
function fileIdentity(path: string): FileIdentity {
const info = statSync(path);
return {
dev: info.dev,
ino: info.ino,
size: info.size,
mtime: info.mtimeMs,
ctime: info.ctimeMs,
};
}
function sameIdentity(a: FileIdentity, b: FileIdentity): boolean {
return (
a.dev === b.dev &&
a.ino === b.ino &&
a.size === b.size &&
a.mtime === b.mtime &&
a.ctime === b.ctime
);
}
function storeKey(text: string): string {
return createHash("sha256").update(text).digest("hex");
}
/** A stored selection: what selecting from the pack's files, with these
* identities, returned. */
interface StoredSelection {
schema: 1;
licenses: RuntimePackManifest["licenses"];
selected: RuntimePackArtifacts;
}
export function selectNativeRuntimePack(
root: string,
target: NativeTargetSpec,
@@ -48,38 +101,197 @@ export function selectNativeRuntimePack(
features: NativeLinkFeatures,
flavor: NativeOptimization,
mode: RuntimePackMode = "executable",
store?: RuntimePackStore,
): NativeRuntimeSelection {
if (!existsSync(join(root, "package.json"))) {
const packagePath = join(root, "package.json");
const manifestPath = join(root, "runtime-pack.json");
if (!existsSync(packagePath)) {
throw new RuntimePackError(
`runtime pack for ${target.name} is not installed; install ${target.runtimePackPackage}@${compilerVersion} in your project or set SCRIPTC_RUNTIME_PACK to its directory`,
"missing",
"scriptc includes the runtime pack for its own platform only; other targets need their runtime pack package in the project",
);
}
const packageText = readFileSync(join(root, "package.json"), "utf8");
const manifestText = readFileSync(join(root, "runtime-pack.json"), "utf8");
// The pack's files by identity, with everything else selection reads.
let key: string | null = null;
let identities: FileIdentity[] = [];
if (store !== undefined) {
try {
identities = [fileIdentity(packagePath), fileIdentity(manifestPath)];
key = storeKey(
JSON.stringify({
schema: 1,
root,
identities,
target: target.name,
compilerVersion,
features,
flavor,
mode,
}),
);
const stored = readStoredSelection(store.read(key));
if (stored !== null) {
const packageText = readFileSync(packagePath, "utf8");
const manifestText = readFileSync(manifestPath, "utf8");
// The texts read are the ones with the stored identities.
if (
sameIdentity(fileIdentity(packagePath), identities[0]!) &&
sameIdentity(fileIdentity(manifestPath), identities[1]!)
) {
return {
root,
licenses: stored.licenses,
packageText,
manifestText,
selected: stored.selected,
flavor,
};
}
}
} catch {
key = null;
}
}
const packageText = readFileSync(packagePath, "utf8");
const manifestText = readFileSync(manifestPath, "utf8");
const identity = JSON.parse(packageText) as { name?: string; version?: string };
const manifest = parseRuntimePackManifest(JSON.parse(manifestText));
validateRuntimePackIdentity(manifest, identity.name, identity.version, target, compilerVersion);
const selected = selectRuntimePackArtifacts(manifest, features, flavor, process.env, mode);
return { root, manifest, packageText, manifestText, selected, flavor };
if (store !== undefined && key !== null) {
try {
// Stored only when the files selected from still have the identities
// in the key.
if (
sameIdentity(fileIdentity(packagePath), identities[0]!) &&
sameIdentity(fileIdentity(manifestPath), identities[1]!)
) {
const entry: StoredSelection = { schema: 1, licenses: manifest.licenses, selected };
store.write(key, JSON.stringify(entry));
}
} catch {
/* A failed write is a later miss. */
}
}
return {
root,
manifest,
licenses: manifest.licenses,
packageText,
manifestText,
selected,
flavor,
};
}
function stageArtifact(root: string, stage: string, artifact: RuntimePackArtifact): string {
/** The selection's parsed manifest. */
export function runtimePackManifest(selection: NativeRuntimeSelection): RuntimePackManifest {
return (selection.manifest ??= parseRuntimePackManifest(JSON.parse(selection.manifestText)));
}
function readStoredSelection(text: string | null): StoredSelection | null {
if (text === null) return null;
const value: unknown = JSON.parse(text);
if (value === null || typeof value !== "object") return null;
const entry = value as Partial<StoredSelection>;
if (
entry.schema !== 1 ||
!Array.isArray(entry.licenses) ||
entry.selected === undefined ||
entry.selected === null ||
typeof entry.selected !== "object" ||
!Array.isArray(entry.selected.runtime) ||
!Array.isArray(entry.selected.archives) ||
!Array.isArray(entry.selected.systemLibraries)
)
return null;
return entry as StoredSelection;
}
/** An artifact path's identity when its bytes last matched the manifest's
* size and digest. */
interface VerifiedArtifact {
path: string;
sha256: string;
dev: number;
ino: number;
size: number;
mtime: number;
ctime: number;
}
interface StoredVerification {
schema: number;
artifacts: VerifiedArtifact[];
}
function readVerified(
store: RuntimePackStore | undefined,
key: string,
): Map<string, VerifiedArtifact> {
const verified = new Map<string, VerifiedArtifact>();
try {
const text = store?.read(key) ?? null;
if (text === null) return verified;
const stored = JSON.parse(text) as StoredVerification;
if (stored.schema === 1)
for (const artifact of stored.artifacts) verified.set(artifact.path, artifact);
} catch {
/* A malformed entry verifies everything again. */
}
return verified;
}
function stageArtifact(
root: string,
stage: string,
artifact: RuntimePackArtifact,
verified: Map<string, VerifiedArtifact>,
newlyVerified: VerifiedArtifact[],
): string {
const source = join(root, artifact.path);
const destination = join(stage, artifact.path);
let bytes: Buffer;
let before: FileIdentity;
let after: FileIdentity;
try {
before = fileIdentity(source);
bytes = readFileSync(source);
after = fileIdentity(source);
} catch {
throw new RuntimePackError(`runtime pack artifact is missing: ${artifact.path}`, "invalid");
}
// The bytes are those last hashed when the file kept the identity it had
// then for the whole read.
const known = verified.get(artifact.path);
const unchanged = sameIdentity(before, after);
const trusted =
known !== undefined &&
known.sha256 === artifact.sha256 &&
known.size === artifact.size &&
unchanged &&
known.dev === before.dev &&
known.ino === before.ino &&
known.size === before.size &&
known.mtime === before.mtime &&
known.ctime === before.ctime;
if (
bytes.length !== artifact.size ||
createHash("sha256").update(bytes).digest("hex") !== artifact.sha256
(!trusted && createHash("sha256").update(bytes).digest("hex") !== artifact.sha256)
) {
throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid");
}
if (!trusted && unchanged)
newlyVerified.push({
path: artifact.path,
sha256: artifact.sha256,
dev: before.dev,
ino: before.ino,
size: before.size,
mtime: before.mtime,
ctime: before.ctime,
});
mkdirSync(dirname(destination), { recursive: true });
const fd = openSync(
destination,
@@ -123,15 +335,19 @@ export function stageNativeRuntimePack(
export function stageNativeRuntimeSelection(
selection: NativeRuntimeSelection,
stageRoot: string,
store?: RuntimePackStore,
): NativeRuntimePack {
const { root, manifest, selected, packageText, manifestText } = selection;
const { root, licenses, selected, packageText, manifestText } = selection;
const packagePath = join(root, "package.json");
const manifestPath = join(root, "runtime-pack.json");
const runtimeObjects = selected.runtime.map((artifact) =>
stageArtifact(root, stageRoot, artifact),
);
const archives = selected.archives.map((artifact) => stageArtifact(root, stageRoot, artifact));
for (const license of manifest.licenses) {
const key = storeKey(JSON.stringify({ schema: 1, verified: root }));
const verified = readVerified(store, key);
const newlyVerified: VerifiedArtifact[] = [];
const stage = (artifact: RuntimePackArtifact): string =>
stageArtifact(root, stageRoot, artifact, verified, newlyVerified);
const runtimeObjects = selected.runtime.map(stage);
const archives = selected.archives.map(stage);
for (const license of licenses) {
try {
readFileSync(join(root, license.path));
} catch {
@@ -144,5 +360,14 @@ export function stageNativeRuntimeSelection(
) {
throw new RuntimePackError("runtime pack changed while staging verified artifacts", "invalid");
}
if (store !== undefined && newlyVerified.length > 0) {
for (const artifact of newlyVerified) verified.set(artifact.path, artifact);
const entry: StoredVerification = { schema: 1, artifacts: [...verified.values()] };
try {
store.write(key, JSON.stringify(entry));
} catch {
/* A failed write only means hashing again next time. */
}
}
return { runtimeObjects, archives, systemLibraries: selected.systemLibraries };
}
+1
View File
@@ -179,6 +179,7 @@ export class NativeCache {
"sanitizer",
"executable",
"linked",
"runtime-pack",
"binary",
"dsym",
"coverage",
+28 -4
View File
@@ -31,7 +31,9 @@ import {
} from "../backend/native-tools.js";
import {
selectNativeRuntimePack,
runtimePackManifest,
stageNativeRuntimeSelection,
type RuntimePackStore,
} from "../backend/runtime-pack-native.js";
import { RuntimePackError } from "../backend/runtime-pack-core.js";
import {
@@ -160,6 +162,17 @@ export class NativeCompiler {
return result;
}
/** Runtime pack selections and verified artifact identities, kept in the
* cache. */
private runtimePackStore(): RuntimePackStore | undefined {
const cache = this.cache;
if (cache === null) return undefined;
return {
read: (key) => cache.read("runtime-pack", key)?.toString("utf8") ?? null,
write: (key, text) => cache.write("runtime-pack", key, text),
};
}
private emitObject(
input: string,
output: string,
@@ -394,6 +407,8 @@ export class NativeCompiler {
toolchain.compilerVersion,
features,
optimization,
"executable",
this.runtimePackStore(),
)
: null;
const executableCache =
@@ -449,16 +464,24 @@ export class NativeCompiler {
toolchain.compilerVersion,
features,
optimization,
"executable",
this.runtimePackStore(),
)
: null;
if (pack !== null) stageNativeRuntimeSelection(pack, join(stage, "runtime"));
if (pack !== null)
stageNativeRuntimeSelection(pack, join(stage, "runtime"), this.runtimePackStore());
const info =
pack === null
? undefined
: formatNativeLinkInfo(
{ programObject: output, target, ffi },
toolchain.compilerVersion,
pack,
{
root: pack.root,
manifest: runtimePackManifest(pack),
selected: pack.selected,
flavor: pack.flavor,
},
);
renameSync(stagedOutput, output);
return {
@@ -473,7 +496,7 @@ export class NativeCompiler {
const pack = executablePack!;
const runtime = options.sanitize
? buildSanitizedRuntime(toolchain, pack, join(stage, "runtime"), "executable", this.cache)
: stageNativeRuntimeSelection(pack, join(stage, "runtime"));
: stageNativeRuntimeSelection(pack, join(stage, "runtime"), this.runtimePackStore());
timing("runtime-stage");
const plan = executableLinkInputs({
target,
@@ -617,6 +640,7 @@ export class NativeCompiler {
features,
profile.optimization,
profile.instancePerThread ? "library-thread" : "library",
this.runtimePackStore(),
);
const runtime = options.sanitize
? buildSanitizedRuntime(
@@ -626,7 +650,7 @@ export class NativeCompiler {
profile.instancePerThread ? "library-thread" : "library",
this.cache,
)
: stageNativeRuntimeSelection(pack, join(stage, "runtime"));
: stageNativeRuntimeSelection(pack, join(stage, "runtime"), this.runtimePackStore());
timing("runtime-stage");
const stagedArchive = join(stage, wasm ? "output.wasm" : "output.a");
const localizeSymbols = libraryLocalizeSymbols(profile);
@@ -52,7 +52,7 @@ function fixture(platform: string, driver: string) {
} as unknown as NativeToolchain;
const pack = {
root: runtime,
manifest: { licenses: [] },
licenses: [],
packageText: "{}",
manifestText: "{}",
selected: { runtime: [], archives: [] },
@@ -331,8 +331,8 @@ export function openNativeExecutableCache(
join(toolchain.helperPackageRoot, "package.json"),
join(toolchain.runtimePackRoot, "package.json"),
join(toolchain.runtimePackRoot, "runtime-pack.json"),
...[...pack.selected.runtime, ...pack.selected.archives, ...pack.manifest.licenses].map(
(artifact) => join(pack.root, artifact.path),
...[...pack.selected.runtime, ...pack.selected.archives, ...pack.licenses].map((artifact) =>
join(pack.root, artifact.path),
),
];
if (platform === "linux") paths.push(join(pack.root, "runtime-pack.json"));
@@ -64,6 +64,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")(
{
root: directory,
manifest,
licenses: manifest.licenses,
packageText: "",
manifestText: "",
flavor: "dev",
+8 -3
View File
@@ -4,7 +4,11 @@ import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { basename, join } from "node:path";
import { NativeCodegenError } from "../backend/native-codegen-core.js";
import { requireNativeArtifact, runNativeTool } from "../backend/native-tools.js";
import type { NativeRuntimePack, NativeRuntimeSelection } from "../backend/runtime-pack-native.js";
import {
runtimePackManifest,
type NativeRuntimePack,
type NativeRuntimeSelection,
} from "../backend/runtime-pack-native.js";
import { runtimePackFlavorKey, type RuntimePackMode } from "../backend/runtime-pack-core.js";
import { LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js";
import { contentDigest, type NativeCache } from "./cache.js";
@@ -119,8 +123,9 @@ export function buildSanitizedRuntime(
}
return output;
};
const key = runtimePackFlavorKey(selection.manifest, selection.flavor, mode);
const flavor = selection.manifest.flavors[key];
const manifest = runtimePackManifest(selection);
const key = runtimePackFlavorKey(manifest, selection.flavor, mode);
const flavor = manifest.flavors[key];
if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`);
const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path));
const common = [