Keep the runtime pack's verified artifacts in a record beside the pack

Each build stages the runtime artifacts it links from the runtime pack and
hashes each one with SHA-256 unless the build cache records that it was
verified with the file identity it still has. A build with an empty cache
therefore hashed every artifact again: on Linux x64, 40 objects and 4.3 MB
for a Benchmarks Game program, about 25 ms of a 300-1100 ms build.

Staging now also reads and writes runtime-pack.verified.json beside the
pack. It lists each verified artifact with its digest and file identity
(device, inode, size, mtime and ctime), as the cache's record does:

- An artifact is trusted only when one of the two records lists it with
  the manifest's digest and size and the identity its file has during the
  read. Any change to the file means hashing it again.
- A missing, unreadable or malformed record, or a malformed entry, is
  treated as absent.
- The record is written to a new file and renamed over the old one, so a
  reader sees a whole record. A write merges the record as it is then.
- When the pack directory cannot be written, the cache's record is the only
  one, as before.

The runtime pack packages do not publish the file. The native CLI build
removes the record its probe builds leave in the bundled pack, and
verify-native-cli.mjs rejects a distribution that has one.

.gitignore

@@ -35,6 +35,7 @@ output/
 /packages/runtime-*/artifacts/
 /packages/runtime-*/runtime-pack.json
 /packages/runtime-*/.runtime-pack-*
+/packages/runtime-*/runtime-pack.verified.json*
 # libnode packs: built by build:native, never committed
 /packages/libnode-*/libnode.so.*
 /packages/libnode-*/libnode.*.dylib

packages/compiler/src/backend/runtime-pack-native.test.ts

@@ -1,8 +1,12 @@
 import { createHash } from "node:crypto";
 import {
+  chmodSync,
+  existsSync,
   mkdirSync,
+  readdirSync,
   readFileSync,
   renameSync,
+  rmSync,
   statSync,
   symlinkSync,
   unlinkSync,
@@ -148,10 +152,12 @@ async function packFixture() {
       store,
     );
   let stages = 0;
-  const stage = (selection: NativeRuntimeSelection) => {
-    const directory = join(root, `stage-${stages++}`);
+  const stage = (selection: NativeRuntimeSelection, with_: RuntimePackStore = store) => {
+    const directory = join(tmpdir(), `scriptc-runtime-pack-stage-${process.pid}-${stages++}`);
+    rmSync(directory, { recursive: true, force: true });
     mkdirSync(directory);
-    return stageNativeRuntimeSelection(selection, directory, store);
+    dirs.push(directory);
+    return stageNativeRuntimeSelection(selection, directory, with_);
   };
   return { root, object, manifestText, entries, select, stage };
 }
@@ -241,3 +247,180 @@ test.skipIf(process.platform === "win32")(
     expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
   },
 );
+
+/** A store that keeps nothing: a build with an empty cache. */
+function emptyStore(): RuntimePackStore {
+  const entries = new Map<string, string>();
+  return {
+    read: (key) => entries.get(key) ?? null,
+    write: (key, text) => void entries.set(key, text),
+  };
+}
+
+const packRecord = (root: string) => join(root, "runtime-pack.verified.json");
+const hashes = () => vi.mocked(createHash).mock.calls.length;
+
+test("a build with an empty cache trusts the record beside the pack", async () => {
+  const f = await packFixture();
+  const selection = f.select();
+  let start = hashes();
+  f.stage(selection);
+  const firstStage = hashes() - start;
+  const record = JSON.parse(readFileSync(packRecord(f.root), "utf8"));
+  expect(record.schema).toBe(1);
+  expect(record.artifacts.map((artifact: { path: string }) => artifact.path)).toEqual([
+    "artifacts/base.o",
+  ]);
+  start = hashes();
+  const staged = f.stage(selection, emptyStore());
+  // Only the store key is hashed.
+  expect(hashes() - start).toBe(firstStage - 1);
+  expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
+});
+
+test.each(["replaced", "rewritten in place", "resized"])(
+  "the record beside the pack does not trust an artifact that was %s",
+  async (change) => {
+    const f = await packFixture();
+    const selection = f.select();
+    f.stage(selection);
+    const before = statSync(f.object);
+    const changed = change === "replaced" ? f.object + ".new" : f.object;
+    writeFileSync(changed, "x".repeat(before.size + (change === "resized" ? 1 : 0)));
+    utimesSync(changed, before.atime, before.mtime);
+    if (changed !== f.object) renameSync(changed, f.object);
+    expect(() => f.stage(selection, emptyStore())).toThrow(
+      "runtime pack artifact hash mismatch: artifacts/base.o",
+    );
+  },
+);
+
+test("an artifact whose mtime changed is hashed again and recorded again", async () => {
+  const f = await packFixture();
+  const selection = f.select();
+  f.stage(selection);
+  const before = statSync(f.object);
+  utimesSync(f.object, before.atime, new Date(before.mtimeMs + 5000));
+  const start = hashes();
+  f.stage(selection, emptyStore());
+  // The store key and the artifact.
+  expect(hashes() - start).toBe(2);
+  const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts;
+  expect(entry.mtime).toBe(statSync(f.object).mtimeMs);
+  const again = hashes();
+  f.stage(selection, emptyStore());
+  expect(hashes() - again).toBe(1);
+});
+
+test.each([
+  ["truncated", () => "{"],
+  ["null", () => "null"],
+  ["another schema", (text: string) => text.replace('"schema":1', '"schema":2')],
+  [
+    "an entry with a string ctime",
+    (text: string) => text.replace(/"ctime":([\d.]+)/, '"ctime":"$1"'),
+  ],
+  ["an entry without an inode", (text: string) => text.replace(/"ino":[\d.]+,/, "")],
+  [
+    "another digest",
+    (text: string) => text.replace(/"sha256":"[0-9a-f]+"/, `"sha256":"${"0".repeat(64)}"`),
+  ],
+])("a record beside the pack that is %s is not trusted", async (_name, rewrite) => {
+  const f = await packFixture();
+  const selection = f.select();
+  f.stage(selection);
+  const text = readFileSync(packRecord(f.root), "utf8");
+  const rewritten = rewrite(text);
+  expect(rewritten).not.toBe(text);
+  writeFileSync(packRecord(f.root), rewritten);
+  const start = hashes();
+  const staged = f.stage(selection, emptyStore());
+  expect(hashes() - start).toBe(2);
+  expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
+  // The record is written again, whole.
+  const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts;
+  expect(entry.ino).toBe(statSync(f.object).ino);
+});
+
+test("an unreadable record beside the pack is not trusted, and staging still succeeds", async () => {
+  const f = await packFixture();
+  const selection = f.select();
+  // A directory where the record would be: it can be neither read nor replaced.
+  mkdirSync(packRecord(f.root));
+  const start = hashes();
+  f.stage(selection, emptyStore());
+  f.stage(selection, emptyStore());
+  expect(hashes() - start).toBe(4);
+  expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
+});
+
+test.skipIf(process.platform === "win32")(
+  "a pack directory that cannot be written keeps the record in the cache only",
+  async (context) => {
+    const f = await packFixture();
+    const selection = f.select();
+    chmodSync(f.root, 0o555);
+    try {
+      // A process that may write any directory cannot run this case.
+      try {
+        writeFileSync(join(f.root, "probe"), "");
+        rmSync(join(f.root, "probe"));
+        context.skip();
+      } catch {
+        /* The directory is read-only for this process. */
+      }
+      const store = emptyStore();
+      let start = hashes();
+      f.stage(selection, store);
+      expect(hashes() - start).toBe(2);
+      expect(existsSync(packRecord(f.root))).toBe(false);
+      expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
+      // The cache's record still answers the next build with that cache.
+      start = hashes();
+      f.stage(selection, store);
+      expect(hashes() - start).toBe(1);
+    } finally {
+      chmodSync(f.root, 0o755);
+    }
+  },
+);
+
+test("writing the record beside the pack keeps the entries another build wrote", async () => {
+  const f = await packFixture();
+  const selection = f.select();
+  const other = {
+    path: "artifacts/other.o",
+    sha256: "1".repeat(64),
+    dev: 1,
+    ino: 2,
+    size: 3,
+    mtime: 4,
+    ctime: 5,
+  };
+  writeFileSync(packRecord(f.root), JSON.stringify({ schema: 1, artifacts: [other] }));
+  f.stage(selection, emptyStore());
+  const record = JSON.parse(readFileSync(packRecord(f.root), "utf8"));
+  expect(record.artifacts.map((artifact: { path: string }) => artifact.path).sort()).toEqual([
+    "artifacts/base.o",
+    "artifacts/other.o",
+  ]);
+  expect(
+    record.artifacts.find((artifact: { path: string }) => artifact.path === other.path),
+  ).toEqual(other);
+  expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
+});
+
+test("a cache that already trusts the artifacts writes the record beside the pack", async () => {
+  const f = await packFixture();
+  const selection = f.select();
+  f.stage(selection);
+  rmSync(packRecord(f.root));
+  const start = hashes();
+  f.stage(selection);
+  // The cache's record answers; only the store key is hashed.
+  expect(hashes() - start).toBe(1);
+  expect(existsSync(packRecord(f.root))).toBe(true);
+  const empty = hashes();
+  f.stage(selection, emptyStore());
+  expect(hashes() - empty).toBe(1);
+});

packages/compiler/src/backend/runtime-pack-native.ts

@@ -2,9 +2,11 @@
  * selected artifact is verified and copied from the same read into private
  * storage. With a store, a pack's selections and the identities of the
  * artifacts it verified are kept between builds: an artifact whose file
- * identity is unchanged since it was hashed is not hashed again. */
+ * identity is unchanged since it was hashed is not hashed again. The
+ * identities are also kept in a record beside the pack when its directory
+ * can be written, which builds with an empty store read too. */
 import type { NativeOptimization } from "./optimization.js";
-import { createHash } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import {
   closeSync,
   constants,
@@ -13,6 +15,8 @@ import {
   mkdirSync,
   openSync,
   readFileSync,
+  renameSync,
+  rmSync,
   statSync,
   writeSync,
   type Stats,
@@ -239,21 +243,53 @@ function readVerified(
   try {
     const text = store?.read(key) ?? null;
     if (text === null) return verified;
-    const stored = JSON.parse(text) as StoredVerification;
-    if (stored.schema === 1)
-      for (const artifact of stored.artifacts) verified.set(artifact.path, artifact);
+    for (const artifact of storedArtifacts(text)) verified.set(artifact.path, artifact);
   } catch {
     /* A malformed entry verifies everything again. */
   }
   return verified;
 }
 
+/** The well-formed entries of a stored verification; none when the text is
+ * not a stored verification. */
+function storedArtifacts(text: string): VerifiedArtifact[] {
+  const stored = JSON.parse(text) as Partial<StoredVerification> | null;
+  if (stored === null || stored.schema !== 1 || !Array.isArray(stored.artifacts)) return [];
+  const number = (value: unknown): boolean => typeof value === "number" && Number.isFinite(value);
+  return stored.artifacts.filter(
+    (artifact): artifact is VerifiedArtifact =>
+      artifact !== null &&
+      typeof artifact === "object" &&
+      typeof artifact.path === "string" &&
+      typeof artifact.sha256 === "string" &&
+      number(artifact.dev) &&
+      number(artifact.ino) &&
+      number(artifact.size) &&
+      number(artifact.mtime) &&
+      number(artifact.ctime),
+  );
+}
+
+/** `known` records `artifact`'s digest and size for a file with `identity`. */
+function verifiedAs(
+  known: VerifiedArtifact | undefined,
+  artifact: { sha256: string; size: number },
+  identity: FileIdentity,
+): boolean {
+  return (
+    known !== undefined &&
+    known.sha256 === artifact.sha256 &&
+    known.size === artifact.size &&
+    sameIdentity(known, identity)
+  );
+}
+
 function stageArtifact(
   root: string,
   stage: string,
   artifact: RuntimePackArtifact,
-  verified: Map<string, VerifiedArtifact>,
-  newlyVerified: VerifiedArtifact[],
+  records: readonly Map<string, VerifiedArtifact>[],
+  verified: VerifiedArtifact[],
 ): string {
   const source = join(root, artifact.path);
   const destination = join(stage, artifact.path);
@@ -276,26 +312,17 @@ function stageArtifact(
   }
   // The bytes are those last hashed when the file kept the identity it had
   // then for the whole read.
-  const known = verified.get(artifact.path);
   const unchanged = sameIdentity(before, after);
   const trusted =
-    known !== undefined &&
-    known.sha256 === artifact.sha256 &&
-    known.size === artifact.size &&
-    unchanged &&
-    known.dev === before.dev &&
-    known.ino === before.ino &&
-    known.size === before.size &&
-    known.mtime === before.mtime &&
-    known.ctime === before.ctime;
+    unchanged && records.some((record) => verifiedAs(record.get(artifact.path), artifact, before));
   if (
     bytes.length !== artifact.size ||
     (!trusted && createHash("sha256").update(bytes).digest("hex") !== artifact.sha256)
   ) {
     throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid");
   }
-  if (!trusted && unchanged)
-    newlyVerified.push({
+  if (unchanged)
+    verified.push({
       path: artifact.path,
       sha256: artifact.sha256,
       dev: before.dev,
@@ -353,10 +380,12 @@ export function stageNativeRuntimeSelection(
   const packagePath = join(root, "package.json");
   const manifestPath = join(root, "runtime-pack.json");
   const key = storeKey(JSON.stringify({ schema: 1, verified: root }));
-  const verified = readVerified(store, key);
-  const newlyVerified: VerifiedArtifact[] = [];
+  const cacheRecord = readVerified(store, key);
+  const packRecordPath = join(root, PACK_VERIFIED_FILE);
+  const packRecord = readPackVerified(packRecordPath);
+  const verified: VerifiedArtifact[] = [];
   const stage = (artifact: RuntimePackArtifact): string =>
-    stageArtifact(root, stageRoot, artifact, verified, newlyVerified);
+    stageArtifact(root, stageRoot, artifact, [cacheRecord, packRecord], verified);
   const runtimeObjects = selected.runtime.map(stage);
   const archives = selected.archives.map(stage);
   for (const license of licenses) {
@@ -372,14 +401,78 @@ export function stageNativeRuntimeSelection(
   ) {
     throw new RuntimePackError("runtime pack changed while staging verified artifacts", "invalid");
   }
-  if (store !== undefined && newlyVerified.length > 0) {
-    for (const artifact of newlyVerified) verified.set(artifact.path, artifact);
-    const entry: StoredVerification = { schema: 1, artifacts: [...verified.values()] };
+  // Each record learns the artifacts verified here that it does not list
+  // with their current identity.
+  const missing = (record: Map<string, VerifiedArtifact>): VerifiedArtifact[] =>
+    verified.filter((artifact) => !verifiedAs(record.get(artifact.path), artifact, artifact));
+  const cacheMissing = missing(cacheRecord);
+  if (store !== undefined && cacheMissing.length > 0) {
+    for (const artifact of cacheMissing) cacheRecord.set(artifact.path, artifact);
+    const entry: StoredVerification = { schema: 1, artifacts: [...cacheRecord.values()] };
     try {
       store.write(key, JSON.stringify(entry));
     } catch {
       /* A failed write only means hashing again next time. */
     }
   }
+  const packMissing = missing(packRecord);
+  if (packMissing.length > 0) writePackVerified(packRecordPath, packMissing);
   return { runtimeObjects, archives, systemLibraries: selected.systemLibraries };
 }
+
+/** The verification record kept beside the runtime pack, so a build with an
+ * empty cache does not hash the pack's artifacts again. Like the cache's
+ * record, it trusts an artifact only while its file keeps the identity
+ * (device, inode, size, mtime, ctime) the record lists. */
+export const PACK_VERIFIED_FILE = "runtime-pack.verified.json";
+
+/** The pack record's well-formed entries; none when it is missing,
+ * unreadable or malformed. */
+function readPackVerified(path: string): Map<string, VerifiedArtifact> {
+  const verified = new Map<string, VerifiedArtifact>();
+  try {
+    for (const artifact of storedArtifacts(readFileSync(path, "utf8")))
+      verified.set(artifact.path, artifact);
+  } catch {
+    /* The artifacts are hashed again. */
+  }
+  return verified;
+}
+
+/** Adds `artifacts` to the pack record: the record as it is now, with these
+ * entries replacing the ones for the same paths, written to a new file and
+ * renamed over the record, so a reader sees a whole record. Of two builds
+ * writing at once, the last rename wins; the entries the other added are
+ * hashed again by a later build. When the pack directory cannot be written,
+ * the cache's record is the only one. */
+function writePackVerified(path: string, artifacts: readonly VerifiedArtifact[]): void {
+  const temporary = `${path}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
+  try {
+    const record = readPackVerified(path);
+    for (const artifact of artifacts) record.set(artifact.path, artifact);
+    const entry: StoredVerification = { schema: 1, artifacts: [...record.values()] };
+    const bytes = Buffer.from(JSON.stringify(entry));
+    const fd = openSync(
+      temporary,
+      constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL,
+      0o644,
+    );
+    try {
+      let offset = 0;
+      while (offset < bytes.length) {
+        const written = writeSync(fd, bytes, offset, bytes.length - offset);
+        if (written === 0) throw new Error("the pack record was not written");
+        offset += written;
+      }
+    } finally {
+      closeSync(fd);
+    }
+    renameSync(temporary, path);
+  } catch {
+    try {
+      rmSync(temporary, { force: true });
+    } catch {
+      /* Cleanup is best effort. */
+    }
+  }
+}

scripts/build-native-cli.mts

@@ -12,6 +12,7 @@ import {
   nativeCodegenTarget,
 } from "../packages/compiler/src/backend/targets.js";
 import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js";
+import { PACK_VERIFIED_FILE } from "../packages/compiler/src/backend/runtime-pack-native.js";
 import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js";
 import { stageNativeCliAssets } from "./native-cli-assets.mjs";
 import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js";
@@ -237,6 +238,12 @@ writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n");
 // for programs without a tsconfig, recorded by the built compiler itself. A
 // compiler this host cannot run ships without them.
 const prebuiltKeys = recordPrebuiltLibDiagnostics(executable);
+// The probes' builds record the bundled runtime pack's verified artifacts
+// beside it, by this host's file identities; the distribution ships without
+// that record.
+rmSync(join(lib, target.runtimePackPackage.replace("@scriptc/", ""), PACK_VERIFIED_FILE), {
+  force: true,
+});
 if (prebuiltKeys === null) {
   console.warn(
     "warning: the built compiler did not run here; it ships without prebuilt lib diagnostics",

scripts/verify-native-cli.mjs

@@ -99,6 +99,15 @@ for (const directory of [distribution, join(distribution, "lib/runtime-sources/v
   if (readdirSync(directory).some((name) => ["node_modules", ".cache", "seed"].includes(name)))
     throw new Error(`development files present in ${directory}`);
 }
+// A build's record of the runtime pack's verified artifacts (runtime-pack-native.ts)
+// names that host's file identities.
+for (const name of readdirSync(join(distribution, "lib"))) {
+  if (
+    name.startsWith("runtime-") &&
+    existsSync(join(distribution, "lib", name, "runtime-pack.verified.json"))
+  )
+    throw new Error(`runtime pack verification record present in lib/${name}`);
+}
 if (process.argv.includes("--run")) {
   if (target.startsWith("linux-")) {
     for (const executable of [