diff --git a/.gitignore b/.gitignore index 04c889158..36b158611 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ output/ /packages/runtime-*/artifacts/ /packages/runtime-*/runtime-pack.json /packages/runtime-*/.runtime-pack-* +/packages/runtime-*/runtime-pack.verified.json* # libnode packs: built by build:native, never committed /packages/libnode-*/libnode.so.* /packages/libnode-*/libnode.*.dylib diff --git a/packages/compiler/src/backend/runtime-pack-native.test.ts b/packages/compiler/src/backend/runtime-pack-native.test.ts index 7a8a2e6c4..18975e1f9 100644 --- a/packages/compiler/src/backend/runtime-pack-native.test.ts +++ b/packages/compiler/src/backend/runtime-pack-native.test.ts @@ -1,8 +1,12 @@ import { createHash } from "node:crypto"; import { + chmodSync, + existsSync, mkdirSync, + readdirSync, readFileSync, renameSync, + rmSync, statSync, symlinkSync, unlinkSync, @@ -148,10 +152,12 @@ async function packFixture() { store, ); let stages = 0; - const stage = (selection: NativeRuntimeSelection) => { - const directory = join(root, `stage-${stages++}`); + const stage = (selection: NativeRuntimeSelection, with_: RuntimePackStore = store) => { + const directory = join(tmpdir(), `scriptc-runtime-pack-stage-${process.pid}-${stages++}`); + rmSync(directory, { recursive: true, force: true }); mkdirSync(directory); - return stageNativeRuntimeSelection(selection, directory, store); + dirs.push(directory); + return stageNativeRuntimeSelection(selection, directory, with_); }; return { root, object, manifestText, entries, select, stage }; } @@ -241,3 +247,180 @@ test.skipIf(process.platform === "win32")( expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object"); }, ); + +/** A store that keeps nothing: a build with an empty cache. */ +function emptyStore(): RuntimePackStore { + const entries = new Map(); + return { + read: (key) => entries.get(key) ?? null, + write: (key, text) => void entries.set(key, text), + }; +} + +const packRecord = (root: string) => join(root, "runtime-pack.verified.json"); +const hashes = () => vi.mocked(createHash).mock.calls.length; + +test("a build with an empty cache trusts the record beside the pack", async () => { + const f = await packFixture(); + const selection = f.select(); + let start = hashes(); + f.stage(selection); + const firstStage = hashes() - start; + const record = JSON.parse(readFileSync(packRecord(f.root), "utf8")); + expect(record.schema).toBe(1); + expect(record.artifacts.map((artifact: { path: string }) => artifact.path)).toEqual([ + "artifacts/base.o", + ]); + start = hashes(); + const staged = f.stage(selection, emptyStore()); + // Only the store key is hashed. + expect(hashes() - start).toBe(firstStage - 1); + expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object"); +}); + +test.each(["replaced", "rewritten in place", "resized"])( + "the record beside the pack does not trust an artifact that was %s", + async (change) => { + const f = await packFixture(); + const selection = f.select(); + f.stage(selection); + const before = statSync(f.object); + const changed = change === "replaced" ? f.object + ".new" : f.object; + writeFileSync(changed, "x".repeat(before.size + (change === "resized" ? 1 : 0))); + utimesSync(changed, before.atime, before.mtime); + if (changed !== f.object) renameSync(changed, f.object); + expect(() => f.stage(selection, emptyStore())).toThrow( + "runtime pack artifact hash mismatch: artifacts/base.o", + ); + }, +); + +test("an artifact whose mtime changed is hashed again and recorded again", async () => { + const f = await packFixture(); + const selection = f.select(); + f.stage(selection); + const before = statSync(f.object); + utimesSync(f.object, before.atime, new Date(before.mtimeMs + 5000)); + const start = hashes(); + f.stage(selection, emptyStore()); + // The store key and the artifact. + expect(hashes() - start).toBe(2); + const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts; + expect(entry.mtime).toBe(statSync(f.object).mtimeMs); + const again = hashes(); + f.stage(selection, emptyStore()); + expect(hashes() - again).toBe(1); +}); + +test.each([ + ["truncated", () => "{"], + ["null", () => "null"], + ["another schema", (text: string) => text.replace('"schema":1', '"schema":2')], + [ + "an entry with a string ctime", + (text: string) => text.replace(/"ctime":([\d.]+)/, '"ctime":"$1"'), + ], + ["an entry without an inode", (text: string) => text.replace(/"ino":[\d.]+,/, "")], + [ + "another digest", + (text: string) => text.replace(/"sha256":"[0-9a-f]+"/, `"sha256":"${"0".repeat(64)}"`), + ], +])("a record beside the pack that is %s is not trusted", async (_name, rewrite) => { + const f = await packFixture(); + const selection = f.select(); + f.stage(selection); + const text = readFileSync(packRecord(f.root), "utf8"); + const rewritten = rewrite(text); + expect(rewritten).not.toBe(text); + writeFileSync(packRecord(f.root), rewritten); + const start = hashes(); + const staged = f.stage(selection, emptyStore()); + expect(hashes() - start).toBe(2); + expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object"); + // The record is written again, whole. + const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts; + expect(entry.ino).toBe(statSync(f.object).ino); +}); + +test("an unreadable record beside the pack is not trusted, and staging still succeeds", async () => { + const f = await packFixture(); + const selection = f.select(); + // A directory where the record would be: it can be neither read nor replaced. + mkdirSync(packRecord(f.root)); + const start = hashes(); + f.stage(selection, emptyStore()); + f.stage(selection, emptyStore()); + expect(hashes() - start).toBe(4); + expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]); +}); + +test.skipIf(process.platform === "win32")( + "a pack directory that cannot be written keeps the record in the cache only", + async (context) => { + const f = await packFixture(); + const selection = f.select(); + chmodSync(f.root, 0o555); + try { + // A process that may write any directory cannot run this case. + try { + writeFileSync(join(f.root, "probe"), ""); + rmSync(join(f.root, "probe")); + context.skip(); + } catch { + /* The directory is read-only for this process. */ + } + const store = emptyStore(); + let start = hashes(); + f.stage(selection, store); + expect(hashes() - start).toBe(2); + expect(existsSync(packRecord(f.root))).toBe(false); + expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]); + // The cache's record still answers the next build with that cache. + start = hashes(); + f.stage(selection, store); + expect(hashes() - start).toBe(1); + } finally { + chmodSync(f.root, 0o755); + } + }, +); + +test("writing the record beside the pack keeps the entries another build wrote", async () => { + const f = await packFixture(); + const selection = f.select(); + const other = { + path: "artifacts/other.o", + sha256: "1".repeat(64), + dev: 1, + ino: 2, + size: 3, + mtime: 4, + ctime: 5, + }; + writeFileSync(packRecord(f.root), JSON.stringify({ schema: 1, artifacts: [other] })); + f.stage(selection, emptyStore()); + const record = JSON.parse(readFileSync(packRecord(f.root), "utf8")); + expect(record.artifacts.map((artifact: { path: string }) => artifact.path).sort()).toEqual([ + "artifacts/base.o", + "artifacts/other.o", + ]); + expect( + record.artifacts.find((artifact: { path: string }) => artifact.path === other.path), + ).toEqual(other); + expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]); +}); + +test("a cache that already trusts the artifacts writes the record beside the pack", async () => { + const f = await packFixture(); + const selection = f.select(); + f.stage(selection); + rmSync(packRecord(f.root)); + const start = hashes(); + f.stage(selection); + // The cache's record answers; only the store key is hashed. + expect(hashes() - start).toBe(1); + expect(existsSync(packRecord(f.root))).toBe(true); + const empty = hashes(); + f.stage(selection, emptyStore()); + expect(hashes() - empty).toBe(1); +}); diff --git a/packages/compiler/src/backend/runtime-pack-native.ts b/packages/compiler/src/backend/runtime-pack-native.ts index 0fff34455..bd827d963 100644 --- a/packages/compiler/src/backend/runtime-pack-native.ts +++ b/packages/compiler/src/backend/runtime-pack-native.ts @@ -2,9 +2,11 @@ * selected artifact is verified and copied from the same read into private * storage. With a store, a pack's selections and the identities of the * artifacts it verified are kept between builds: an artifact whose file - * identity is unchanged since it was hashed is not hashed again. */ + * identity is unchanged since it was hashed is not hashed again. The + * identities are also kept in a record beside the pack when its directory + * can be written, which builds with an empty store read too. */ import type { NativeOptimization } from "./optimization.js"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { closeSync, constants, @@ -13,6 +15,8 @@ import { mkdirSync, openSync, readFileSync, + renameSync, + rmSync, statSync, writeSync, type Stats, @@ -239,21 +243,53 @@ function readVerified( try { const text = store?.read(key) ?? null; if (text === null) return verified; - const stored = JSON.parse(text) as StoredVerification; - if (stored.schema === 1) - for (const artifact of stored.artifacts) verified.set(artifact.path, artifact); + for (const artifact of storedArtifacts(text)) verified.set(artifact.path, artifact); } catch { /* A malformed entry verifies everything again. */ } return verified; } +/** The well-formed entries of a stored verification; none when the text is + * not a stored verification. */ +function storedArtifacts(text: string): VerifiedArtifact[] { + const stored = JSON.parse(text) as Partial | null; + if (stored === null || stored.schema !== 1 || !Array.isArray(stored.artifacts)) return []; + const number = (value: unknown): boolean => typeof value === "number" && Number.isFinite(value); + return stored.artifacts.filter( + (artifact): artifact is VerifiedArtifact => + artifact !== null && + typeof artifact === "object" && + typeof artifact.path === "string" && + typeof artifact.sha256 === "string" && + number(artifact.dev) && + number(artifact.ino) && + number(artifact.size) && + number(artifact.mtime) && + number(artifact.ctime), + ); +} + +/** `known` records `artifact`'s digest and size for a file with `identity`. */ +function verifiedAs( + known: VerifiedArtifact | undefined, + artifact: { sha256: string; size: number }, + identity: FileIdentity, +): boolean { + return ( + known !== undefined && + known.sha256 === artifact.sha256 && + known.size === artifact.size && + sameIdentity(known, identity) + ); +} + function stageArtifact( root: string, stage: string, artifact: RuntimePackArtifact, - verified: Map, - newlyVerified: VerifiedArtifact[], + records: readonly Map[], + verified: VerifiedArtifact[], ): string { const source = join(root, artifact.path); const destination = join(stage, artifact.path); @@ -276,26 +312,17 @@ function stageArtifact( } // The bytes are those last hashed when the file kept the identity it had // then for the whole read. - const known = verified.get(artifact.path); const unchanged = sameIdentity(before, after); const trusted = - known !== undefined && - known.sha256 === artifact.sha256 && - known.size === artifact.size && - unchanged && - known.dev === before.dev && - known.ino === before.ino && - known.size === before.size && - known.mtime === before.mtime && - known.ctime === before.ctime; + unchanged && records.some((record) => verifiedAs(record.get(artifact.path), artifact, before)); if ( bytes.length !== artifact.size || (!trusted && createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) ) { throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid"); } - if (!trusted && unchanged) - newlyVerified.push({ + if (unchanged) + verified.push({ path: artifact.path, sha256: artifact.sha256, dev: before.dev, @@ -353,10 +380,12 @@ export function stageNativeRuntimeSelection( const packagePath = join(root, "package.json"); const manifestPath = join(root, "runtime-pack.json"); const key = storeKey(JSON.stringify({ schema: 1, verified: root })); - const verified = readVerified(store, key); - const newlyVerified: VerifiedArtifact[] = []; + const cacheRecord = readVerified(store, key); + const packRecordPath = join(root, PACK_VERIFIED_FILE); + const packRecord = readPackVerified(packRecordPath); + const verified: VerifiedArtifact[] = []; const stage = (artifact: RuntimePackArtifact): string => - stageArtifact(root, stageRoot, artifact, verified, newlyVerified); + stageArtifact(root, stageRoot, artifact, [cacheRecord, packRecord], verified); const runtimeObjects = selected.runtime.map(stage); const archives = selected.archives.map(stage); for (const license of licenses) { @@ -372,14 +401,78 @@ export function stageNativeRuntimeSelection( ) { throw new RuntimePackError("runtime pack changed while staging verified artifacts", "invalid"); } - if (store !== undefined && newlyVerified.length > 0) { - for (const artifact of newlyVerified) verified.set(artifact.path, artifact); - const entry: StoredVerification = { schema: 1, artifacts: [...verified.values()] }; + // Each record learns the artifacts verified here that it does not list + // with their current identity. + const missing = (record: Map): VerifiedArtifact[] => + verified.filter((artifact) => !verifiedAs(record.get(artifact.path), artifact, artifact)); + const cacheMissing = missing(cacheRecord); + if (store !== undefined && cacheMissing.length > 0) { + for (const artifact of cacheMissing) cacheRecord.set(artifact.path, artifact); + const entry: StoredVerification = { schema: 1, artifacts: [...cacheRecord.values()] }; try { store.write(key, JSON.stringify(entry)); } catch { /* A failed write only means hashing again next time. */ } } + const packMissing = missing(packRecord); + if (packMissing.length > 0) writePackVerified(packRecordPath, packMissing); return { runtimeObjects, archives, systemLibraries: selected.systemLibraries }; } + +/** The verification record kept beside the runtime pack, so a build with an + * empty cache does not hash the pack's artifacts again. Like the cache's + * record, it trusts an artifact only while its file keeps the identity + * (device, inode, size, mtime, ctime) the record lists. */ +export const PACK_VERIFIED_FILE = "runtime-pack.verified.json"; + +/** The pack record's well-formed entries; none when it is missing, + * unreadable or malformed. */ +function readPackVerified(path: string): Map { + const verified = new Map(); + try { + for (const artifact of storedArtifacts(readFileSync(path, "utf8"))) + verified.set(artifact.path, artifact); + } catch { + /* The artifacts are hashed again. */ + } + return verified; +} + +/** Adds `artifacts` to the pack record: the record as it is now, with these + * entries replacing the ones for the same paths, written to a new file and + * renamed over the record, so a reader sees a whole record. Of two builds + * writing at once, the last rename wins; the entries the other added are + * hashed again by a later build. When the pack directory cannot be written, + * the cache's record is the only one. */ +function writePackVerified(path: string, artifacts: readonly VerifiedArtifact[]): void { + const temporary = `${path}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; + try { + const record = readPackVerified(path); + for (const artifact of artifacts) record.set(artifact.path, artifact); + const entry: StoredVerification = { schema: 1, artifacts: [...record.values()] }; + const bytes = Buffer.from(JSON.stringify(entry)); + const fd = openSync( + temporary, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, + 0o644, + ); + try { + let offset = 0; + while (offset < bytes.length) { + const written = writeSync(fd, bytes, offset, bytes.length - offset); + if (written === 0) throw new Error("the pack record was not written"); + offset += written; + } + } finally { + closeSync(fd); + } + renameSync(temporary, path); + } catch { + try { + rmSync(temporary, { force: true }); + } catch { + /* Cleanup is best effort. */ + } + } +} diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 6c17b829b..88f86f4fd 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -12,6 +12,7 @@ import { nativeCodegenTarget, } from "../packages/compiler/src/backend/targets.js"; import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { PACK_VERIFIED_FILE } from "../packages/compiler/src/backend/runtime-pack-native.js"; import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; @@ -237,6 +238,12 @@ writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); // for programs without a tsconfig, recorded by the built compiler itself. A // compiler this host cannot run ships without them. const prebuiltKeys = recordPrebuiltLibDiagnostics(executable); +// The probes' builds record the bundled runtime pack's verified artifacts +// beside it, by this host's file identities; the distribution ships without +// that record. +rmSync(join(lib, target.runtimePackPackage.replace("@scriptc/", ""), PACK_VERIFIED_FILE), { + force: true, +}); if (prebuiltKeys === null) { console.warn( "warning: the built compiler did not run here; it ships without prebuilt lib diagnostics", diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index fc94927be..4fa312121 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -99,6 +99,15 @@ for (const directory of [distribution, join(distribution, "lib/runtime-sources/v if (readdirSync(directory).some((name) => ["node_modules", ".cache", "seed"].includes(name))) throw new Error(`development files present in ${directory}`); } +// A build's record of the runtime pack's verified artifacts (runtime-pack-native.ts) +// names that host's file identities. +for (const name of readdirSync(join(distribution, "lib"))) { + if ( + name.startsWith("runtime-") && + existsSync(join(distribution, "lib", name, "runtime-pack.verified.json")) + ) + throw new Error(`runtime pack verification record present in lib/${name}`); +} if (process.argv.includes("--run")) { if (target.startsWith("linux-")) { for (const executable of [