Check unchanged frontend inputs by file identity before hashing them (#2128)

packages/compiler/src/frontend/input-tracker.test.ts

@@ -270,3 +270,49 @@ test("synchronous tracking restores parents after throws and propagates unstable
   trackedFileExists(join(dir, "outside.ts"));
   expect(parent.snapshot()).toEqual(before);
 });
+
+test("a settled file is checked by its identity, and any write invalidates it", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
+  scratch.push(dir);
+  const source = join(dir, "entry.ts");
+  await writeFile(source, "export const answer = 1;\n");
+  const written = fs.statSync(source);
+  // The file was last changed long enough before the read.
+  vi.useFakeTimers({ toFake: ["Date"] });
+  vi.setSystemTime(Date.now() + 5000);
+  const tracker = new FrontendInputTracker();
+  try {
+    tracker.run(() => trackedReadFile(source));
+  } finally {
+    vi.useRealTimers();
+  }
+  const snapshot = tracker.snapshot();
+  const probe = snapshot.probes.find((entry) => entry.op === "file");
+  expect(probe?.op === "file" && probe.identity !== undefined).toBe(true);
+  expect(validFrontendInputSnapshot(snapshot)).toBe(true);
+  const reads = vi.mocked(fs.readFileSync);
+  reads.mockClear();
+  expect(frontendInputsStillMatch(snapshot)).toBe(true);
+  expect(reads.mock.calls.filter(([path]) => path === source)).toHaveLength(0);
+  // A same-size rewrite that restores the modification time still changes
+  // the change time.
+  await writeFile(source, "export const answer = 2;\n");
+  fs.utimesSync(source, written.atime, written.mtime);
+  expect(frontendInputsStillMatch(snapshot)).toBe(false);
+});
+
+test("a file changed shortly before the read is hashed on every check", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
+  scratch.push(dir);
+  const source = join(dir, "entry.ts");
+  await writeFile(source, "export const answer = 1;\n");
+  const tracker = new FrontendInputTracker();
+  tracker.run(() => trackedReadFile(source));
+  const snapshot = tracker.snapshot();
+  const probe = snapshot.probes.find((entry) => entry.op === "file");
+  expect(probe?.op === "file" && probe.identity).toBeUndefined();
+  const reads = vi.mocked(fs.readFileSync);
+  reads.mockClear();
+  expect(frontendInputsStillMatch(snapshot)).toBe(true);
+  expect(reads.mock.calls.filter(([path]) => path === source)).toHaveLength(1);
+});

packages/compiler/src/frontend/input-tracker.ts

@@ -12,7 +12,8 @@ import { observeSourceText } from "./ts7/ast-node.js";
  * behind an old resolution answer.
  */
 export type FrontendInputProbe =
-  | { op: "file"; path: string; digest: string }
+  /** `identity` (fileIdentity) lets a later check skip hashing the file. */
+  | { op: "file"; path: string; digest: string; identity?: string }
   | { op: "read-error"; path: string }
   | { op: "kind"; path: string; kind: "file" | "directory" | "other" | "missing" }
   | { op: "entries"; path: string; files: string[]; directories: string[] }
@@ -53,6 +54,36 @@ function frontendSourceDigest(text: string): string {
   return createHash("sha256").update(text).digest("hex");
 }
 
+/** How long a file must have been unchanged before a read for its identity
+ * to stand for its content: a later write then gets a newer change time
+ * even on file systems whose timestamps tick once a second or every two. */
+const IDENTITY_SETTLE_MS = 2000;
+
+/** The device, inode, size, modification time and change time of `path`. A
+ * write changes the change time, which no tool can set back, so an equal
+ * identity means the content is the one hashed with it. */
+function fileIdentity(path: string): string | null {
+  try {
+    const info = statSync(path);
+    return `${info.dev}:${info.ino}:${info.size}:${info.mtimeMs}:${info.ctimeMs}`;
+  } catch {
+    return null;
+  }
+}
+
+/** The text of `path` with its identity when the identity was the same before
+ * and after the read and the file had been unchanged for IDENTITY_SETTLE_MS. */
+function readWithIdentity(path: string): { text: string; identity: string | null } {
+  const readAt = Date.now();
+  const before = statSync(path);
+  const text = readFileSync(path, "utf8");
+  const after = fileIdentity(path);
+  const identity = `${before.dev}:${before.ino}:${before.size}:${before.mtimeMs}:${before.ctimeMs}`;
+  const settled =
+    before.mtimeMs <= readAt - IDENTITY_SETTLE_MS && before.ctimeMs <= readAt - IDENTITY_SETTLE_MS;
+  return { text, identity: settled && after === identity ? identity : null };
+}
+
 function pathKind(path: string): Extract<FrontendInputProbe, { op: "kind" }>["kind"] {
   try {
     const info = statSync(path);
@@ -199,8 +230,11 @@ export function trackedReadFile(path: string): string | null {
   const virtual = virtualSources.get(path);
   if (virtual !== undefined) return virtual;
   try {
-    const text = readFileSync(path, "utf8");
-    record({ op: "file", path, digest: frontendSourceDigest(text) });
+    const { text, identity } = readWithIdentity(path);
+    const digest = frontendSourceDigest(text);
+    record(
+      identity === null ? { op: "file", path, digest } : { op: "file", path, digest, identity },
+    );
     return text;
   } catch {
     record({ op: "read-error", path });
@@ -337,6 +371,9 @@ export function frontendInputsStillMatch(
     }
     switch (probe.op) {
       case "file": {
+        // An unchanged identity stands for the content hashed with it.
+        if (probe.identity !== undefined && fileIdentity(probe.path) === probe.identity)
+          return true;
         try {
           return frontendSourceDigest(readFileSync(probe.path, "utf8")) === probe.digest;
         } catch {
@@ -443,6 +480,11 @@ export function frontendInputsSemanticallyMatch(
       probes.push(probe);
       continue;
     }
+    if (probe.identity !== undefined && fileIdentity(probe.path) === probe.identity) {
+      currentSources.set(probe.path, previous);
+      probes.push(probe);
+      continue;
+    }
     let current: string;
     try {
       current = readFileSync(probe.path, "utf8");
@@ -460,7 +502,7 @@ export function frontendInputsSemanticallyMatch(
     if (rawText.has(probe.path) || !isSemanticEquivalent(probe.path, previous, current))
       return null;
     changed.push({ path: probe.path, previous, current });
-    probes.push({ ...probe, digest: currentDigest });
+    probes.push({ op: "file", path: probe.path, digest: currentDigest });
   }
   const adjusted: FrontendInputSnapshot = { ...snapshot, probes };
   if (!frontendInputsStillMatch(adjusted, exclusions)) return null;
@@ -488,7 +530,11 @@ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is Front
     if (typeof value.path !== "string" || typeof value.op !== "string") return false;
     switch (value.op) {
       case "file":
-        return typeof value.digest === "string" && /^[0-9a-f]{64}$/.test(value.digest);
+        return (
+          typeof value.digest === "string" &&
+          /^[0-9a-f]{64}$/.test(value.digest) &&
+          (value.identity === undefined || typeof value.identity === "string")
+        );
       case "read-error":
         return true;
       case "kind":