Reuse dev builds' frontend output after edits that keep token positions

A native build reuses its stale frontend cache entry when every changed
source is token-equivalent to the text the entry was built from, but dev
builds never did: their debug information records source lines and
columns, which a token-equivalent edit can move.

Builds with debug information (dev builds without --strip) now reuse the
entry when every changed source is token-equivalent and every token keeps
its line and column (tokenPositionEquivalentSource): edits to a comment's
text within its line, comments and whitespace after the last token, and
trailing whitespace. Edits that move a token still rebuild.

The token scanner also reads a `/` after a non-null assertion as division
(`data[i]! / 7`). It used to read it as the start of a regular expression
and give up on the file, so comment edits in files with that pattern
always rebuilt. A `!` after a line break, or after an operator or a
keyword such as `return`, still starts a negated regular expression, and
a `!` after `)`, `}`, `++` or `--` gives up.

packages/compiler/src/frontend/token-equivalence.test.ts

@@ -1,8 +1,10 @@
 import { expect, test } from "vitest";
-import { tokenEquivalentSource } from "./token-equivalence.js";
+import { tokenEquivalentSource, tokenPositionEquivalentSource } from "./token-equivalence.js";
 
 const equivalent = (previous: string, current: string, path = "/src/a.ts") =>
   tokenEquivalentSource(path, previous, current);
+const positionEquivalent = (previous: string, current: string, path = "/src/a.ts") =>
+  tokenPositionEquivalentSource(path, previous, current);
 
 test("whitespace and plain comments are trivia", () => {
   expect(equivalent("const a = 1;\n", "const a = 1; // one\n")).toBe(true);
@@ -105,3 +107,50 @@ test("text the scanner cannot classify is never equivalent", () => {
   expect(equivalent("const a = <b/>; // a\n", "const a = <b/>; // b\n", "/src/a.tsx")).toBe(false);
   expect(equivalent('const s = "open\n', 'const s = "open // \n')).toBe(false);
 });
+
+test("a / after a non-null assertion divides", () => {
+  expect(
+    equivalent("const q = data[i]! / 7; // ratio\n", "const q = data[i]! / 7; // the ratio\n"),
+  ).toBe(true);
+  expect(
+    equivalent("const q = value! / 7; // ratio\n", "const q = value! / 7; // the ratio\n"),
+  ).toBe(true);
+  expect(equivalent("const q = this! / 7; // ratio\n", "const q = this! / 7; // the ratio\n")).toBe(
+    true,
+  );
+  // A prefix `!` negates a regular expression, whose `//` is code.
+  expect(
+    equivalent("const t = !/[//]/.test(s); c = 1;\n", "const t = !/[//]/.test(s); c = 2;\n"),
+  ).toBe(false);
+  expect(equivalent("return !/[//]/.test(s); // one\n", "return !/[//]/.test(s); // two\n")).toBe(
+    true,
+  );
+  // After a line break, `!` starts a new expression.
+  expect(equivalent("a\n!/[//]/.test(s); c = 1;\n", "a\n!/[//]/.test(s); c = 2;\n")).toBe(false);
+  // `f()!` and `if (a) !` read the same up to the `/`.
+  expect(equivalent("const q = f()! / 2; // one\n", "const q = f()! / 2; // two\n")).toBe(false);
+});
+
+test("position equivalence also keeps every token's line and column", () => {
+  const source = "const a = 1; // one\nconst b = a; /* two */ f(b);\n";
+  // A comment's text within its line, after the line's last token.
+  expect(
+    positionEquivalent(source, "const a = 1; // the first\nconst b = a; /* two */ f(b);\n"),
+  ).toBe(true);
+  // Comments and whitespace after the last token, and trailing whitespace.
+  expect(positionEquivalent(source, source + "// appended\n\n")).toBe(true);
+  expect(
+    positionEquivalent(source, "const a = 1;   // one\nconst b = a; /* two */ f(b);  \n"),
+  ).toBe(true);
+  expect(positionEquivalent(source, source.replaceAll("\n", "\r\n"))).toBe(true);
+  // A longer comment before a token moves its column; a new line moves the
+  // lines below it. Both stay token-equivalent.
+  const column = "const a = 1; // one\nconst b = a; /* the second */ f(b);\n";
+  const line = "// header\nconst a = 1; // one\nconst b = a; /* two */ f(b);\n";
+  expect(positionEquivalent(source, column)).toBe(false);
+  expect(positionEquivalent(source, line)).toBe(false);
+  expect(equivalent(source, column)).toBe(true);
+  expect(equivalent(source, line)).toBe(true);
+  // A token edit is never equivalent.
+  expect(positionEquivalent(source, source.replace("1", "2"))).toBe(false);
+});

packages/compiler/src/frontend/token-equivalence.ts

@@ -17,7 +17,10 @@
  *   before it. After `)` it divides unless the parenthesis closes an `if`,
  *   `while`, `for`, `for await`, or `with` header. After `}`, `++` or
  *   `--` the scanner cannot tell which, so it gives up, as it does for
- *   escaped identifiers, which can spell keywords.
+ *   escaped identifiers, which can spell keywords. After `!` in
+ *   TypeScript it divides when the `!` follows an operand on the same line
+ *   (a non-null assertion, `x! / 2`) and gives up when that operand is a
+ *   `)`, `}`, `++` or `--`.
  * - A `/` with a line break (or a comment that spans lines) since the
  *   previous token gives up: whether automatic semicolon insertion ends the
  *   statement before it decides how it reads.
@@ -35,6 +38,8 @@ interface Token {
   text: string;
   comment: boolean;
   lineBreakBefore: boolean;
+  /** The offset of the token's first code unit. */
+  start: number;
 }
 
 /** Keywords after which `/` starts a regular expression. */
@@ -135,14 +140,20 @@ function scan(path: string, source: string): Token[] | null {
   // A line break, or a comment spanning lines, since the last token other
   // than a comment.
   let breakSinceCode = false;
-  // The previous token and the one before it, and whether a `)` closed a
-  // statement header.
-  const last: { token: string | null; before: string | null } = { token: null, before: null };
+  // The previous token and the one before it, whether a line break came
+  // before the previous token, and whether a `)` closed a statement header.
+  const last: { token: string | null; before: string | null; broke: boolean } = {
+    token: null,
+    before: null,
+    broke: false,
+  };
   let closedHeader = false;
+  // Every token ends at `index` when it is pushed.
   const push = (text: string, comment = false): void => {
-    tokens.push({ text, comment, lineBreakBefore });
+    tokens.push({ text, comment, lineBreakBefore, start: index - text.length });
     lineBreakBefore = false;
     if (!comment) {
+      last.broke = breakSinceCode;
       breakSinceCode = false;
       last.before = last.token;
       last.token = text;
@@ -291,7 +302,19 @@ function scan(path: string, source: string): Token[] | null {
       else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`"))
         regex = false;
       else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false;
-      else regex = true;
+      else if (before === "!" && typeScript && !last.broke && last.before !== null) {
+        // `x! / 2` divides: a `!` right after an operand on the same line is
+        // a non-null assertion. `!/re/.test(s)` negates a regular expression.
+        const operand = last.before;
+        if (operand === ")" || operand === "}" || operand === "++" || operand === "--") return null;
+        regex = !(
+          operand === "]" ||
+          operand.startsWith("'") ||
+          operand.startsWith('"') ||
+          operand.endsWith("`") ||
+          (/^[A-Za-z0-9_$]/.test(operand) && !EXPRESSION_KEYWORDS.has(operand))
+        );
+      } else regex = true;
       if (regex) {
         const start = index;
         index++;
@@ -373,9 +396,33 @@ function scan(path: string, source: string): Token[] | null {
   return tokens;
 }
 
-/** Whether `current` differs from `previous` only in whitespace and in
- * comments that cannot affect compilation. */
-export function tokenEquivalentSource(path: string, previous: string, current: string): boolean {
+/** The line and column of each token's start, with the line terminators
+ * source locations count (backend/source-locations.ts). */
+function tokenPositions(source: string, tokens: readonly Token[]): number[] {
+  const positions: number[] = [];
+  let line = 0;
+  let lineStart = 0;
+  let offset = 0;
+  for (const token of tokens) {
+    for (; offset < token.start; offset++) {
+      const code = source.charCodeAt(offset);
+      if (code === 13 && source.charCodeAt(offset + 1) === 10) offset++;
+      if (code === 10 || code === 13 || code === 0x2028 || code === 0x2029) {
+        line++;
+        lineStart = offset + 1;
+      }
+    }
+    positions.push(line, token.start - lineStart);
+  }
+  return positions;
+}
+
+function equivalentSources(
+  path: string,
+  previous: string,
+  current: string,
+  positions: boolean,
+): boolean {
   if (previous === current) return true;
   const left = scan(path, previous);
   if (left === null) return false;
@@ -387,5 +434,27 @@ export function tokenEquivalentSource(path: string, previous: string, current: s
     if (a.text !== b.text || a.comment !== b.comment) return false;
     if (index > 0 && a.lineBreakBefore !== b.lineBreakBefore) return false;
   }
-  return true;
+  if (!positions) return true;
+  const before = tokenPositions(previous, left);
+  const after = tokenPositions(current, right);
+  return before.every((value, index) => value === after[index]);
+}
+
+/** Whether `current` differs from `previous` only in whitespace and in
+ * comments that cannot affect compilation. */
+export function tokenEquivalentSource(path: string, previous: string, current: string): boolean {
+  return equivalentSources(path, previous, current, false);
+}
+
+/** Whether `current` is token-equivalent to `previous` (tokenEquivalentSource)
+ * and every token keeps its line and column: the edit leaves the source
+ * positions debug information records unchanged (a comment's text within
+ * its line, comments and whitespace after the last token, trailing
+ * whitespace). */
+export function tokenPositionEquivalentSource(
+  path: string,
+  previous: string,
+  current: string,
+): boolean {
+  return equivalentSources(path, previous, current, true);
 }

packages/compiler/src/native/prepare.test.ts

@@ -70,7 +70,7 @@ function fixture() {
     loads++;
     return runFrontend(path, loadProgram, npmStatic, externalTypes);
   };
-  const prepare = (extra: { emitIr?: boolean } = {}) =>
+  const prepare = (extra: { emitIr?: boolean; optimization?: "release" | "dev" } = {}) =>
     prepareNativeExecutable(entry, { ...options, ...extra }, null, toolchain, frontend, cache);
   return { root, entry, dependency, cache, options, prepare, loads: () => loads };
 }
@@ -123,6 +123,38 @@ test("comment and whitespace edits reuse the frontend output a fresh build repro
   expect(edited.ok && edited.llvm.join("")).not.toBe(first.llvm.join(""));
 });
 
+test("a dev build reuses its frontend output after edits that keep every token's position", () => {
+  const f = fixture();
+  const dev = { optimization: "dev" as const };
+  const first = f.prepare(dev);
+  expect(first.ok, JSON.stringify(first)).toBe(true);
+  if (!first.ok) return;
+  // Debug information records lines and columns: a comment after the last
+  // token and trailing whitespace keep them.
+  const appended = 'export const value = "first";  \n// The exported value.\n';
+  writeFileSync(f.dependency, appended);
+  const reused = f.prepare(dev);
+  expect(f.loads()).toBe(1);
+  expect(reused.ok).toBe(true);
+  if (!reused.ok) return;
+  expect(reused.llvm.join("")).toBe(first.llvm.join(""));
+  expect(new Map(reused.sources).get(f.dependency)).toBe(appended);
+  const fresh = prepareNativeExecutable(
+    f.entry,
+    { ...f.options, ...dev },
+    null,
+    fixtureToolchain(),
+    (path, npmStatic, externalTypes) => runFrontend(path, loadProgram, npmStatic, externalTypes),
+    new NativeCache(join(f.root, "fresh-cache")),
+  );
+  expect(fresh.ok && fresh.llvm.join("")).toBe(first.llvm.join(""));
+  // A comment that moves the token after it is rebuilt.
+  writeFileSync(f.dependency, '// The exported value.\nexport const value = "first";\n');
+  const moved = f.prepare(dev);
+  expect(f.loads()).toBe(2);
+  expect(moved.ok && moved.llvm.join("")).not.toBe(first.llvm.join(""));
+});
+
 test("a comment edit inside source text the output carries is rebuilt", () => {
   const f = fixture();
   // A failing assert(value) reports its call's source text.

packages/compiler/src/native/prepare.ts

@@ -32,7 +32,10 @@ import {
 import { compilerOwnedFileRoots } from "../frontend/dts-paths.js";
 import { provenanceSources } from "../frontend/provenance-registry.js";
 import { processFork } from "../frontend/process-fork.js";
-import { tokenEquivalentSource } from "../frontend/token-equivalence.js";
+import {
+  tokenEquivalentSource,
+  tokenPositionEquivalentSource,
+} from "../frontend/token-equivalence.js";
 import { llvmBytes, wtf8Bytes } from "../backend/literals.js";
 import { prepareLibrary } from "../library/prepare.js";
 import type { LibraryProfile } from "../library/library-profile.js";
@@ -143,9 +146,11 @@ function readFrontendCache(
 /** The stale entry's input when every changed source is token-equivalent to
  * the text the entry was built from (edits to whitespace and to comments
  * that cannot affect compilation; see token-equivalence.ts). Equivalent
- * sources lower to the same IR apart from source locations, which only dev
- * builds' debug information and the serialized IR carry. Diagnostics carry
- * locations too, so an entry that reported warnings is rebuilt. A changed
+ * sources lower to the same IR apart from source locations, which only the
+ * debug information of dev builds and the serialized IR carry. A build
+ * with debug information accepts only edits that keep every token's line
+ * and column, the positions the debug information records. Diagnostics
+ * carry locations too, so an entry that reported warnings is rebuilt. A changed
  * source whose raw text or line positions the frontend read (call-site text
  * in messages, comptime callbacks, test locations) is rebuilt as well:
  * frontendInputsSemanticallyMatch refuses its `text` probe. */
@@ -155,17 +160,17 @@ function equivalentFrontendInput(
   exclusions: FrontendInputExclusions,
 ): { input: NativeExecutableInput; probes: FrontendInputSnapshot } | null {
   if (
-    options.optimization === "dev" ||
     options.sanitize === true ||
     options.emitIr === true ||
     stale.ir !== null ||
     (stale.input.warnings?.length ?? 0) > 0
   )
     return null;
+  const debugInfo = options.optimization === "dev" && !options.strip;
   const match = frontendInputsSemanticallyMatch(
     stale.probes,
     new Map(stale.input.sources),
-    tokenEquivalentSource,
+    debugInfo ? tokenPositionEquivalentSource : tokenEquivalentSource,
     exclusions,
   );
   if (match === null || match.changed.length === 0) return null;