diff --git a/packages/compiler/src/frontend/token-equivalence.test.ts b/packages/compiler/src/frontend/token-equivalence.test.ts
index 880a07221..7bbe08929 100644
--- a/packages/compiler/src/frontend/token-equivalence.test.ts
+++ b/packages/compiler/src/frontend/token-equivalence.test.ts
@@ -1,8 +1,10 @@
import { expect, test } from "vitest";
-import { tokenEquivalentSource } from "./token-equivalence.js";
+import { tokenEquivalentSource, tokenPositionEquivalentSource } from "./token-equivalence.js";
const equivalent = (previous: string, current: string, path = "/src/a.ts") =>
tokenEquivalentSource(path, previous, current);
+const positionEquivalent = (previous: string, current: string, path = "/src/a.ts") =>
+ tokenPositionEquivalentSource(path, previous, current);
test("whitespace and plain comments are trivia", () => {
expect(equivalent("const a = 1;\n", "const a = 1; // one\n")).toBe(true);
@@ -105,3 +107,50 @@ test("text the scanner cannot classify is never equivalent", () => {
expect(equivalent("const a = ; // a\n", "const a = ; // b\n", "/src/a.tsx")).toBe(false);
expect(equivalent('const s = "open\n', 'const s = "open // \n')).toBe(false);
});
+
+test("a / after a non-null assertion divides", () => {
+ expect(
+ equivalent("const q = data[i]! / 7; // ratio\n", "const q = data[i]! / 7; // the ratio\n"),
+ ).toBe(true);
+ expect(
+ equivalent("const q = value! / 7; // ratio\n", "const q = value! / 7; // the ratio\n"),
+ ).toBe(true);
+ expect(equivalent("const q = this! / 7; // ratio\n", "const q = this! / 7; // the ratio\n")).toBe(
+ true,
+ );
+ // A prefix `!` negates a regular expression, whose `//` is code.
+ expect(
+ equivalent("const t = !/[//]/.test(s); c = 1;\n", "const t = !/[//]/.test(s); c = 2;\n"),
+ ).toBe(false);
+ expect(equivalent("return !/[//]/.test(s); // one\n", "return !/[//]/.test(s); // two\n")).toBe(
+ true,
+ );
+ // After a line break, `!` starts a new expression.
+ expect(equivalent("a\n!/[//]/.test(s); c = 1;\n", "a\n!/[//]/.test(s); c = 2;\n")).toBe(false);
+ // `f()!` and `if (a) !` read the same up to the `/`.
+ expect(equivalent("const q = f()! / 2; // one\n", "const q = f()! / 2; // two\n")).toBe(false);
+});
+
+test("position equivalence also keeps every token's line and column", () => {
+ const source = "const a = 1; // one\nconst b = a; /* two */ f(b);\n";
+ // A comment's text within its line, after the line's last token.
+ expect(
+ positionEquivalent(source, "const a = 1; // the first\nconst b = a; /* two */ f(b);\n"),
+ ).toBe(true);
+ // Comments and whitespace after the last token, and trailing whitespace.
+ expect(positionEquivalent(source, source + "// appended\n\n")).toBe(true);
+ expect(
+ positionEquivalent(source, "const a = 1; // one\nconst b = a; /* two */ f(b); \n"),
+ ).toBe(true);
+ expect(positionEquivalent(source, source.replaceAll("\n", "\r\n"))).toBe(true);
+ // A longer comment before a token moves its column; a new line moves the
+ // lines below it. Both stay token-equivalent.
+ const column = "const a = 1; // one\nconst b = a; /* the second */ f(b);\n";
+ const line = "// header\nconst a = 1; // one\nconst b = a; /* two */ f(b);\n";
+ expect(positionEquivalent(source, column)).toBe(false);
+ expect(positionEquivalent(source, line)).toBe(false);
+ expect(equivalent(source, column)).toBe(true);
+ expect(equivalent(source, line)).toBe(true);
+ // A token edit is never equivalent.
+ expect(positionEquivalent(source, source.replace("1", "2"))).toBe(false);
+});
diff --git a/packages/compiler/src/frontend/token-equivalence.ts b/packages/compiler/src/frontend/token-equivalence.ts
index 7a0f6cc79..9f6fd4613 100644
--- a/packages/compiler/src/frontend/token-equivalence.ts
+++ b/packages/compiler/src/frontend/token-equivalence.ts
@@ -17,7 +17,10 @@
* before it. After `)` it divides unless the parenthesis closes an `if`,
* `while`, `for`, `for await`, or `with` header. After `}`, `++` or
* `--` the scanner cannot tell which, so it gives up, as it does for
- * escaped identifiers, which can spell keywords.
+ * escaped identifiers, which can spell keywords. After `!` in
+ * TypeScript it divides when the `!` follows an operand on the same line
+ * (a non-null assertion, `x! / 2`) and gives up when that operand is a
+ * `)`, `}`, `++` or `--`.
* - A `/` with a line break (or a comment that spans lines) since the
* previous token gives up: whether automatic semicolon insertion ends the
* statement before it decides how it reads.
@@ -35,6 +38,8 @@ interface Token {
text: string;
comment: boolean;
lineBreakBefore: boolean;
+ /** The offset of the token's first code unit. */
+ start: number;
}
/** Keywords after which `/` starts a regular expression. */
@@ -135,14 +140,20 @@ function scan(path: string, source: string): Token[] | null {
// A line break, or a comment spanning lines, since the last token other
// than a comment.
let breakSinceCode = false;
- // The previous token and the one before it, and whether a `)` closed a
- // statement header.
- const last: { token: string | null; before: string | null } = { token: null, before: null };
+ // The previous token and the one before it, whether a line break came
+ // before the previous token, and whether a `)` closed a statement header.
+ const last: { token: string | null; before: string | null; broke: boolean } = {
+ token: null,
+ before: null,
+ broke: false,
+ };
let closedHeader = false;
+ // Every token ends at `index` when it is pushed.
const push = (text: string, comment = false): void => {
- tokens.push({ text, comment, lineBreakBefore });
+ tokens.push({ text, comment, lineBreakBefore, start: index - text.length });
lineBreakBefore = false;
if (!comment) {
+ last.broke = breakSinceCode;
breakSinceCode = false;
last.before = last.token;
last.token = text;
@@ -291,7 +302,19 @@ function scan(path: string, source: string): Token[] | null {
else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`"))
regex = false;
else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false;
- else regex = true;
+ else if (before === "!" && typeScript && !last.broke && last.before !== null) {
+ // `x! / 2` divides: a `!` right after an operand on the same line is
+ // a non-null assertion. `!/re/.test(s)` negates a regular expression.
+ const operand = last.before;
+ if (operand === ")" || operand === "}" || operand === "++" || operand === "--") return null;
+ regex = !(
+ operand === "]" ||
+ operand.startsWith("'") ||
+ operand.startsWith('"') ||
+ operand.endsWith("`") ||
+ (/^[A-Za-z0-9_$]/.test(operand) && !EXPRESSION_KEYWORDS.has(operand))
+ );
+ } else regex = true;
if (regex) {
const start = index;
index++;
@@ -373,9 +396,33 @@ function scan(path: string, source: string): Token[] | null {
return tokens;
}
-/** Whether `current` differs from `previous` only in whitespace and in
- * comments that cannot affect compilation. */
-export function tokenEquivalentSource(path: string, previous: string, current: string): boolean {
+/** The line and column of each token's start, with the line terminators
+ * source locations count (backend/source-locations.ts). */
+function tokenPositions(source: string, tokens: readonly Token[]): number[] {
+ const positions: number[] = [];
+ let line = 0;
+ let lineStart = 0;
+ let offset = 0;
+ for (const token of tokens) {
+ for (; offset < token.start; offset++) {
+ const code = source.charCodeAt(offset);
+ if (code === 13 && source.charCodeAt(offset + 1) === 10) offset++;
+ if (code === 10 || code === 13 || code === 0x2028 || code === 0x2029) {
+ line++;
+ lineStart = offset + 1;
+ }
+ }
+ positions.push(line, token.start - lineStart);
+ }
+ return positions;
+}
+
+function equivalentSources(
+ path: string,
+ previous: string,
+ current: string,
+ positions: boolean,
+): boolean {
if (previous === current) return true;
const left = scan(path, previous);
if (left === null) return false;
@@ -387,5 +434,27 @@ export function tokenEquivalentSource(path: string, previous: string, current: s
if (a.text !== b.text || a.comment !== b.comment) return false;
if (index > 0 && a.lineBreakBefore !== b.lineBreakBefore) return false;
}
- return true;
+ if (!positions) return true;
+ const before = tokenPositions(previous, left);
+ const after = tokenPositions(current, right);
+ return before.every((value, index) => value === after[index]);
+}
+
+/** Whether `current` differs from `previous` only in whitespace and in
+ * comments that cannot affect compilation. */
+export function tokenEquivalentSource(path: string, previous: string, current: string): boolean {
+ return equivalentSources(path, previous, current, false);
+}
+
+/** Whether `current` is token-equivalent to `previous` (tokenEquivalentSource)
+ * and every token keeps its line and column: the edit leaves the source
+ * positions debug information records unchanged (a comment's text within
+ * its line, comments and whitespace after the last token, trailing
+ * whitespace). */
+export function tokenPositionEquivalentSource(
+ path: string,
+ previous: string,
+ current: string,
+): boolean {
+ return equivalentSources(path, previous, current, true);
}
diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts
index 7ae2d1d5f..5d0f9faae 100644
--- a/packages/compiler/src/native/prepare.test.ts
+++ b/packages/compiler/src/native/prepare.test.ts
@@ -70,7 +70,7 @@ function fixture() {
loads++;
return runFrontend(path, loadProgram, npmStatic, externalTypes);
};
- const prepare = (extra: { emitIr?: boolean } = {}) =>
+ const prepare = (extra: { emitIr?: boolean; optimization?: "release" | "dev" } = {}) =>
prepareNativeExecutable(entry, { ...options, ...extra }, null, toolchain, frontend, cache);
return { root, entry, dependency, cache, options, prepare, loads: () => loads };
}
@@ -123,6 +123,38 @@ test("comment and whitespace edits reuse the frontend output a fresh build repro
expect(edited.ok && edited.llvm.join("")).not.toBe(first.llvm.join(""));
});
+test("a dev build reuses its frontend output after edits that keep every token's position", () => {
+ const f = fixture();
+ const dev = { optimization: "dev" as const };
+ const first = f.prepare(dev);
+ expect(first.ok, JSON.stringify(first)).toBe(true);
+ if (!first.ok) return;
+ // Debug information records lines and columns: a comment after the last
+ // token and trailing whitespace keep them.
+ const appended = 'export const value = "first"; \n// The exported value.\n';
+ writeFileSync(f.dependency, appended);
+ const reused = f.prepare(dev);
+ expect(f.loads()).toBe(1);
+ expect(reused.ok).toBe(true);
+ if (!reused.ok) return;
+ expect(reused.llvm.join("")).toBe(first.llvm.join(""));
+ expect(new Map(reused.sources).get(f.dependency)).toBe(appended);
+ const fresh = prepareNativeExecutable(
+ f.entry,
+ { ...f.options, ...dev },
+ null,
+ fixtureToolchain(),
+ (path, npmStatic, externalTypes) => runFrontend(path, loadProgram, npmStatic, externalTypes),
+ new NativeCache(join(f.root, "fresh-cache")),
+ );
+ expect(fresh.ok && fresh.llvm.join("")).toBe(first.llvm.join(""));
+ // A comment that moves the token after it is rebuilt.
+ writeFileSync(f.dependency, '// The exported value.\nexport const value = "first";\n');
+ const moved = f.prepare(dev);
+ expect(f.loads()).toBe(2);
+ expect(moved.ok && moved.llvm.join("")).not.toBe(first.llvm.join(""));
+});
+
test("a comment edit inside source text the output carries is rebuilt", () => {
const f = fixture();
// A failing assert(value) reports its call's source text.
diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts
index 2efc3e4d7..7118e47b9 100644
--- a/packages/compiler/src/native/prepare.ts
+++ b/packages/compiler/src/native/prepare.ts
@@ -32,7 +32,10 @@ import {
import { compilerOwnedFileRoots } from "../frontend/dts-paths.js";
import { provenanceSources } from "../frontend/provenance-registry.js";
import { processFork } from "../frontend/process-fork.js";
-import { tokenEquivalentSource } from "../frontend/token-equivalence.js";
+import {
+ tokenEquivalentSource,
+ tokenPositionEquivalentSource,
+} from "../frontend/token-equivalence.js";
import { llvmBytes, wtf8Bytes } from "../backend/literals.js";
import { prepareLibrary } from "../library/prepare.js";
import type { LibraryProfile } from "../library/library-profile.js";
@@ -143,9 +146,11 @@ function readFrontendCache(
/** The stale entry's input when every changed source is token-equivalent to
* the text the entry was built from (edits to whitespace and to comments
* that cannot affect compilation; see token-equivalence.ts). Equivalent
- * sources lower to the same IR apart from source locations, which only dev
- * builds' debug information and the serialized IR carry. Diagnostics carry
- * locations too, so an entry that reported warnings is rebuilt. A changed
+ * sources lower to the same IR apart from source locations, which only the
+ * debug information of dev builds and the serialized IR carry. A build
+ * with debug information accepts only edits that keep every token's line
+ * and column, the positions the debug information records. Diagnostics
+ * carry locations too, so an entry that reported warnings is rebuilt. A changed
* source whose raw text or line positions the frontend read (call-site text
* in messages, comptime callbacks, test locations) is rebuilt as well:
* frontendInputsSemanticallyMatch refuses its `text` probe. */
@@ -155,17 +160,17 @@ function equivalentFrontendInput(
exclusions: FrontendInputExclusions,
): { input: NativeExecutableInput; probes: FrontendInputSnapshot } | null {
if (
- options.optimization === "dev" ||
options.sanitize === true ||
options.emitIr === true ||
stale.ir !== null ||
(stale.input.warnings?.length ?? 0) > 0
)
return null;
+ const debugInfo = options.optimization === "dev" && !options.strip;
const match = frontendInputsSemanticallyMatch(
stale.probes,
new Map(stale.input.sources),
- tokenEquivalentSource,
+ debugInfo ? tokenPositionEquivalentSource : tokenEquivalentSource,
exclusions,
);
if (match === null || match.changed.length === 0) return null;