diff --git a/packages/compiler/src/frontend/token-equivalence.test.ts b/packages/compiler/src/frontend/token-equivalence.test.ts index 880a07221..7bbe08929 100644 --- a/packages/compiler/src/frontend/token-equivalence.test.ts +++ b/packages/compiler/src/frontend/token-equivalence.test.ts @@ -1,8 +1,10 @@ import { expect, test } from "vitest"; -import { tokenEquivalentSource } from "./token-equivalence.js"; +import { tokenEquivalentSource, tokenPositionEquivalentSource } from "./token-equivalence.js"; const equivalent = (previous: string, current: string, path = "/src/a.ts") => tokenEquivalentSource(path, previous, current); +const positionEquivalent = (previous: string, current: string, path = "/src/a.ts") => + tokenPositionEquivalentSource(path, previous, current); test("whitespace and plain comments are trivia", () => { expect(equivalent("const a = 1;\n", "const a = 1; // one\n")).toBe(true); @@ -105,3 +107,50 @@ test("text the scanner cannot classify is never equivalent", () => { expect(equivalent("const a = ; // a\n", "const a = ; // b\n", "/src/a.tsx")).toBe(false); expect(equivalent('const s = "open\n', 'const s = "open // \n')).toBe(false); }); + +test("a / after a non-null assertion divides", () => { + expect( + equivalent("const q = data[i]! / 7; // ratio\n", "const q = data[i]! / 7; // the ratio\n"), + ).toBe(true); + expect( + equivalent("const q = value! / 7; // ratio\n", "const q = value! / 7; // the ratio\n"), + ).toBe(true); + expect(equivalent("const q = this! / 7; // ratio\n", "const q = this! / 7; // the ratio\n")).toBe( + true, + ); + // A prefix `!` negates a regular expression, whose `//` is code. + expect( + equivalent("const t = !/[//]/.test(s); c = 1;\n", "const t = !/[//]/.test(s); c = 2;\n"), + ).toBe(false); + expect(equivalent("return !/[//]/.test(s); // one\n", "return !/[//]/.test(s); // two\n")).toBe( + true, + ); + // After a line break, `!` starts a new expression. + expect(equivalent("a\n!/[//]/.test(s); c = 1;\n", "a\n!/[//]/.test(s); c = 2;\n")).toBe(false); + // `f()!` and `if (a) !` read the same up to the `/`. + expect(equivalent("const q = f()! / 2; // one\n", "const q = f()! / 2; // two\n")).toBe(false); +}); + +test("position equivalence also keeps every token's line and column", () => { + const source = "const a = 1; // one\nconst b = a; /* two */ f(b);\n"; + // A comment's text within its line, after the line's last token. + expect( + positionEquivalent(source, "const a = 1; // the first\nconst b = a; /* two */ f(b);\n"), + ).toBe(true); + // Comments and whitespace after the last token, and trailing whitespace. + expect(positionEquivalent(source, source + "// appended\n\n")).toBe(true); + expect( + positionEquivalent(source, "const a = 1; // one\nconst b = a; /* two */ f(b); \n"), + ).toBe(true); + expect(positionEquivalent(source, source.replaceAll("\n", "\r\n"))).toBe(true); + // A longer comment before a token moves its column; a new line moves the + // lines below it. Both stay token-equivalent. + const column = "const a = 1; // one\nconst b = a; /* the second */ f(b);\n"; + const line = "// header\nconst a = 1; // one\nconst b = a; /* two */ f(b);\n"; + expect(positionEquivalent(source, column)).toBe(false); + expect(positionEquivalent(source, line)).toBe(false); + expect(equivalent(source, column)).toBe(true); + expect(equivalent(source, line)).toBe(true); + // A token edit is never equivalent. + expect(positionEquivalent(source, source.replace("1", "2"))).toBe(false); +}); diff --git a/packages/compiler/src/frontend/token-equivalence.ts b/packages/compiler/src/frontend/token-equivalence.ts index 7a0f6cc79..9f6fd4613 100644 --- a/packages/compiler/src/frontend/token-equivalence.ts +++ b/packages/compiler/src/frontend/token-equivalence.ts @@ -17,7 +17,10 @@ * before it. After `)` it divides unless the parenthesis closes an `if`, * `while`, `for`, `for await`, or `with` header. After `}`, `++` or * `--` the scanner cannot tell which, so it gives up, as it does for - * escaped identifiers, which can spell keywords. + * escaped identifiers, which can spell keywords. After `!` in + * TypeScript it divides when the `!` follows an operand on the same line + * (a non-null assertion, `x! / 2`) and gives up when that operand is a + * `)`, `}`, `++` or `--`. * - A `/` with a line break (or a comment that spans lines) since the * previous token gives up: whether automatic semicolon insertion ends the * statement before it decides how it reads. @@ -35,6 +38,8 @@ interface Token { text: string; comment: boolean; lineBreakBefore: boolean; + /** The offset of the token's first code unit. */ + start: number; } /** Keywords after which `/` starts a regular expression. */ @@ -135,14 +140,20 @@ function scan(path: string, source: string): Token[] | null { // A line break, or a comment spanning lines, since the last token other // than a comment. let breakSinceCode = false; - // The previous token and the one before it, and whether a `)` closed a - // statement header. - const last: { token: string | null; before: string | null } = { token: null, before: null }; + // The previous token and the one before it, whether a line break came + // before the previous token, and whether a `)` closed a statement header. + const last: { token: string | null; before: string | null; broke: boolean } = { + token: null, + before: null, + broke: false, + }; let closedHeader = false; + // Every token ends at `index` when it is pushed. const push = (text: string, comment = false): void => { - tokens.push({ text, comment, lineBreakBefore }); + tokens.push({ text, comment, lineBreakBefore, start: index - text.length }); lineBreakBefore = false; if (!comment) { + last.broke = breakSinceCode; breakSinceCode = false; last.before = last.token; last.token = text; @@ -291,7 +302,19 @@ function scan(path: string, source: string): Token[] | null { else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`")) regex = false; else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false; - else regex = true; + else if (before === "!" && typeScript && !last.broke && last.before !== null) { + // `x! / 2` divides: a `!` right after an operand on the same line is + // a non-null assertion. `!/re/.test(s)` negates a regular expression. + const operand = last.before; + if (operand === ")" || operand === "}" || operand === "++" || operand === "--") return null; + regex = !( + operand === "]" || + operand.startsWith("'") || + operand.startsWith('"') || + operand.endsWith("`") || + (/^[A-Za-z0-9_$]/.test(operand) && !EXPRESSION_KEYWORDS.has(operand)) + ); + } else regex = true; if (regex) { const start = index; index++; @@ -373,9 +396,33 @@ function scan(path: string, source: string): Token[] | null { return tokens; } -/** Whether `current` differs from `previous` only in whitespace and in - * comments that cannot affect compilation. */ -export function tokenEquivalentSource(path: string, previous: string, current: string): boolean { +/** The line and column of each token's start, with the line terminators + * source locations count (backend/source-locations.ts). */ +function tokenPositions(source: string, tokens: readonly Token[]): number[] { + const positions: number[] = []; + let line = 0; + let lineStart = 0; + let offset = 0; + for (const token of tokens) { + for (; offset < token.start; offset++) { + const code = source.charCodeAt(offset); + if (code === 13 && source.charCodeAt(offset + 1) === 10) offset++; + if (code === 10 || code === 13 || code === 0x2028 || code === 0x2029) { + line++; + lineStart = offset + 1; + } + } + positions.push(line, token.start - lineStart); + } + return positions; +} + +function equivalentSources( + path: string, + previous: string, + current: string, + positions: boolean, +): boolean { if (previous === current) return true; const left = scan(path, previous); if (left === null) return false; @@ -387,5 +434,27 @@ export function tokenEquivalentSource(path: string, previous: string, current: s if (a.text !== b.text || a.comment !== b.comment) return false; if (index > 0 && a.lineBreakBefore !== b.lineBreakBefore) return false; } - return true; + if (!positions) return true; + const before = tokenPositions(previous, left); + const after = tokenPositions(current, right); + return before.every((value, index) => value === after[index]); +} + +/** Whether `current` differs from `previous` only in whitespace and in + * comments that cannot affect compilation. */ +export function tokenEquivalentSource(path: string, previous: string, current: string): boolean { + return equivalentSources(path, previous, current, false); +} + +/** Whether `current` is token-equivalent to `previous` (tokenEquivalentSource) + * and every token keeps its line and column: the edit leaves the source + * positions debug information records unchanged (a comment's text within + * its line, comments and whitespace after the last token, trailing + * whitespace). */ +export function tokenPositionEquivalentSource( + path: string, + previous: string, + current: string, +): boolean { + return equivalentSources(path, previous, current, true); } diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts index 7ae2d1d5f..5d0f9faae 100644 --- a/packages/compiler/src/native/prepare.test.ts +++ b/packages/compiler/src/native/prepare.test.ts @@ -70,7 +70,7 @@ function fixture() { loads++; return runFrontend(path, loadProgram, npmStatic, externalTypes); }; - const prepare = (extra: { emitIr?: boolean } = {}) => + const prepare = (extra: { emitIr?: boolean; optimization?: "release" | "dev" } = {}) => prepareNativeExecutable(entry, { ...options, ...extra }, null, toolchain, frontend, cache); return { root, entry, dependency, cache, options, prepare, loads: () => loads }; } @@ -123,6 +123,38 @@ test("comment and whitespace edits reuse the frontend output a fresh build repro expect(edited.ok && edited.llvm.join("")).not.toBe(first.llvm.join("")); }); +test("a dev build reuses its frontend output after edits that keep every token's position", () => { + const f = fixture(); + const dev = { optimization: "dev" as const }; + const first = f.prepare(dev); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + // Debug information records lines and columns: a comment after the last + // token and trailing whitespace keep them. + const appended = 'export const value = "first"; \n// The exported value.\n'; + writeFileSync(f.dependency, appended); + const reused = f.prepare(dev); + expect(f.loads()).toBe(1); + expect(reused.ok).toBe(true); + if (!reused.ok) return; + expect(reused.llvm.join("")).toBe(first.llvm.join("")); + expect(new Map(reused.sources).get(f.dependency)).toBe(appended); + const fresh = prepareNativeExecutable( + f.entry, + { ...f.options, ...dev }, + null, + fixtureToolchain(), + (path, npmStatic, externalTypes) => runFrontend(path, loadProgram, npmStatic, externalTypes), + new NativeCache(join(f.root, "fresh-cache")), + ); + expect(fresh.ok && fresh.llvm.join("")).toBe(first.llvm.join("")); + // A comment that moves the token after it is rebuilt. + writeFileSync(f.dependency, '// The exported value.\nexport const value = "first";\n'); + const moved = f.prepare(dev); + expect(f.loads()).toBe(2); + expect(moved.ok && moved.llvm.join("")).not.toBe(first.llvm.join("")); +}); + test("a comment edit inside source text the output carries is rebuilt", () => { const f = fixture(); // A failing assert(value) reports its call's source text. diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 2efc3e4d7..7118e47b9 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -32,7 +32,10 @@ import { import { compilerOwnedFileRoots } from "../frontend/dts-paths.js"; import { provenanceSources } from "../frontend/provenance-registry.js"; import { processFork } from "../frontend/process-fork.js"; -import { tokenEquivalentSource } from "../frontend/token-equivalence.js"; +import { + tokenEquivalentSource, + tokenPositionEquivalentSource, +} from "../frontend/token-equivalence.js"; import { llvmBytes, wtf8Bytes } from "../backend/literals.js"; import { prepareLibrary } from "../library/prepare.js"; import type { LibraryProfile } from "../library/library-profile.js"; @@ -143,9 +146,11 @@ function readFrontendCache( /** The stale entry's input when every changed source is token-equivalent to * the text the entry was built from (edits to whitespace and to comments * that cannot affect compilation; see token-equivalence.ts). Equivalent - * sources lower to the same IR apart from source locations, which only dev - * builds' debug information and the serialized IR carry. Diagnostics carry - * locations too, so an entry that reported warnings is rebuilt. A changed + * sources lower to the same IR apart from source locations, which only the + * debug information of dev builds and the serialized IR carry. A build + * with debug information accepts only edits that keep every token's line + * and column, the positions the debug information records. Diagnostics + * carry locations too, so an entry that reported warnings is rebuilt. A changed * source whose raw text or line positions the frontend read (call-site text * in messages, comptime callbacks, test locations) is rebuilt as well: * frontendInputsSemanticallyMatch refuses its `text` probe. */ @@ -155,17 +160,17 @@ function equivalentFrontendInput( exclusions: FrontendInputExclusions, ): { input: NativeExecutableInput; probes: FrontendInputSnapshot } | null { if ( - options.optimization === "dev" || options.sanitize === true || options.emitIr === true || stale.ir !== null || (stale.input.warnings?.length ?? 0) > 0 ) return null; + const debugInfo = options.optimization === "dev" && !options.strip; const match = frontendInputsSemanticallyMatch( stale.probes, new Map(stale.input.sources), - tokenEquivalentSource, + debugInfo ? tokenPositionEquivalentSource : tokenEquivalentSource, exclusions, ); if (match === null || match.changed.length === 0) return null;