Scan regular expressions after jump statements, and give up on non-ASCII whitespace

packages/compiler/src/frontend/token-equivalence.test.ts

@@ -37,6 +37,28 @@ test("comment-like text inside regular expressions is code", () => {
   ).toBe(false);
 });
 
+test("a regular expression can start after a statement that ends at its keyword or label", () => {
+  const loop = (body: string) => `a: for (;;) {\n${body}\n}\n`;
+  expect(
+    equivalent(loop("break\n/[//]/.test(x); c = 1;"), loop("break\n/[//]/.test(x); c = 2;")),
+  ).toBe(false);
+  expect(
+    equivalent(loop("break a\n/[//]/.test(x); c = 1;"), loop("break a\n/[//]/.test(x); c = 2;")),
+  ).toBe(false);
+  expect(
+    equivalent("debugger\n/[//]/.test(x); c = 1;\n", "debugger\n/[//]/.test(x); c = 2;\n"),
+  ).toBe(false);
+  // The next line's `x` is no label: it starts a new statement, and `/` divides.
+  expect(equivalent(loop("continue\nx / 2; // one"), loop("continue\nx / 2; // two"))).toBe(true);
+});
+
+test("whitespace outside ASCII is never equivalent", () => {
+  expect(
+    equivalent("return\u3000/[//]/.test(x); c = 1;\n", "return\u3000/[//]/.test(x); c = 2;\n"),
+  ).toBe(false);
+  expect(equivalent("const a = 1;\u00a0// one\n", "const a = 1;\u00a0// two\n")).toBe(false);
+});
+
 test("U+2028 and U+2029 end a line comment, so their files are never equivalent", () => {
   expect(equivalent("// a\u2028c = 1;\n", "// a\u2028c = 2;\n")).toBe(false);
   expect(equivalent("/* a\u2029*/ c = 1;\n", "/* b\u2029*/ c = 1;\n")).toBe(false);

packages/compiler/src/frontend/token-equivalence.ts

@@ -19,10 +19,13 @@
  *   cannot tell which, so it gives up, as it does for escaped identifiers,
  *   which can spell keywords.
  * - Template literals track their `${` substitutions on a brace stack.
- * - JSX files, files that contain U+2028 or U+2029 anywhere (they end a
- *   line comment and separate lines for automatic semicolon insertion),
- *   and unterminated strings, comments, templates and regular expressions
- *   give up.
+ * - After `break`, `continue` and `debugger`, and after the label of a
+ *   `break` or `continue`, a `/` starts a regular expression: it can only
+ *   follow there once automatic semicolon insertion ends the statement.
+ * - JSX files, files that contain whitespace or a line terminator outside
+ *   ASCII anywhere (the identifier scan would take it as part of a word, and
+ *   U+2028 and U+2029 end a line comment), and unterminated strings,
+ *   comments, templates and regular expressions give up.
  *
  * Plain character scans keep the scanner free of Unicode property escapes:
  * any non-ASCII code unit outside strings, templates, regular expressions
@@ -55,6 +58,12 @@ const EXPRESSION_KEYWORDS = new Set([
   "extends",
 ]);
 
+/** Statements that end at their keyword, or at the label after it. */
+const JUMP_KEYWORDS = new Set(["break", "continue", "debugger"]);
+
+/** Whitespace and line terminators outside ASCII. */
+const NON_ASCII_SPACE = /[\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/;
+
 /** Keywords whose parenthesized header is followed by a statement. */
 const HEADER_KEYWORDS = new Set(["if", "while", "for", "with"]);
 
@@ -120,7 +129,7 @@ function isSpace(code: number): boolean {
 /** The tokens of `source`, or null when the scanner cannot classify it. */
 function scan(path: string, source: string): Token[] | null {
   if (path.endsWith(".tsx") || path.endsWith(".jsx")) return null;
-  if (source.includes("\u2028") || source.includes("\u2029")) return null;
+  if (NON_ASCII_SPACE.test(source)) return null;
   const typeScript = isTypeScript(path);
   const tokens: Token[] = [];
   // Brace kinds: true for a template substitution, false otherwise.
@@ -128,14 +137,23 @@ function scan(path: string, source: string): Token[] | null {
   // Paren kinds: true for a statement header.
   const parens: boolean[] = [];
   let lineBreakBefore = false;
-  // The previous token and the one before it, and whether a `)` closed a
-  // statement header.
-  const last: { token: string | null; before: string | null } = { token: null, before: null };
+  // The previous token and the one before it, whether the previous token is
+  // the label of a `break` or `continue` on its line, and whether a `)`
+  // closed a statement header.
+  const last: { token: string | null; before: string | null; label: boolean } = {
+    token: null,
+    before: null,
+    label: false,
+  };
   let closedHeader = false;
   const push = (text: string, comment = false): void => {
     tokens.push({ text, comment, lineBreakBefore });
     lineBreakBefore = false;
     if (!comment) {
+      last.label =
+        (last.token === "break" || last.token === "continue") &&
+        !tokens[tokens.length - 1]!.lineBreakBefore &&
+        /^[A-Za-z_$]/.test(text);
       last.before = last.token;
       last.token = text;
     }
@@ -184,7 +202,6 @@ function scan(path: string, source: string): Token[] | null {
       index++;
       continue;
     }
-    if (code > 127 && (code === 0xa0 || code === 0xfeff)) return null;
     const next = source.charCodeAt(index + 1);
     // Comments.
     if (code === 47 && (next === 47 || next === 42)) {
@@ -276,7 +293,7 @@ function scan(path: string, source: string): Token[] | null {
       else if (before === "++" || before === "--") return null;
       else if (before === "]") regex = false;
       else if (/^[A-Za-z0-9_$\\]/.test(before) || before.charCodeAt(0) > 127)
-        regex = EXPRESSION_KEYWORDS.has(before);
+        regex = EXPRESSION_KEYWORDS.has(before) || JUMP_KEYWORDS.has(before) || last.label;
       else if (before.startsWith("'") || before.startsWith('"') || before.endsWith("`"))
         regex = false;
       else if (before.startsWith("/") && before.length > 1 && before !== "/=") regex = false;