Compare optional string index reads as code units

`s[i]` with an index the frontend cannot prove in bounds reads the
character or undefined: temps for the string and the index, a range and
integer test, and a string-or-undefined union. A lexer that tests
`src[pos] === " "` or `c < "a"` on `const c = src[pos]` compared that
union with a union-wrapped literal, or boxed it for a dynamic
comparison, on every character.

Tests of such a read against a literal of one UTF-16 code unit now
compare `i === Math.floor(i) ? s.charCodeAt(i) : NaN` instead.
Undefined, like NaN, equals no literal and fails every ordered
comparison, and charCodeAt reads NaN out of range. A read of plain
operands is evaluated in place; others keep the read's temps.

An immutable local initialized from such a read, and used only in
those tests, in tests for undefined and in a switch over one-unit
cases, holds the code unit. Tests for undefined become NaN tests.
Copies of the local, such as a switch's discriminant temp, follow it.
The code-unit reads are charCodeAt reads, so a loop version can cover
them.

packages/compiler/src/ir/char-unit-locals.test.ts

@@ -96,3 +96,73 @@ console.log(escapes("a b", 1), rebound("  x", 0), wide("ab", 0), against("a", "b
     expect(fn(after, name)).toBe(fn(before, name));
   }
 });
+
+/** The union and dynamic comparisons left in the function. */
+function unionTests(f: IrFunction): string[] {
+  const out: string[] = [];
+  everyStmtList(f.body, {
+    stmt: () => true,
+    expr: (e) => {
+      if (e.kind === "unionEq" || e.kind === "unionIsTag") out.push(e.kind);
+      if (e.kind === "libCall" && e.fn === "js.compare") out.push(e.fn);
+      if (e.kind === "strIntrinsic") out.push(e.method);
+      return true;
+    },
+  });
+  return out;
+}
+
+test("optional character reads compare their code unit", async () => {
+  const mod = holdCharLocalsAsUnits(
+    await lower(`
+function scan(src: string, pos: number): number {
+  let n = 0;
+  if (src[pos] === " ") n += 1;
+  if (src[pos] !== "x") n += 2;
+  if (src[pos] < "a" || "z" <= src[pos]) n += 4;
+  if (src[pos + 1] === "=") n += 8;
+  const c = src[pos];
+  if (c === undefined) n += 16;
+  if (c !== undefined && c > "m") n += 32;
+  switch (c) {
+    case "q":
+      n += 64;
+      break;
+    case "r":
+      n += 128;
+  }
+  return n;
+}
+console.log(scan("a b", 1), scan("q", 0), scan("ab", 5), scan("r=", 0.5));
+`),
+  );
+  const scan = fn(mod, "scan");
+  // Only charCodeAt reads remain: no union tests, dynamic compares, charAt
+  // or length reads.
+  expect(new Set(unionTests(scan))).toEqual(new Set(["charCodeAt"]));
+  expect(scan.locals.find((l) => l.name === "c")!.type).toEqual({ kind: "f64" });
+});
+
+test("optional character reads with other uses stay reads", async () => {
+  const before = await lower(`
+function kept(src: string, pos: number): string {
+  const c = src[pos];
+  if (c === " ") return "space";
+  return c ?? "end";
+}
+function longer(src: string, pos: number): boolean {
+  return src[pos] === "ab";
+}
+function twice(src: string, pos: number): boolean {
+  return src[pos * 2] === undefined;
+}
+console.log(kept("a b", 1), kept("ab", 0), kept("", 3), longer("ab", 0), twice("ab", 1));
+`);
+  const after = holdCharLocalsAsUnits(before);
+  for (const name of ["kept", "longer", "twice"]) {
+    expect(fn(before, name)).toBeDefined();
+    expect(unionTests(fn(after, name))).toEqual(unionTests(fn(before, name)));
+  }
+  // The local's own test still reads it as a union.
+  expect(unionTests(fn(after, "kept"))).toContain("unionEq");
+});

packages/compiler/src/ir/char-unit-locals.ts

@@ -11,8 +11,11 @@ import { mapExprChildren, mapStmtChildren } from "./traverse.js";
 
 type CharAt = IrExpr & { kind: "strIntrinsic" };
 type UnitUse = IrExpr & { kind: "strEq" | "strCmp" };
+type CompareOp = "<" | "<=" | ">" | ">=";
 
-/** Character locals held as their code unit.
+const FLIPPED = { "<": ">", "<=": ">=", ">": "<", ">=": "<=" } as const;
+
+/** Character reads and locals held as their code unit.
  *
  * `const c = s[i]` (or `s.charAt(i)`) followed only by tests such as
  * `c === " "`, `c >= "a" && c <= "z"` or `switch (c) { case "x": ... }`
@@ -28,7 +31,16 @@ type UnitUse = IrExpr & { kind: "strEq" | "strCmp" };
  *
  * Only immutable unboxed string locals qualify, with one declaration whose
  * initializer is a one-argument charAt, and with no other mention of the
- * local. */
+ * local.
+ *
+ * `s[i]` with an index not proven in bounds reads the character or
+ * undefined (an optional read, see `optionalCharRead`). Its tests against
+ * one-unit literals compare `i === Math.floor(i) ? s.charCodeAt(i) : NaN`
+ * instead: undefined, like NaN, equals no literal and fails every ordered
+ * comparison, and charCodeAt answers NaN out of range. That holds both for
+ * the read in place and for an immutable local initialized from it and
+ * used only in such tests and in tests for undefined, which become NaN
+ * tests. */
 export function holdCharLocalsAsUnits(mod: IrModule): IrModule {
   let changed = false;
   const functions = mod.functions.map((fn) => {
@@ -39,11 +51,275 @@ export function holdCharLocalsAsUnits(mod: IrModule): IrModule {
   return changed ? { ...mod, functions } : mod;
 }
 
+/** The read of `s[i]` for an index not proven in bounds: temps for the
+ * string and the index, then the string's charAt when the index is an
+ * integer in [0, length) and undefined otherwise. */
+interface OptionalCharRead {
+  /** The temps' declarations, and whether both operands are plain (local
+   * reads and literals), which the unit read may evaluate in place. */
+  temps: [IrStmt & { kind: "varDecl" }, IrStmt & { kind: "varDecl" }];
+  plain: boolean;
+  receiver: IrExpr;
+  index: IrExpr;
+  unionId: string;
+  stringTag: number;
+  undefinedTag: number;
+}
+
+function optionalCharRead(e: IrExpr): OptionalCharRead | null {
+  if (e.kind !== "seqExpr" || e.stmts.length !== 2 || e.diverges) return null;
+  const [str, idx] = e.stmts;
+  if (str?.kind !== "varDecl" || idx?.kind !== "varDecl" || !str.init || !idx.init) return null;
+  const isStr = (x: IrExpr): boolean => x.kind === "varRef" && x.localId === str.localId;
+  const isIdx = (x: IrExpr): boolean => x.kind === "varRef" && x.localId === idx.localId;
+  const r = e.result;
+  if (r.kind !== "ternary") return null;
+  const c = r.cond;
+  if (c.kind !== "logical" || c.op !== "&&" || c.left.kind !== "logical" || c.left.op !== "&&")
+    return null;
+  const [low, high, whole] = [c.left.left, c.left.right, c.right];
+  if (
+    !(
+      low.kind === "bin" &&
+      low.op === ">=" &&
+      isIdx(low.left) &&
+      low.right.kind === "numLit" &&
+      low.right.value === 0
+    ) ||
+    !(
+      high.kind === "bin" &&
+      high.op === "<" &&
+      isIdx(high.left) &&
+      high.right.kind === "strIntrinsic" &&
+      high.right.method === "length" &&
+      isStr(high.right.receiver)
+    ) ||
+    !(
+      whole.kind === "bin" &&
+      whole.op === "===" &&
+      isIdx(whole.left) &&
+      whole.right.kind === "libCall" &&
+      whole.right.fn === "math.floor" &&
+      whole.right.args.length === 1 &&
+      isIdx(whole.right.args[0]!)
+    )
+  )
+    return null;
+  const [present, absent] = [r.then, r.else_];
+  if (present.kind !== "unionWrap" || absent.kind !== "unionWrap") return null;
+  if (present.unionId !== absent.unionId) return null;
+  const read = present.value;
+  if (
+    read.kind !== "strIntrinsic" ||
+    read.method !== "charAt" ||
+    !isStr(read.receiver) ||
+    read.args.length !== 1 ||
+    !isIdx(read.args[0]!)
+  )
+    return null;
+  if (absent.value.kind !== "unitLit" || absent.value.unit !== "undefined") return null;
+  return {
+    temps: [str, idx],
+    plain: plainOperand(str.init) && plainOperand(idx.init),
+    receiver: str.init,
+    index: idx.init,
+    unionId: present.unionId,
+    stringTag: present.tag,
+    undefinedTag: absent.tag,
+  };
+}
+
+function plainOperand(e: IrExpr): boolean {
+  return e.kind === "varRef" || e.kind === "numLit" || e.kind === "strLit";
+}
+
+/** `i === Math.floor(i) ? s.charCodeAt(i) : NaN`: the code unit of an
+ * optional read, or NaN where it reads undefined. Plain operands are read
+ * in place; others keep the read's temps, so each is evaluated once. */
+function unitRead(read: OptionalCharRead, loc: IrExpr["loc"]): IrExpr {
+  if (!read.plain) {
+    const [str, idx] = read.temps;
+    const temp = (decl: IrStmt & { kind: "varDecl" }, type: IrExpr["type"]): IrExpr => ({
+      kind: "varRef",
+      localId: decl.localId,
+      type,
+      loc,
+    });
+    return {
+      kind: "seqExpr",
+      stmts: [{ ...str }, { ...idx }],
+      result: unitOf(temp(str, read.receiver.type), () => temp(idx, F64), loc),
+      type: F64,
+      loc,
+    };
+  }
+  return unitOf({ ...read.receiver }, () => ({ ...read.index }), loc);
+}
+
+function unitOf(receiver: IrExpr, index: () => IrExpr, loc: IrExpr["loc"]): IrExpr {
+  return {
+    kind: "ternary",
+    cond: {
+      kind: "bin",
+      op: "===",
+      left: index(),
+      right: { kind: "libCall", fn: "math.floor", args: [index()], type: F64, loc },
+      type: BOOL,
+      loc,
+    },
+    then: {
+      kind: "strIntrinsic",
+      method: "charCodeAt",
+      receiver,
+      args: [index()],
+      type: F64,
+      loc,
+    },
+    else_: { kind: "numLit", value: NaN, type: F64, loc },
+    type: F64,
+    loc,
+  };
+}
+
+/** A test of an operand that reads one character or undefined, against a
+ * literal of one code unit or for undefined, and the same test on the
+ * operand's code unit. `tagsOf` gives the operand's union tags. */
+interface UnitTest {
+  operand: IrExpr;
+  /** The test reads the unit twice. */
+  twice?: boolean;
+  test: (unit: () => IrExpr) => IrExpr;
+}
+
+function optionalUnitTest(
+  e: IrExpr,
+  tagsOf: (operand: IrExpr) => OptionalCharRead | null,
+): UnitTest | null {
+  const loc = e.loc;
+  const compare = (op: "===" | "!==" | CompareOp, u: IrExpr, right: IrExpr): IrExpr => ({
+    kind: "bin",
+    op,
+    left: u,
+    right,
+    type: BOOL,
+    loc,
+  });
+  const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc });
+  if (e.kind === "unionEq" && !e.sameValue) {
+    for (const [operand, other] of [
+      [e.left, e.right],
+      [e.right, e.left],
+    ] as const) {
+      const tags = tagsOf(operand);
+      if (
+        tags &&
+        tags.unionId === e.unionId &&
+        other.kind === "unionWrap" &&
+        other.tag === tags.stringTag &&
+        other.value.kind === "strLit" &&
+        other.value.value.length === 1
+      ) {
+        const code = other.value.value.charCodeAt(0);
+        return {
+          operand,
+          test: (unit) => compare(e.negated ? "!==" : "===", unit(), num(code)),
+        };
+      }
+      // `case undefined:` of a switch: undefined is a NaN unit.
+      if (
+        tags &&
+        tags.unionId === e.unionId &&
+        other.kind === "unionWrap" &&
+        other.tag === tags.undefinedTag &&
+        other.value.kind === "unitLit" &&
+        other.value.unit === "undefined"
+      )
+        return {
+          operand,
+          twice: true,
+          test: (unit) => compare(e.negated ? "===" : "!==", unit(), unit()),
+        };
+    }
+    return null;
+  }
+  if (e.kind === "libCall" && e.fn === "js.compare" && e.args.length === 3) {
+    const [a, b, opArg] = e.args as [IrExpr, IrExpr, IrExpr];
+    if (opArg.kind !== "strLit" || !(opArg.value in FLIPPED)) return null;
+    const op = opArg.value as CompareOp;
+    const literalOf = (x: IrExpr): number | null =>
+      x.kind === "libCall" &&
+      x.fn === "js.fromStr" &&
+      x.args.length === 1 &&
+      x.args[0]!.kind === "strLit" &&
+      x.args[0]!.value.length === 1
+        ? x.args[0]!.value.charCodeAt(0)
+        : null;
+    const operandOf = (x: IrExpr): IrExpr | null =>
+      x.kind === "libCall" &&
+      x.fn === "js.fromDyn" &&
+      x.args.length === 1 &&
+      x.args[0]!.kind === "dynFrom" &&
+      tagsOf(x.args[0]!.value)
+        ? x.args[0]!.value
+        : null;
+    // The comparison with the operand on the left.
+    const left = operandOf(a);
+    const right = operandOf(b);
+    const code = left ? literalOf(b) : right ? literalOf(a) : null;
+    const operand = left ?? right;
+    if (operand === null || code === null) return null;
+    const ordered = left ? op : FLIPPED[op];
+    return { operand, test: (unit) => compare(ordered, unit(), num(code)) };
+  }
+  if (e.kind === "unionIsTag") {
+    const tags = tagsOf(e.value);
+    if (!tags || tags.unionId !== e.unionId) return null;
+    if (e.tag !== tags.undefinedTag && e.tag !== tags.stringTag) return null;
+    // Undefined is a NaN unit; a character is any other.
+    const present = (e.tag === tags.stringTag) !== e.negated;
+    return {
+      operand: e.value,
+      twice: true,
+      test: (unit) => compare(present ? "===" : "!==", unit(), unit()),
+    };
+  }
+  return null;
+}
+
 function rewriteFunction(fn: IrFunction): IrFunction {
+  const locals = new Map(fn.locals.map((l) => [l.id, l]));
   const candidates = new Set(
-    fn.locals.filter((l) => !l.mutable && !l.boxed && l.type.kind === "string").map((l) => l.id),
+    fn.locals
+      .filter(
+        (l) => !l.mutable && !l.boxed && (l.type.kind === "string" || l.type.kind === "union"),
+      )
+      .map((l) => l.id),
   );
-  if (candidates.size === 0) return fn;
+  // The optional reads candidate union locals are declared from, directly
+  // or as copies of another such local (a switch's discriminant temp).
+  const reads = new Map<string, OptionalCharRead>();
+  const copyOf = new Map<string, string>();
+  const collect = (value: unknown): void => {
+    if (Array.isArray(value)) for (const item of value) collect(item);
+    else if (value !== null && typeof value === "object") {
+      const node = value as Record<string, unknown>;
+      if (node.kind === "varDecl" && candidates.has(node.localId as string) && node.init) {
+        const id = node.localId as string;
+        const init = node.init as IrExpr;
+        const read = optionalCharRead(init);
+        if (read) reads.set(id, read);
+        else if (init.kind === "varRef" && reads.has(init.localId)) {
+          reads.set(id, reads.get(init.localId)!);
+          copyOf.set(id, init.localId);
+        }
+      }
+      for (const key in node) if (key !== "loc") collect(node[key]);
+    }
+  };
+  collect(fn.body);
+  const localTags = (operand: IrExpr): OptionalCharRead | null =>
+    operand.kind === "varRef" ? (reads.get(operand.localId) ?? null) : null;
+
   // Every mention of a candidate, in any field, against the mentions the
   // rewrite understands.
   const mentions = new Map<string, number>();
@@ -59,7 +335,7 @@ function rewriteFunction(fn: IrFunction): IrFunction {
       for (const item of value) count(item);
     } else if (value !== null && typeof value === "object") {
       const node = value as Record<string, unknown>;
-      const id = understoodMention(node);
+      const id = understoodMention(node, locals, reads, localTags);
       if (id !== null && candidates.has(id)) {
         note(understood, id);
         if (node.kind === "varDecl") note(declarations, id);
@@ -68,15 +344,26 @@ function rewriteFunction(fn: IrFunction): IrFunction {
     }
   };
   count(fn.body);
+  // A copy is held only with its source, and the source's mention in the
+  // copy's initializer is understood only while the copy is held.
   const held = new Set<string>();
   for (const id of candidates)
-    if (
-      declarations.get(id) === 1 &&
-      (mentions.get(id) ?? 0) > 0 &&
-      mentions.get(id) === understood.get(id)
-    )
-      held.add(id);
-  if (held.size === 0) return fn;
+    if (declarations.get(id) === 1 && (mentions.get(id) ?? 0) > 0) held.add(id);
+  for (let settled = false; !settled;) {
+    settled = true;
+    for (const id of held) {
+      let copies = 0;
+      for (const [copy, source] of copyOf) if (source === id && held.has(copy)) copies++;
+      const source = copyOf.get(id);
+      if (
+        mentions.get(id) !== (understood.get(id) ?? 0) + copies ||
+        (source !== undefined && !held.has(source))
+      ) {
+        held.delete(id);
+        settled = false;
+      }
+    }
+  }
 
   const unitRef = (id: string, loc: IrExpr["loc"]): IrExpr => ({
     kind: "varRef",
@@ -84,10 +371,19 @@ function rewriteFunction(fn: IrFunction): IrFunction {
     type: F64,
     loc,
   });
+  // Operands of optional unit tests: held union locals and reads in place.
+  const operandTags = (operand: IrExpr): OptionalCharRead | null =>
+    operand.kind === "varRef"
+      ? held.has(operand.localId)
+        ? localTags(operand)
+        : null
+      : optionalCharRead(operand);
+  let changed = false;
   const rewriteExpr = (e: IrExpr): IrExpr => {
     if (e.kind === "strEq" || e.kind === "strCmp") {
       const use = unitUse(e);
       if (use && held.has(use.id)) {
+        changed = true;
         const code: IrExpr = { kind: "numLit", value: use.code, type: F64, loc: e.loc };
         const c = unitRef(use.id, e.loc);
         if (e.kind === "strEq")
@@ -100,9 +396,7 @@ function rewriteFunction(fn: IrFunction): IrFunction {
             loc: e.loc,
           };
         // The comparison with the local on the left.
-        const op = use.left
-          ? e.op
-          : ({ "<": ">", "<=": ">=", ">": "<", ">=": "<=" } as const)[e.op];
+        const op = use.left ? e.op : FLIPPED[e.op];
         if (op === ">" || op === ">=")
           return { kind: "bin", op, left: c, right: code, type: BOOL, loc: e.loc };
         const negation: IrExpr = {
@@ -116,21 +410,43 @@ function rewriteFunction(fn: IrFunction): IrFunction {
         return { kind: "unary", op: "!", operand: negation, type: BOOL, loc: e.loc };
       }
     }
+    const optional = optionalUnitTest(e, operandTags);
+    if (optional) {
+      const operand = optional.operand;
+      if (operand.kind === "varRef") {
+        changed = true;
+        return optional.test(() => unitRef(operand.localId, e.loc));
+      }
+      // A read that keeps its temps declares them once: one unit read.
+      const read = optionalCharRead(operand)!;
+      if (read.plain || !optional.twice) {
+        changed = true;
+        return optional.test(() => unitRead(read, operand.loc));
+      }
+    }
     return mapExprChildren(e, rewriteExpr, rewriteStmt);
   };
   const rewriteStmt = (s: IrStmt): IrStmt => {
-    if (s.kind === "varDecl" && held.has(s.localId) && s.init?.kind === "strIntrinsic")
-      return {
-        ...s,
-        init: {
-          ...s.init,
-          method: "charCodeAt",
-          receiver: rewriteExpr(s.init.receiver),
-          args: s.init.args.map(rewriteExpr),
-          type: F64,
-        },
-      };
-    if (s.kind === "switch" && s.disc.kind === "varRef" && held.has(s.disc.localId))
+    if (s.kind === "varDecl" && held.has(s.localId) && s.init) {
+      changed = true;
+      const source = copyOf.get(s.localId);
+      if (source !== undefined) return { ...s, init: unitRef(source, s.init.loc) };
+      const read = reads.get(s.localId);
+      if (read) return { ...s, init: unitRead(read, s.init.loc) };
+      if (s.init.kind === "strIntrinsic")
+        return {
+          ...s,
+          init: {
+            ...s.init,
+            method: "charCodeAt",
+            receiver: rewriteExpr(s.init.receiver),
+            args: s.init.args.map(rewriteExpr),
+            type: F64,
+          },
+        };
+    }
+    if (s.kind === "switch" && s.disc.kind === "varRef" && held.has(s.disc.localId)) {
+      changed = true;
       return {
         ...s,
         disc: unitRef(s.disc.localId, s.disc.loc),
@@ -143,11 +459,14 @@ function rewriteFunction(fn: IrFunction): IrFunction {
           body: c.body.map(rewriteStmt),
         })),
       };
+    }
     return mapStmtChildren(s, rewriteExpr, rewriteStmt);
   };
+  const body = fn.body.map(rewriteStmt);
+  if (!changed) return fn;
   return {
     ...fn,
-    body: fn.body.map(rewriteStmt),
+    body,
     locals: fn.locals.map((l) => (held.has(l.id) ? unitLocal(l) : l)),
   };
 }
@@ -157,22 +476,37 @@ function unitLocal(local: IrLocal): IrLocal {
   return { ...local, type: F64 };
 }
 
-/** The local a node mentions in a form the rewrite handles: the
- * declaration from a one-argument charAt, a one-unit test, or a switch
- * over one-unit cases. */
-function understoodMention(node: Record<string, unknown>): string | null {
+/** The local a node mentions in a form the rewrite handles: a string
+ * local's declaration from a one-argument charAt, one-unit test, or switch
+ * over one-unit cases; a union local's declaration from an optional read,
+ * or its test against a one-unit literal or for undefined. */
+function understoodMention(
+  node: Record<string, unknown>,
+  locals: ReadonlyMap<string, IrLocal>,
+  reads: ReadonlyMap<string, OptionalCharRead>,
+  localTags: (operand: IrExpr) => OptionalCharRead | null,
+): string | null {
   const n = node as unknown as IrExpr | IrStmt;
-  if (n.kind === "varDecl") return n.init && isCharAt(n.init) ? n.localId : null;
+  if (n.kind === "varDecl") {
+    if (!n.init) return null;
+    const local = locals.get(n.localId);
+    if (local?.type.kind === "string") return isCharAt(n.init) ? n.localId : null;
+    const read = reads.get(n.localId);
+    return read && local?.type.kind === "union" && local.type.unionId === read.unionId
+      ? n.localId
+      : null;
+  }
   if (n.kind === "strEq" || n.kind === "strCmp") return unitUse(n)?.id ?? null;
   if (n.kind === "switch") {
-    if (n.disc.kind !== "varRef") return null;
+    if (n.disc.kind !== "varRef" || n.disc.type.kind !== "string") return null;
     if (!n.cases.some((c) => c.test !== null)) return null;
     const units = n.cases.every(
       (c) => c.test === null || (c.test.kind === "strLit" && c.test.value.length === 1),
     );
     return units ? n.disc.localId : null;
   }
-  return null;
+  const optional = optionalUnitTest(n as IrExpr, localTags);
+  return optional?.operand.kind === "varRef" ? optional.operand.localId : null;
 }
 
 function isCharAt(e: IrExpr): e is CharAt {

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7304-string-optional-char-reads.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/7304-string-optional-char-reads.ts"
+ ],
+ "diags": []
+}

tests/corpus/7304-string-optional-char-reads.ts

@@ -0,0 +1,121 @@
+// s[i] with an index not proven in bounds, tested against one-unit
+// literals: equality, inequality and ordered comparisons on either side,
+// peeks at i - 1 and i + 1, tests for undefined, switches, character
+// locals with and without other uses, fractional, negative, -0, NaN and
+// infinite indices, reads past the end, a two-unit literal, Object.is, a
+// class-based lexer with its position in a field, and ASCII, Latin-1, BMP,
+// surrogate pairs, lone surrogates and empty strings.
+
+class Lexer {
+  pos = 0;
+  src: string;
+  constructor(src: string) {
+    this.src = src;
+  }
+  skipSpaces(): void {
+    const src = this.src;
+    let pos = this.pos;
+    while (pos < src.length && (src[pos] === " " || src[pos] === "\t")) pos++;
+    this.pos = pos;
+  }
+  word(): string {
+    const src = this.src;
+    const start = this.pos;
+    let pos = start;
+    while (pos < src.length) {
+      const c = src[pos];
+      if ((c < "a" || c > "z") && (c < "0" || c > "9")) break;
+      pos++;
+    }
+    this.pos = pos;
+    return src.slice(start, pos);
+  }
+  operator(): string {
+    const src = this.src;
+    const c = src[this.pos];
+    switch (c) {
+      case "=":
+        this.pos += src[this.pos + 1] === "=" ? 2 : 1;
+        return src[this.pos - 1] === "=" && src[this.pos - 2] === "=" ? "==" : "=";
+      case "+":
+      case "-":
+        this.pos++;
+        return c;
+      case undefined:
+        return "";
+      default:
+        this.pos++;
+        return "?";
+    }
+  }
+  tokens(): string[] {
+    const out: string[] = [];
+    while (this.pos < this.src.length) {
+      this.skipSpaces();
+      const w = this.word();
+      if (w !== "") out.push(w);
+      else if (this.pos < this.src.length) out.push(this.operator());
+    }
+    return out;
+  }
+}
+
+function tests(src: string, pos: number): string {
+  const bits: number[] = [];
+  bits.push(src[pos] === " " ? 1 : 0, src[pos] !== "x" ? 1 : 0);
+  bits.push(src[pos] < "a" ? 1 : 0, src[pos] <= "a" ? 1 : 0);
+  bits.push(src[pos] > "a" ? 1 : 0, src[pos] >= "a" ? 1 : 0);
+  bits.push("m" < src[pos] ? 1 : 0, "m" >= src[pos] ? 1 : 0);
+  bits.push(src[pos + 1] === "b" ? 1 : 0, src[pos - 1] > "\0" ? 1 : 0);
+  bits.push(src[pos] === undefined ? 1 : 0, src[pos] !== undefined ? 1 : 0);
+  bits.push(src[pos * 2] === undefined ? 1 : 0, src[pos] === "ab" ? 1 : 0);
+  bits.push(Object.is(src[pos], "a") ? 1 : 0);
+  return bits.join("");
+}
+
+function local(src: string, pos: number): string {
+  const c = src[pos];
+  let n = 0;
+  if (c === undefined) n += 1;
+  if (c !== undefined && c >= "a") n += 2;
+  if ("z" > c) n += 4;
+  switch (c) {
+    case "é":
+      n += 8;
+      break;
+    case "b":
+      n += 16;
+      break;
+    default:
+      n += 32;
+  }
+  return String(n);
+}
+
+function kept(src: string, pos: number): string {
+  const c = src[pos];
+  if (c === " ") return "space";
+  return c ?? "end";
+}
+
+const samples = ["abc", "", " ", "a b\tc", "café", "日本 語", "😀x", "\uD800z", "ab=b"];
+const indices = [0, 1, 2, 3, -1, -0, 0.5, 1.5, NaN, Infinity, 10];
+for (const s of samples) {
+  const row: string[] = [];
+  for (const i of indices) row.push(`${tests(s, i)}:${local(s, i)}:${kept(s, i)}`);
+  console.log(JSON.stringify(s), row.join(" "));
+}
+
+const programs = [
+  "let x = a + b - c",
+  "  alpha==beta  gamma = delta1 ",
+  "x=y==z+",
+  "",
+  "abc é def",
+  "\tq\t=\t9",
+];
+for (const p of programs) console.log(JSON.stringify(p), JSON.stringify(new Lexer(p).tokens()));
+
+let text = "";
+for (let k = 0; k < 30; k++) text += (k % 4 === 0 ? "==" : "w" + k) + " ";
+console.log(new Lexer(text).tokens().length, new Lexer(text).tokens().slice(0, 6).join(","));

tests/harness/corpus-costs/7304-string-optional-char-reads.ts.json

@@ -0,0 +1 @@
+{ "plain": 2700, "san": 4200 }