Iterate strings over their bytes (#2233)

benchmarks/runtime/README.md

@@ -32,6 +32,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea
 | `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` |
 | `regex-replacers` | cpu | regex `replace`/`replaceAll` with inline callbacks whose parameters have no annotations: dictionary placeholders, snake_case to camelCase, date reordering, optional captures, capitalization, and a string search that reads the offset |
 | `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds |
+| `string-iteration` | cpu | code point iteration over text that mixes ASCII, Latin-1 letters, CJK and emoji: a for-of word tokenizer, spread reversal, and `Array.from` over words |
 | `template-render` | cpu | template literals, escaping, `map`/`join` |
 | `throw-chains` | cpu | exceptions caught 2, 5 and 20 recursive calls above the throw, through direct calls and through a callback, and a backtracking parser that throws on a failed alternative |
 | `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread |

benchmarks/runtime/workloads/string-iteration.json

@@ -0,0 +1,6 @@
+{
+  "entry": "workloads/string-iteration.ts",
+  "args": [
+    "1"
+  ]
+}

benchmarks/runtime/workloads/string-iteration.ts

@@ -0,0 +1,59 @@
+// Code point iteration over mixed text: a word tokenizer over for-of, a
+// character histogram, reversal through spread, and Array.from over
+// words. The text mixes ASCII, Latin-1 letters, CJK and emoji.
+let seed = 11;
+function random(): number {
+  seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
+  return seed / 4294967296;
+}
+const pieces = [
+  "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog",
+  "café", "naïve", "über", "señor", "straße", "déjà", "vu",
+  "日本", "語", "テキスト", "中文", "한국어",
+  "😀", "🚀", "👍", "🎉",
+];
+const punctuation = [" ", " ", " ", ", ", ". ", "! ", "\n"];
+const scale = Number(process.argv[2] ?? "1");
+const lines: string[] = [];
+for (let i = 0; i < Math.floor(4000 * scale); i++) {
+  let line = "";
+  const words = 6 + Math.floor(random() * 10);
+  for (let w = 0; w < words; w++) {
+    line += pieces[Math.floor(random() * pieces.length)]!;
+    line += punctuation[Math.floor(random() * punctuation.length)]!;
+  }
+  lines.push(line);
+}
+
+function isWordChar(c: string): boolean {
+  const code = c.codePointAt(0)!;
+  return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || code >= 0xc0;
+}
+
+let tokens = 0;
+let tokenUnits = 0;
+let reversedLength = 0;
+let fromLength = 0;
+const histogram = new Map<string, number>();
+for (let round = 0; round < 30; round++) {
+  for (const line of lines) {
+    let word = "";
+    for (const c of line) {
+      if (isWordChar(c)) {
+        word += c;
+        continue;
+      }
+      if (word !== "") {
+        tokens++;
+        tokenUnits += word.length;
+        if (round === 0) histogram.set(word, (histogram.get(word) ?? 0) + 1);
+        fromLength += Array.from(word).length;
+        word = "";
+      }
+    }
+    reversedLength += [...line].reverse().join("").length;
+  }
+}
+const top = [...histogram].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5);
+console.log(top.map(([word, count]) => `${word}:${count}`).join(" "));
+console.log("tokens", tokens, "units", tokenUnits, "reversed", reversedLength, "from", fromLength);

packages/compiler/src/backend/llvm/expr-containers.ts

@@ -50,11 +50,13 @@ import { emitFusedMapSet, rememberMapEntry } from "./map-entry-fusion.js";
 import { emitScratchStringSlice, emitStringSliceRead } from "./string-slices.js";
 import {
   STRING_CHAR_AT_DECL,
+  STRING_CP_AT_BYTE_DECL,
   STRING_CHAR_AT_INT_MARKER,
   STRING_CHAR_CODE_AT_DECL,
   STRING_CHAR_CODE_AT_INT_MARKER,
   emitAsciiChar,
   emitAsciiCharCode,
+  emitStringByteLength,
   emitUnitCharCode,
   STRING_UTF16_LEN_DECL,
 } from "./string-reads.js";
@@ -565,16 +567,30 @@ export function emitStrIntrinsic(
       return call("scr_str_is_well_formed", "zeroext i1 (ptr)", `ptr ${r.name}`, "i1", false);
     case "toWellFormed":
       return call("scr_str_to_well_formed", "ptr (ptr)", `ptr ${r.name}`, "ptr", true);
-    case "cpAt":
-      // The code point AT an index as a one-code-point string (+1) —
-      // the string-for-of desugar's read.
-      return call(
-        "scr_str_cp_at",
-        "ptr (ptr, double)",
-        `ptr ${r.name}, double ${args[0]!.name}`,
-        "ptr",
-        true,
-      );
+    case "byteLength": {
+      const t = B.tmp();
+      // A byte count fits in a signed size, whose conversion is one instruction.
+      B.line(`${t} = sitofp ${host.sizeType} ${emitStringByteLength(host, r.name)} to double`);
+      return { name: t, type: e.type };
+    }
+    case "cpAtByte": {
+      // sc_str_cp_at_byte (string-reads.ts) answers ASCII bytes from the
+      // charAt table inline. The cursor is a character start, an exact
+      // integer below the byte count.
+      host.declare(STRING_CHAR_AT_DECL);
+      host.declare(STRING_CP_AT_BYTE_DECL);
+      const index = e.args[0] ? exactInteger(host, args[0]!, e.args[0]) : null;
+      let wide: string;
+      if (index) {
+        wide = widenInteger(host, index);
+      } else {
+        wide = B.tmp();
+        B.line(`${wide} = fptosi double ${args[0]!.name} to i64`);
+      }
+      const t = B.tmp();
+      B.line(`${t} = call ptr @sc_str_cp_at_byte(ptr ${r.name}, i64 ${wide})`);
+      return host.own({ name: t, type: e.type });
+    }
     default: {
       const _exhaustive: never = method;
       void _exhaustive;

packages/compiler/src/backend/llvm/lib-shared.ts

@@ -104,6 +104,7 @@ export const LIB_FN_SYMS: Record<string, string> = {
   "str.localeCase": "scr_str_locale_case",
   "str.replaceLiteral": "scr_str_replace_literal",
   "str.replaceChars": "scr_str_replace_chars",
+  "str.codePoints": "scr_str_code_points",
   "num.fromString": "scr_string_to_number",
   "math.round": "scr_math_round",
   "math.sin": "sin",

packages/compiler/src/backend/llvm/string-lifetimes.test.ts

@@ -341,7 +341,7 @@ const methodCases: {
   { method: "normalize", args: [ref("argument")], result: STRING, target: "normalize" },
   { method: "isWellFormed", args: [], result: BOOL, target: "is_well_formed" },
   { method: "toWellFormed", args: [], result: STRING, target: "to_well_formed" },
-  { method: "cpAt", args: [num(1)], result: STRING, target: "cp_at" },
+  { method: "cpAtByte", args: [num(1)], result: STRING, target: "cp_at_byte", inline: true },
 ];
 
 test.each(methodCases)(

packages/compiler/src/backend/llvm/string-lifetimes.ts

@@ -30,7 +30,8 @@ export function borrowsStringInputs(method: IrStrIntrinsicMethod): boolean {
     case "normalize":
     case "isWellFormed":
     case "toWellFormed":
-    case "cpAt":
+    case "byteLength":
+    case "cpAtByte":
       return true;
     default:
       return false;

packages/compiler/src/backend/llvm/string-reads.ts

@@ -43,6 +43,10 @@ export const STRING_CHAR_CODE_AT_INT_MARKER = "; uses @sc_str_char_code_at_int";
 export const STRING_CHAR_AT_DECL = "declare ptr @scr_str_char_at(ptr, double)";
 /** Requests the integer-index charAt helper, as the marker above does. */
 export const STRING_CHAR_AT_INT_MARKER = "; uses @sc_str_char_at_int";
+/** The code-point character starting at a byte offset (the string
+ * iterator's step). Requested together with STRING_CHAR_AT_DECL, which
+ * defines the ASCII table. */
+export const STRING_CP_AT_BYTE_DECL = "declare ptr @scr_str_cp_at_byte(ptr, i64)";
 
 /** The proven-ASCII flag of string `s`, as an i1. */
 export function emitStringAsciiFlag(host: LlvmEmitterContext, s: string): string {
@@ -351,6 +355,29 @@ export const STRING_READ_HELPERS: readonly StringReadHelper[] = [
       ];
     },
   },
+  {
+    // The string iterator's step at byte `%b`, a character start below the
+    // byte count: an ASCII byte is the table entry for it; any other lead
+    // byte takes the runtime, which copies the character's bytes.
+    decl: STRING_CP_AT_BYTE_DECL,
+    define: (sz, attrs) => [
+      `define internal ptr @sc_str_cp_at_byte(ptr %s, i64 %b) ${attrs} {`,
+      `entry:`,
+      `  %data = getelementptr inbounds %ScrStr, ptr %s, i32 1`,
+      `  %p = getelementptr inbounds i8, ptr %data, i64 %b`,
+      `  %byte = load i8, ptr %p`,
+      `  %ascii = icmp sge i8 %byte, 0`,
+      `  br i1 %ascii, label %read, label %slow`,
+      `read:`,
+      `  %code = zext i8 %byte to i64`,
+      `  %char = getelementptr inbounds [128 x { ${sz}, ${sz}, ${sz}, [2 x i8] }], ptr @sc_ascii_chars, i64 0, i64 %code`,
+      `  ret ptr %char`,
+      `slow:`,
+      `  %r = call ptr @scr_str_cp_at_byte(ptr %s, i64 %b) nounwind willreturn`,
+      `  ret ptr %r`,
+      `}`,
+    ],
+  },
   {
     // An exact integer index, as for sc_str_char_code_at_int. Requested
     // together with STRING_CHAR_AT_DECL, which defines the table.

packages/compiler/src/frontend/lowering/containers/array-construction.ts

@@ -1081,9 +1081,7 @@ export function lowerArrayFromCall(
   if ((args.length === 1 || args.length === 2) && !ts.isObjectLiteralExpression(args[0]!)) {
     const source = lowerer.lowerExpr(args[0]!);
     let src =
-      source.type.kind === "string" && args.length === 2
-        ? strCharsCall(lowerer, source, loc)
-        : source;
+      source.type.kind === "string" && args.length === 2 ? strCharsCall(source, loc) : source;
     if (src.type.kind === "array") {
       const arrT = src.type;
       if (args.length === 1) {
@@ -1171,7 +1169,7 @@ export function lowerArrayFromCall(
     }
     // String iteration advances by code point; the same helper serves
     // `[...s]` and both Array.from(s) forms.
-    if (src.type.kind === "string" && args.length === 1) return strCharsCall(lowerer, src, loc);
+    if (src.type.kind === "string" && args.length === 1) return strCharsCall(src, loc);
     if (src.type.kind === "dyn" && args.length === 1) {
       return lowerCheckedArrayFrom(lowerer, src, loc);
     }
@@ -1318,112 +1316,9 @@ function buildArrayFromArrayFn(
 }
 
 /** `Array.from(s)` / `[...s]` on a STRING: the code-point split into a
- * fresh string[], through one interned helper per module. */
-export function strCharsCall(lowerer: Lowerer, src: IrExpr, loc: SrcLoc): IrExpr {
-  const key = "strChars";
-  let helper = lowerer.arrHofHelpers.get(key);
-  if (!helper) {
-    helper = "%str.chars";
-    lowerer.arrHofHelpers.set(key, helper);
-    lowerer.liftedFns.push(buildStrCharsFn(helper, loc));
-  }
-  return { kind: "call", callee: helper, args: [src], type: arrayOf(STRING), loc };
-}
-
-/** The code-point split, from existing IR nodes — the string for-of
- * desugar's UTF-16 cursor as a function:
- *
- *   out = [];
- *   i = 0;
- *   while (i < s.length) { ch = cpAt(s, i); i += ch.length; out.push(ch); }
- *   return out;
- */
-function buildStrCharsFn(name: string, loc: SrcLoc): IrFunction {
-  const outT = arrayOf(STRING);
-
-  const sLen = (recv: IrExpr): IrExpr => ({
-    kind: "strIntrinsic",
-    method: "length",
-    receiver: recv,
-    args: [],
-    type: F64,
-    loc,
-  });
-  const body: IrStmt[] = [
-    {
-      kind: "varDecl",
-      localId: "out.0",
-      init: { kind: "arrayLit", elems: [], type: outT, loc },
-      loc,
-    },
-    { kind: "varDecl", localId: "i.0", init: { kind: "numLit", value: 0, type: F64, loc }, loc },
-    {
-      kind: "while",
-      cond: {
-        kind: "bin",
-        op: "<",
-        left: varRef("i.0", F64, loc),
-        right: sLen(varRef("s.0", STRING, loc)),
-        type: BOOL,
-        loc,
-      },
-      body: [
-        {
-          kind: "varDecl",
-          localId: "ch.0",
-          init: {
-            kind: "strIntrinsic",
-            method: "cpAt",
-            receiver: varRef("s.0", STRING, loc),
-            args: [varRef("i.0", F64, loc)],
-            type: STRING,
-            loc,
-          },
-          loc,
-        },
-        {
-          kind: "assign",
-          localId: "i.0",
-          value: {
-            kind: "bin",
-            op: "+",
-            left: varRef("i.0", F64, loc),
-            right: sLen(varRef("ch.0", STRING, loc)),
-            type: F64,
-            loc,
-          },
-          loc,
-        },
-        {
-          kind: "exprStmt",
-          expr: {
-            kind: "arrIntrinsic",
-            method: "push",
-            receiver: varRef("out.0", outT, loc),
-            args: [varRef("ch.0", STRING, loc)],
-            type: F64,
-            loc,
-          },
-          loc,
-        },
-      ],
-      loc,
-    },
-    { kind: "return", value: varRef("out.0", outT, loc), loc },
-  ];
-  return {
-    name,
-    params: [{ localId: "s.0", name: "s", type: STRING }],
-    returnType: outT,
-    locals: [
-      { id: "s.0", name: "s", type: STRING, mutable: true },
-      { id: "out.0", name: "out", type: outT, mutable: false },
-      { id: "i.0", name: "i", type: F64, mutable: true },
-      { id: "ch.0", name: "ch", type: STRING, mutable: false },
-    ],
-    body,
-    loc,
-  };
+ * fresh string[] (scr_str_code_points). */
+export function strCharsCall(src: IrExpr, loc: SrcLoc): IrExpr {
+  return { kind: "libCall", fn: "str.codePoints", args: [src], type: arrayOf(STRING), loc };
 }
 
 /** The lowered `length` value of the one-property source literal, or null

packages/compiler/src/frontend/lowering/containers/array-pipelines.ts

@@ -157,7 +157,8 @@ const PURE_STRING = new Set<string>([
   "toUpperCase",
   "isWellFormed",
   "toWellFormed",
-  "cpAt",
+  "byteLength",
+  "cpAtByte",
 ]);
 
 function primitive(t: IrType): boolean {

packages/compiler/src/frontend/lowering/containers/collection-methods.ts

@@ -1135,7 +1135,7 @@ function setFromSeedValue(
   if (source.type.kind === "set" && typeEquals(source.type, setT)) {
     return { kind: "setIntrinsic", method: "clone", receiver: source, args: [], type: setT, loc };
   } else if (source.type.kind === "string" && setT.elem.kind === "string") {
-    seed = strCharsCall(lowerer, source, loc);
+    seed = strCharsCall(source, loc);
   } else if (source.type.kind === "array" && typeEquals(source.type.elem, setT.elem)) {
     seed = source;
   } else if (

packages/compiler/src/frontend/lowering/expressions/array-literals.ts

@@ -504,7 +504,7 @@ export function lowerArrayLiteral(
       // interned helper as Array.from(s); the result rides the array
       // machinery below like any string[] source.
       if (src.type.kind === "string") {
-        src = strCharsCall(lowerer, src, locOf(el));
+        src = strCharsCall(src, locOf(el));
       }
       // `[...typedArray]`: represented typed arrays are dense numeric
       // iterables, so drain their elements into the fresh number[] that

packages/compiler/src/frontend/lowering/generator-delegation.ts

@@ -577,10 +577,11 @@ export function lowerBuiltinDelegation(
       loc,
     });
   } else if (sourceT.kind === "string") {
+    // A byte cursor over the WTF-8 data, as for-of over a string walks.
     const ch = lowerer.declareHiddenLocal("%delegateElement", STRING);
     const length = (receiver: IrExpr): IrExpr => ({
       kind: "strIntrinsic",
-      method: "length",
+      method: "byteLength",
       receiver,
       args: [],
       type: F64,
@@ -595,7 +596,7 @@ export function lowerBuiltinDelegation(
           localId: ch.id,
           init: {
             kind: "strIntrinsic",
-            method: "cpAt",
+            method: "cpAtByte",
             receiver: src(),
             args: [at()],
             type: STRING,

packages/compiler/src/frontend/lowering/iterator-adapters.ts

@@ -743,7 +743,7 @@ export function lowerStringIteratorValue(
             left: index,
             right: {
               kind: "strIntrinsic",
-              method: "length",
+              method: "byteLength",
               receiver: text,
               args: [],
               type: F64,
@@ -758,7 +758,7 @@ export function lowerStringIteratorValue(
               localId: "point.0",
               init: {
                 kind: "strIntrinsic",
-                method: "cpAt",
+                method: "cpAtByte",
                 receiver: text,
                 args: [index],
                 type: STRING,
@@ -775,7 +775,7 @@ export function lowerStringIteratorValue(
                 left: index,
                 right: {
                   kind: "strIntrinsic",
-                  method: "length",
+                  method: "byteLength",
                   receiver: point,
                   args: [],
                   type: F64,

packages/compiler/src/frontend/lowering/lower-stmts.ts

@@ -2668,7 +2668,7 @@ export function lowerBindingPattern(
           lowerer.unsupported("SC1031", el, "rest elements over a union of array or tuple layouts");
         const loc = locOf(el);
         let value = unionArrayValueRead(lowerer, srcRef(), numLit(i, loc), loc, (text) =>
-          strCharsCall(lowerer, text, loc),
+          strCharsCall(text, loc),
         );
         if (!value)
           lowerer.unsupported(
@@ -2798,7 +2798,7 @@ export function lowerBindingPattern(
     }
     // STRING sources (`const [a, b] = str`): array destructuring walks
     // the STRING ITERATOR — code points, astral characters whole —
-    // which is Array.from(string)'s own machinery (%str.chars), so the
+    // which is Array.from(string)'s own machinery (str.codePoints), so the
     // source splits once into a hidden chars array and the pattern
     // lowers as an array pattern over string[]. Positions, holes,
     // defaults (with the bounds test), rest (the remaining code points
@@ -2817,7 +2817,7 @@ export function lowerBindingPattern(
       out.push({
         kind: "varDecl",
         localId: chars.id,
-        init: strCharsCall(lowerer, srcRef(), loc),
+        init: strCharsCall(srcRef(), loc),
         loc,
       });
       lowerer.lowerBindingPattern(
@@ -9826,7 +9826,7 @@ function lowerArrayAssignInto(
   let srcType = tmpRef().type;
   // A STRING source (`[a, b] = s`): the declaration path's rule in
   // assignment position — the string iterator's code-point split
-  // (%str.chars, astral characters whole) into a hidden chars array,
+  // (str.codePoints, astral characters whole) into a hidden chars array,
   // then the pattern proceeds as an array pattern over string[]
   // (defaults with the bounds test, rest packing the remaining code
   // points, elisions; positions past the last code point inherit the
@@ -9839,7 +9839,7 @@ function lowerArrayAssignInto(
     out.push({
       kind: "varDecl",
       localId: chars.id,
-      init: strCharsCall(lowerer, tmpRef(), loc),
+      init: strCharsCall(tmpRef(), loc),
       loc,
     });
     tmpRef = () => ({ kind: "varRef", localId: chars.id, type: charsT, loc });
@@ -11383,16 +11383,18 @@ function lowerForOfMatchAll(
 /** `for (const ch of s)` over a STRING: JS's string iterator, which walks
  * code POINTS — each pass binds the whole character (a two-unit string
  * for astral chars, where charAt/slice would return one surrogate half).
- * Desugars over the UTF-16 cursor machinery: a hidden unit
- * index reads the character with the cpAt intrinsic and advances by ITS
- * length before the body runs (so `continue` never skips the advance):
+ * Desugars over a byte cursor into the WTF-8 data: a hidden byte offset
+ * reads the character that starts there with the cpAtByte intrinsic and
+ * advances by ITS byte count before the body runs (so `continue` never
+ * skips the advance):
  *
  *   { const %s = <expr>; let %i = 0;
- *     while (%i < %s.length) { const ch = cpAt(%s, %i); %i += ch.length; <body> } }
+ *     while (%i < byteLength(%s)) { const ch = cpAtByte(%s, %i); %i += byteLength(ch); <body> } }
  *
- * Strings are immutable, so the length is stable and there is no
+ * Strings are immutable, so the byte count is stable and there is no
  * iterator state to unwind on throw; a lone surrogate is a one-unit
- * character, so every step lands on a character start. */
+ * character with its own three-byte encoding, so every step lands on a
+ * character start. */
 /** for-of over a CLASS ITERABLE (classIteratorOf's shape) — the
  * iterator protocol as ordinary method calls:
  *
@@ -12087,7 +12089,7 @@ function lowerForOfString(
   try {
     const s = lowerer.declareHiddenLocal("%strof", STRING);
     const i = lowerer.declareHiddenLocal("%iterof", F64);
-    i.mutable = true; // the unit index reassigns (hidden locals default const)
+    i.mutable = true; // the byte cursor reassigns (hidden locals default const)
     const sRef = (): IrExpr => ({ kind: "varRef", localId: s.id, type: STRING, loc });
     const iRef = (): IrExpr => ({ kind: "varRef", localId: i.id, type: F64, loc });
     // `for (var c of str)`: the character mechanics stay on a hidden
@@ -12118,7 +12120,7 @@ function lowerForOfString(
         localId: ch.id,
         init: {
           kind: "strIntrinsic",
-          method: "cpAt",
+          method: "cpAtByte",
           receiver: sRef(),
           args: [iRef()],
           type: STRING,
@@ -12145,7 +12147,7 @@ function lowerForOfString(
           left: iRef(),
           right: {
             kind: "strIntrinsic",
-            method: "length",
+            method: "byteLength",
             receiver: chRef,
             args: [],
             type: F64,
@@ -12177,7 +12179,7 @@ function lowerForOfString(
             left: iRef(),
             right: {
               kind: "strIntrinsic",
-              method: "length",
+              method: "byteLength",
               receiver: sRef(),
               args: [],
               type: F64,

packages/compiler/src/ir/builtin-signatures.ts

@@ -848,6 +848,7 @@ function buildLibFnSignatures() {
     // (subject, characters, NUL-separated replacement texts): the fused
     // chain of global one-character replacements.
     "str.replaceChars": { argTypes: [STRING, STRING, STRING], result: STRING },
+    "str.codePoints": { argTypes: [STRING], result: arrayOf(STRING) },
     /* ToNumber(string) — ECMA-262 7.1.4.1 StringToNumber (scr_string.c):
      * trim the JS StrWhiteSpace set, empty/whitespace-only → +0, then the
      * whole span must be one StrNumericLiteral — signed decimal (Infinity

packages/compiler/src/ir/integer-ranges.ts

@@ -867,6 +867,8 @@ export function analyzeIntegerRangeFacts(
         );
         if (e.type.kind === "f64") {
           if (e.method === "length") range = { min: 0, max: STRING_MAX_UNITS };
+          // A string holds at most SCR_STR_MAX_CAP (0x7fffffff) bytes.
+          else if (e.method === "byteLength") range = { min: 0, max: STRING_MAX_UNITS };
           // An empty needle is found at the clamped position, up to the length.
           else if (e.method === "indexOf") range = { min: -1, max: STRING_MAX_UNITS };
         }

packages/compiler/src/ir/intrinsic-signatures.ts

@@ -31,7 +31,8 @@ export const STR_INTRINSIC_SIGS: Record<
   normalize: { argTypes: [STRING], minArgs: 1, result: STRING },
   isWellFormed: { argTypes: [], minArgs: 0, result: BOOL },
   toWellFormed: { argTypes: [], minArgs: 0, result: STRING },
-  cpAt: { argTypes: [F64], minArgs: 1, result: STRING },
+  byteLength: { argTypes: [], minArgs: 0, result: F64 },
+  cpAtByte: { argTypes: [F64], minArgs: 1, result: STRING },
 };
 
 /** Per-method signature for regexIntrinsic. test/source/flags take the

packages/compiler/src/ir/ir.ts

@@ -2451,10 +2451,13 @@ export type IrStrIntrinsicMethod =
   // surrogate, and the copy with each one replaced by U+FFFD.
   | "isWellFormed"
   | "toWellFormed"
-  // The string iterator's step: the full code-POINT character at a UTF-16
-  // index (astral chars come back whole where charAt truncates) — for-of
-  // over strings desugars onto it, advancing by the result's length.
-  | "cpAt";
+  // The string iterator over the WTF-8 bytes: the byte count of a string,
+  // and the code-point character that starts at a byte offset (a lone
+  // surrogate is a one-unit character). for-of, spread and Array.from of a
+  // string walk a byte cursor with them, advancing by the character's
+  // byte count; the cursor is always a character start below the count.
+  | "byteLength"
+  | "cpAtByte";
 
 /** The typed-array/Buffer method surface (bytesIntrinsic). Receiver/arg
  * conventions (validated): `length`/`byteLength` are property reads → f64;

packages/compiler/test/string-iteration-emission.test.ts

@@ -0,0 +1,47 @@
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+import { compile } from "../src/index.js";
+
+const directories: string[] = [];
+afterEach(async () => {
+  await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+async function emit(source: string): Promise<string> {
+  const outDir = await mkdtemp(join(tmpdir(), "scriptc-string-iteration-"));
+  directories.push(outDir);
+  const entry = join(outDir, "main.ts");
+  await writeFile(entry, source);
+  const outPath = join(outDir, "program.ll");
+  const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" });
+  expect(result.ok, JSON.stringify(result)).toBe(true);
+  return readFile(outPath, "utf8");
+}
+
+const calls = (llvm: string, symbol: string): string[] =>
+  llvm.split("\n").filter((line) => line.includes("call ") && line.includes(`@${symbol}(`));
+
+test("for-of over a string walks a byte cursor", async () => {
+  const llvm = await emit(`
+function count(s: string): number {
+  let n = 0;
+  for (const c of s) n += c.length;
+  return n;
+}
+console.log(count(process.argv[1] ?? ""));
+`);
+  expect(calls(llvm, "sc_str_cp_at_byte")).toHaveLength(1);
+  expect(calls(llvm, "scr_str_cp_at")).toHaveLength(0);
+  // The helper answers ASCII bytes from the one-character table.
+  expect(llvm).toContain("define internal ptr @sc_str_cp_at_byte(ptr %s, i64 %b)");
+});
+
+test("spread and Array.from of a string build the array in one call", async () => {
+  const llvm = await emit(`
+const s = process.argv[1] ?? "";
+console.log([...s].length, Array.from(s).length);
+`);
+  expect(calls(llvm, "scr_str_code_points")).toHaveLength(2);
+});

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8347-string-iteration-code-points.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8347-string-iteration-code-points.ts"
+ ],
+ "diags": []
+}

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8348-string-iteration-worker-characters.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8348-string-iteration-worker-characters.ts"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_string.c

@@ -3514,6 +3514,8 @@ ScrStr *scr_bool_to_scrstr(bool b) {
  * decode is a whole char; an i addressing the LOW half of an astral pair
  * (unreachable from the iterator, which only lands on char starts) answers
  * the containing char. Out of range → empty string. Borrows s; +1. */
+static ScrStr *scr_str_char_of(const char *bytes, size_t adv);
+
 ScrStr *scr_str_cp_at(ScrStr *s, double i) {
   double idx = scr_to_integer_or_infinity(i);
   if (!(idx >= 0)) return scr_str_empty();
@@ -3528,7 +3530,80 @@ ScrStr *scr_str_cp_at(ScrStr *s, double i) {
   if (off >= s->len) return scr_str_empty(); /* out of range */
   size_t adv;
   (void)scr_utf8_decode(s->data + off, &adv);
-  return scr_str_from_span(s->data + off, adv);
+  return scr_str_char_of(s->data + off, adv);
+}
+
+/* The one-character strings of U+0080..U+07FF (two-byte sequences, lead
+ * bytes 0xC2..0xDF), made immortal on first use and shared by every
+ * thread, as the ASCII ones are static: the string iterator returns them
+ * instead of a new string per character. A string is immortal before its
+ * slot publishes it; a thread that loses the race to fill a slot keeps its
+ * own immortal copy, so at most one extra string per slot and thread. The
+ * strings come from the system allocator, as other state that outlives a
+ * thread does: a worker's own memory goes back when it exits without
+ * sharing (scr_alloc.c), and the table outlives the worker that filled a
+ * slot. */
+static ScrStr *scr_two_byte_chars[0x800 - 0x80];
+
+static ScrStr *scr_two_byte_char(const char *bytes) {
+  uint32_t cp = ((uint32_t)((unsigned char)bytes[0] & 0x1F) << 6) | ((unsigned char)bytes[1] & 0x3F);
+  ScrStr **slot = &scr_two_byte_chars[cp - 0x80];
+  ScrStr *known = __atomic_load_n(slot, __ATOMIC_ACQUIRE);
+  if (known) return known;
+  ScrStr *made = calloc(1, sizeof(ScrStr) + 3);
+  if (!made) scr_oom();
+  made->len = 2;
+  made->cap = 2;
+  memcpy(made->data, bytes, 2);
+  scr_str_hash_prime(made);
+  made->rc = SIZE_MAX;
+  scr_lsan_immortal(made);
+  ScrStr *expected = NULL;
+  if (__atomic_compare_exchange_n(slot, &expected, made, false, __ATOMIC_ACQ_REL,
+                                  __ATOMIC_ACQUIRE))
+    return made;
+  return expected;
+}
+
+/* The character whose encoding starts at bytes and is adv bytes long (+1,
+ * or immortal). */
+static ScrStr *scr_str_char_of(const char *bytes, size_t adv) {
+  unsigned char lead = (unsigned char)bytes[0];
+  if (adv == 1 && lead < 0x80) return (ScrStr *)&scr_ascii1[lead];
+  if (adv == 2) return scr_two_byte_char(bytes);
+  return scr_str_from_span(bytes, adv);
+}
+
+ScrStr *scr_str_cp_at_byte(ScrStr *s, uint64_t b) {
+  if (b >= s->len) return scr_str_empty();
+  size_t adv;
+  (void)scr_utf8_decode(s->data + b, &adv);
+  return scr_str_char_of(s->data + b, adv);
+}
+
+ScrArr *scr_str_code_points(ScrStr *s) {
+  /* Every byte but a continuation byte starts one character; a lone
+   * surrogate's three bytes are one character. */
+  size_t count = s->len;
+  if (!s->ascii)
+    for (size_t b = 0; b < s->len; b++) count -= ((unsigned char)s->data[b] & 0xC0) == 0x80;
+  ScrArr *out = scr_arr_new_inline(SCR_ELEM_STR, count, false);
+  size_t k = 0;
+  for (size_t b = 0; b < s->len;) {
+    unsigned char c = (unsigned char)s->data[b];
+    if (c < 0x80) {
+      out->data[k++] = (uint64_t)(uintptr_t)&scr_ascii1[c];
+      b++;
+      continue;
+    }
+    size_t adv;
+    (void)scr_utf8_decode(s->data + b, &adv);
+    out->data[k++] = (uint64_t)(uintptr_t)scr_str_char_of(s->data + b, adv);
+    b += adv;
+  }
+  out->len = (uint32_t)count;
+  out->filled = (uint32_t)count;
+  return out;
 }
 
 /* ── the URI component codecs (encodeURIComponent/decodeURIComponent) ──

packages/runtime/src/scr_string_methods.h

@@ -150,6 +150,15 @@ ScrStr *scr_str_char_at(ScrStr *s, double i);
  * (two units); the consuming loop advances by the result's length.
  * Out of range → empty string. Returns +1. */
 ScrStr *scr_str_cp_at(ScrStr *s, double i);
+/* The string iterator's step over the WTF-8 bytes: the code-point
+ * character whose encoding starts at byte offset b, a character start
+ * below s->len (a lone surrogate is its own three-byte character). The
+ * compiler answers ASCII bytes inline and calls this for the rest.
+ * Returns +1. */
+ScrStr *scr_str_cp_at_byte(ScrStr *s, uint64_t b);
+/* [...s] and Array.from(s): the code-point characters of s in order, as a
+ * fresh string array of exactly that length. Returns +1. */
+struct ScrArr *scr_str_code_points(ScrStr *s);
 
 /* trimStart()/trimEnd(): the one-sided halves of trim. Returns +1. */
 ScrStr *scr_str_trim_start(ScrStr *s);

tests/corpus/1561-forof-strings.ts

@@ -1,7 +1,7 @@
 // for-of over STRINGS: JS's string iterator walks code POINTS — each pass
 // binds the whole character, so astral chars arrive as one two-unit string
-// (where charAt/slice would truncate the halves). Desugars over the UTF-16
-// cursor with the cpAt step intrinsic.
+// (where charAt/slice would truncate the halves). Desugars over a byte
+// cursor with the cpAtByte step intrinsic.
 for (const ch of "héllo 🎉x") console.log(ch, ch.length);
 
 // The empty string iterates zero times.

tests/corpus/8347-string-iteration-code-points.ts

@@ -0,0 +1,96 @@
+// The string iterator: for-of, spread, Array.from, yield* and
+// [Symbol.iterator]() walk code points. Astral characters come back whole,
+// a lone surrogate is a one-unit character, and halves joined by
+// concatenation form one character, as in Node.
+function unitCodes(c: string): string {
+  const codes: string[] = [];
+  for (let k = 0; k < c.length; k++) codes.push(c.charCodeAt(k).toString(16));
+  return `${c.length}:${codes.join(".")}`;
+}
+const show = (chars: string[]): string => chars.map(unitCodes).join(" ");
+
+const samples = [
+  "",
+  "abc",
+  "aé😀b",
+  "\u0000\u007f\u0080߿ࠀ￿",
+  "\u{10000}\u{10ffff}",
+  "é ñ",
+  "\ud800",
+  "\udc00",
+  "x\ud83d",
+  "\ude00x",
+  "\udc00\ud800",
+  "\ud83d" + "\ude00",
+  "a\ud83d" + "\ude00b\ud83d",
+  "日本語テキスト",
+];
+for (const s of samples) {
+  const looped: string[] = [];
+  for (const c of s) looped.push(c);
+  console.log(JSON.stringify(s), show(looped), show([...s]), show(Array.from(s)), [...s].length, s.length);
+}
+
+// Loop control, heads and bindings.
+const text = "a😀b\ud800c";
+let out = "";
+outer: for (const c of text) {
+  if (c === "b") continue;
+  for (const d of "xy") {
+    if (d === "y" && c === "c") continue outer;
+    out += c + d;
+  }
+  if (c === "\ud800") out += "|";
+}
+console.log(out);
+for (var v of "hé") out += v;
+console.log(out);
+let last = "";
+for (last of "q😀") out += last.length;
+console.log(out, last);
+for (const [first] of ["😀z", "ab"].join("")) out += first;
+console.log(out);
+let source = "abc";
+for (const c of source) {
+  source += c;
+  out += source.length;
+}
+console.log(out, source);
+let n = 0;
+for (const c of "stop😀here") {
+  if (c === "😀") break;
+  n += c.length;
+}
+console.log(n);
+
+// Characters as values: lengths, Map keys and code points.
+const counts = new Map<string, number>();
+for (const c of "abracadabra😀😀é") counts.set(c, (counts.get(c) ?? 0) + 1);
+console.log([...counts].map(([k, v]) => `${k}=${v}`).join(","));
+let points = 0;
+for (const c of "😀é\ud800a") points += c.codePointAt(0)!;
+console.log(points);
+
+// Generators and explicit iterators.
+function* letters(s: string): Generator<string, void, undefined> {
+  yield* s;
+}
+console.log(show([...letters("x😀\udc00y")]));
+const it = "o😀"[Symbol.iterator]();
+let step = it.next();
+while (!step.done) {
+  out += step.value.length;
+  step = it.next();
+}
+console.log(out);
+
+// A long string built from mixed pieces.
+let long = "";
+for (let i = 0; i < 2000; i++) long += i % 3 === 0 ? "a" : i % 3 === 1 ? "é" : "😀";
+let units = 0;
+let chars = 0;
+for (const c of long) {
+  units += c.length;
+  chars++;
+}
+console.log(units, chars, long.length, [...long].length);

tests/corpus/8348-string-iteration-worker-characters.ts

@@ -0,0 +1,44 @@
+import { Worker, isMainThread, parentPort } from "node:worker_threads";
+
+// The one-character strings of U+0080..U+07FF that string iteration returns
+// are shared by every thread. A worker that is the first to iterate them
+// and exits without sharing anything else leaves them in use: the main
+// thread iterates the same characters after the worker's memory goes back.
+
+const text = "aé ñ ß ü Ω ж א ع ߿";
+
+function codes(s: string): string {
+  const out: string[] = [];
+  for (const c of s) out.push(`${c.length}:${c.charCodeAt(0).toString(16)}`);
+  return out.join(" ");
+}
+
+function runWorker(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(new URL(import.meta.url));
+    let count = -1;
+    worker.on("message", (value: number) => {
+      count = value;
+    });
+    worker.on("error", reject);
+    worker.on("exit", () => resolve(count));
+  });
+}
+
+async function main(): Promise<void> {
+  console.log("worker", await runWorker());
+  // The worker's memory goes back shortly after it exits.
+  await new Promise((resolve) => setTimeout(resolve, 300));
+  console.log("main", codes(text));
+  console.log("spread", [...text].join("|"));
+  console.log("from", Array.from("ñandú").map((c) => c.length).join(","));
+}
+
+if (isMainThread) {
+  void main();
+} else {
+  let count = 0;
+  for (const c of text) count += c.length;
+  count += [...text].length + Array.from("ñandú").length;
+  parentPort!.postMessage(count);
+}

tests/harness/corpus-costs/8347-string-iteration-code-points.ts.json

@@ -0,0 +1 @@
+{ "plain": 2770, "san": 3750 }

tests/harness/corpus-costs/8348-string-iteration-worker-characters.ts.json

@@ -0,0 +1 @@
+{ "plain": 3960, "san": 3900 }

tests/harness/string-iteration-workers.test.ts

@@ -0,0 +1,44 @@
+/* The one-character strings that string iteration shares between threads
+ * outlive the worker that first made them. On Linux a worker that exits
+ * without sharing anything returns its memory; with SCR_SA_DISCARD=poison
+ * that memory becomes inaccessible instead, so a cached character
+ * allocated from the worker's memory would fault when the main thread
+ * reads it. The corpus program pins Node.js parity; this suite runs it
+ * with the discard poisoned and immediate. Under SCRIPTC_SAN=1 the runtime
+ * keeps every worker's memory, so the check does not apply. */
+import { execFile } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { expect, test } from "vitest";
+import { compile } from "@scriptc/compiler";
+
+const exec = promisify(execFile);
+const program = join(import.meta.dirname, "../corpus/8348-string-iteration-worker-characters.ts");
+const sanitize = process.env["SCRIPTC_SAN"] === "1";
+
+test.skipIf(process.platform !== "linux" || sanitize)(
+  "characters a discarded worker cached stay readable",
+  { timeout: 240_000 },
+  async () => {
+    const outDir = await mkdtemp(join(tmpdir(), "scriptc-string-iteration-workers-"));
+    try {
+      const result = await compile(program, {
+        backend: "llvm",
+        outDir,
+        outPath: join(outDir, "program"),
+      });
+      expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true);
+      if (!result.ok) return;
+      const { stdout } = await exec(result.binaryPath, [], {
+        env: { ...process.env, SCR_SA_DISCARD: "poison", SCR_SA_DISCARD_DELAY_MS: "0" },
+        timeout: 120_000,
+      });
+      expect(stdout).toContain("main 1:61 1:e9 1:20 1:f1");
+      expect(stdout).toContain("spread a|é| |ñ");
+    } finally {
+      await rm(outDir, { recursive: true, force: true });
+    }
+  },
+);