diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index af3d3d239..0157503fd 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -32,6 +32,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | | `regex-replacers` | cpu | regex `replace`/`replaceAll` with inline callbacks whose parameters have no annotations: dictionary placeholders, snake_case to camelCase, date reordering, optional captures, capitalization, and a string search that reads the offset | | `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | +| `string-iteration` | cpu | code point iteration over text that mixes ASCII, Latin-1 letters, CJK and emoji: a for-of word tokenizer, spread reversal, and `Array.from` over words | | `template-render` | cpu | template literals, escaping, `map`/`join` | | `throw-chains` | cpu | exceptions caught 2, 5 and 20 recursive calls above the throw, through direct calls and through a callback, and a backtracking parser that throws on a failed alternative | | `thread-slots`, `thread-slots-cache` | cpu | worker threads reading one published type graph from an `InternTable` and keeping per-thread state for each type they visit: a `ThreadSlots` family, or a direct-mapped cache by id in front of a `Map` by id (the `-cache` entry); under Node.js both are per thread | diff --git a/benchmarks/runtime/workloads/string-iteration.json b/benchmarks/runtime/workloads/string-iteration.json new file mode 100644 index 000000000..21456bd23 --- /dev/null +++ b/benchmarks/runtime/workloads/string-iteration.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/string-iteration.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/string-iteration.ts b/benchmarks/runtime/workloads/string-iteration.ts new file mode 100644 index 000000000..2abebd635 --- /dev/null +++ b/benchmarks/runtime/workloads/string-iteration.ts @@ -0,0 +1,59 @@ +// Code point iteration over mixed text: a word tokenizer over for-of, a +// character histogram, reversal through spread, and Array.from over +// words. The text mixes ASCII, Latin-1 letters, CJK and emoji. +let seed = 11; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const pieces = [ + "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", + "café", "naïve", "über", "señor", "straße", "déjà", "vu", + "日本", "語", "テキスト", "中文", "한국어", + "😀", "🚀", "👍", "🎉", +]; +const punctuation = [" ", " ", " ", ", ", ". ", "! ", "\n"]; +const scale = Number(process.argv[2] ?? "1"); +const lines: string[] = []; +for (let i = 0; i < Math.floor(4000 * scale); i++) { + let line = ""; + const words = 6 + Math.floor(random() * 10); + for (let w = 0; w < words; w++) { + line += pieces[Math.floor(random() * pieces.length)]!; + line += punctuation[Math.floor(random() * punctuation.length)]!; + } + lines.push(line); +} + +function isWordChar(c: string): boolean { + const code = c.codePointAt(0)!; + return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || code >= 0xc0; +} + +let tokens = 0; +let tokenUnits = 0; +let reversedLength = 0; +let fromLength = 0; +const histogram = new Map(); +for (let round = 0; round < 30; round++) { + for (const line of lines) { + let word = ""; + for (const c of line) { + if (isWordChar(c)) { + word += c; + continue; + } + if (word !== "") { + tokens++; + tokenUnits += word.length; + if (round === 0) histogram.set(word, (histogram.get(word) ?? 0) + 1); + fromLength += Array.from(word).length; + word = ""; + } + } + reversedLength += [...line].reverse().join("").length; + } +} +const top = [...histogram].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5); +console.log(top.map(([word, count]) => `${word}:${count}`).join(" ")); +console.log("tokens", tokens, "units", tokenUnits, "reversed", reversedLength, "from", fromLength); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 46c206f76..696dad3ab 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -50,11 +50,13 @@ import { emitFusedMapSet, rememberMapEntry } from "./map-entry-fusion.js"; import { emitScratchStringSlice, emitStringSliceRead } from "./string-slices.js"; import { STRING_CHAR_AT_DECL, + STRING_CP_AT_BYTE_DECL, STRING_CHAR_AT_INT_MARKER, STRING_CHAR_CODE_AT_DECL, STRING_CHAR_CODE_AT_INT_MARKER, emitAsciiChar, emitAsciiCharCode, + emitStringByteLength, emitUnitCharCode, STRING_UTF16_LEN_DECL, } from "./string-reads.js"; @@ -565,16 +567,30 @@ export function emitStrIntrinsic( return call("scr_str_is_well_formed", "zeroext i1 (ptr)", `ptr ${r.name}`, "i1", false); case "toWellFormed": return call("scr_str_to_well_formed", "ptr (ptr)", `ptr ${r.name}`, "ptr", true); - case "cpAt": - // The code point AT an index as a one-code-point string (+1) — - // the string-for-of desugar's read. - return call( - "scr_str_cp_at", - "ptr (ptr, double)", - `ptr ${r.name}, double ${args[0]!.name}`, - "ptr", - true, - ); + case "byteLength": { + const t = B.tmp(); + // A byte count fits in a signed size, whose conversion is one instruction. + B.line(`${t} = sitofp ${host.sizeType} ${emitStringByteLength(host, r.name)} to double`); + return { name: t, type: e.type }; + } + case "cpAtByte": { + // sc_str_cp_at_byte (string-reads.ts) answers ASCII bytes from the + // charAt table inline. The cursor is a character start, an exact + // integer below the byte count. + host.declare(STRING_CHAR_AT_DECL); + host.declare(STRING_CP_AT_BYTE_DECL); + const index = e.args[0] ? exactInteger(host, args[0]!, e.args[0]) : null; + let wide: string; + if (index) { + wide = widenInteger(host, index); + } else { + wide = B.tmp(); + B.line(`${wide} = fptosi double ${args[0]!.name} to i64`); + } + const t = B.tmp(); + B.line(`${t} = call ptr @sc_str_cp_at_byte(ptr ${r.name}, i64 ${wide})`); + return host.own({ name: t, type: e.type }); + } default: { const _exhaustive: never = method; void _exhaustive; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 9f51f5455..fb294aa30 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -104,6 +104,7 @@ export const LIB_FN_SYMS: Record = { "str.localeCase": "scr_str_locale_case", "str.replaceLiteral": "scr_str_replace_literal", "str.replaceChars": "scr_str_replace_chars", + "str.codePoints": "scr_str_code_points", "num.fromString": "scr_string_to_number", "math.round": "scr_math_round", "math.sin": "sin", diff --git a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts index 07ad31153..72fc57752 100644 --- a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts +++ b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts @@ -341,7 +341,7 @@ const methodCases: { { method: "normalize", args: [ref("argument")], result: STRING, target: "normalize" }, { method: "isWellFormed", args: [], result: BOOL, target: "is_well_formed" }, { method: "toWellFormed", args: [], result: STRING, target: "to_well_formed" }, - { method: "cpAt", args: [num(1)], result: STRING, target: "cp_at" }, + { method: "cpAtByte", args: [num(1)], result: STRING, target: "cp_at_byte", inline: true }, ]; test.each(methodCases)( diff --git a/packages/compiler/src/backend/llvm/string-lifetimes.ts b/packages/compiler/src/backend/llvm/string-lifetimes.ts index 6c3079cc1..5a4504813 100644 --- a/packages/compiler/src/backend/llvm/string-lifetimes.ts +++ b/packages/compiler/src/backend/llvm/string-lifetimes.ts @@ -30,7 +30,8 @@ export function borrowsStringInputs(method: IrStrIntrinsicMethod): boolean { case "normalize": case "isWellFormed": case "toWellFormed": - case "cpAt": + case "byteLength": + case "cpAtByte": return true; default: return false; diff --git a/packages/compiler/src/backend/llvm/string-reads.ts b/packages/compiler/src/backend/llvm/string-reads.ts index 5b7a26c6c..1f77de2c2 100644 --- a/packages/compiler/src/backend/llvm/string-reads.ts +++ b/packages/compiler/src/backend/llvm/string-reads.ts @@ -43,6 +43,10 @@ export const STRING_CHAR_CODE_AT_INT_MARKER = "; uses @sc_str_char_code_at_int"; export const STRING_CHAR_AT_DECL = "declare ptr @scr_str_char_at(ptr, double)"; /** Requests the integer-index charAt helper, as the marker above does. */ export const STRING_CHAR_AT_INT_MARKER = "; uses @sc_str_char_at_int"; +/** The code-point character starting at a byte offset (the string + * iterator's step). Requested together with STRING_CHAR_AT_DECL, which + * defines the ASCII table. */ +export const STRING_CP_AT_BYTE_DECL = "declare ptr @scr_str_cp_at_byte(ptr, i64)"; /** The proven-ASCII flag of string `s`, as an i1. */ export function emitStringAsciiFlag(host: LlvmEmitterContext, s: string): string { @@ -351,6 +355,29 @@ export const STRING_READ_HELPERS: readonly StringReadHelper[] = [ ]; }, }, + { + // The string iterator's step at byte `%b`, a character start below the + // byte count: an ASCII byte is the table entry for it; any other lead + // byte takes the runtime, which copies the character's bytes. + decl: STRING_CP_AT_BYTE_DECL, + define: (sz, attrs) => [ + `define internal ptr @sc_str_cp_at_byte(ptr %s, i64 %b) ${attrs} {`, + `entry:`, + ` %data = getelementptr inbounds %ScrStr, ptr %s, i32 1`, + ` %p = getelementptr inbounds i8, ptr %data, i64 %b`, + ` %byte = load i8, ptr %p`, + ` %ascii = icmp sge i8 %byte, 0`, + ` br i1 %ascii, label %read, label %slow`, + `read:`, + ` %code = zext i8 %byte to i64`, + ` %char = getelementptr inbounds [128 x { ${sz}, ${sz}, ${sz}, [2 x i8] }], ptr @sc_ascii_chars, i64 0, i64 %code`, + ` ret ptr %char`, + `slow:`, + ` %r = call ptr @scr_str_cp_at_byte(ptr %s, i64 %b) nounwind willreturn`, + ` ret ptr %r`, + `}`, + ], + }, { // An exact integer index, as for sc_str_char_code_at_int. Requested // together with STRING_CHAR_AT_DECL, which defines the table. diff --git a/packages/compiler/src/frontend/lowering/containers/array-construction.ts b/packages/compiler/src/frontend/lowering/containers/array-construction.ts index 5029abc36..702837e40 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-construction.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-construction.ts @@ -1081,9 +1081,7 @@ export function lowerArrayFromCall( if ((args.length === 1 || args.length === 2) && !ts.isObjectLiteralExpression(args[0]!)) { const source = lowerer.lowerExpr(args[0]!); let src = - source.type.kind === "string" && args.length === 2 - ? strCharsCall(lowerer, source, loc) - : source; + source.type.kind === "string" && args.length === 2 ? strCharsCall(source, loc) : source; if (src.type.kind === "array") { const arrT = src.type; if (args.length === 1) { @@ -1171,7 +1169,7 @@ export function lowerArrayFromCall( } // String iteration advances by code point; the same helper serves // `[...s]` and both Array.from(s) forms. - if (src.type.kind === "string" && args.length === 1) return strCharsCall(lowerer, src, loc); + if (src.type.kind === "string" && args.length === 1) return strCharsCall(src, loc); if (src.type.kind === "dyn" && args.length === 1) { return lowerCheckedArrayFrom(lowerer, src, loc); } @@ -1318,112 +1316,9 @@ function buildArrayFromArrayFn( } /** `Array.from(s)` / `[...s]` on a STRING: the code-point split into a - * fresh string[], through one interned helper per module. */ -export function strCharsCall(lowerer: Lowerer, src: IrExpr, loc: SrcLoc): IrExpr { - const key = "strChars"; - let helper = lowerer.arrHofHelpers.get(key); - if (!helper) { - helper = "%str.chars"; - lowerer.arrHofHelpers.set(key, helper); - lowerer.liftedFns.push(buildStrCharsFn(helper, loc)); - } - return { kind: "call", callee: helper, args: [src], type: arrayOf(STRING), loc }; -} - -/** The code-point split, from existing IR nodes — the string for-of - * desugar's UTF-16 cursor as a function: - * - * out = []; - * i = 0; - * while (i < s.length) { ch = cpAt(s, i); i += ch.length; out.push(ch); } - * return out; - */ -function buildStrCharsFn(name: string, loc: SrcLoc): IrFunction { - const outT = arrayOf(STRING); - - const sLen = (recv: IrExpr): IrExpr => ({ - kind: "strIntrinsic", - method: "length", - receiver: recv, - args: [], - type: F64, - loc, - }); - const body: IrStmt[] = [ - { - kind: "varDecl", - localId: "out.0", - init: { kind: "arrayLit", elems: [], type: outT, loc }, - loc, - }, - { kind: "varDecl", localId: "i.0", init: { kind: "numLit", value: 0, type: F64, loc }, loc }, - { - kind: "while", - cond: { - kind: "bin", - op: "<", - left: varRef("i.0", F64, loc), - right: sLen(varRef("s.0", STRING, loc)), - type: BOOL, - loc, - }, - body: [ - { - kind: "varDecl", - localId: "ch.0", - init: { - kind: "strIntrinsic", - method: "cpAt", - receiver: varRef("s.0", STRING, loc), - args: [varRef("i.0", F64, loc)], - type: STRING, - loc, - }, - loc, - }, - { - kind: "assign", - localId: "i.0", - value: { - kind: "bin", - op: "+", - left: varRef("i.0", F64, loc), - right: sLen(varRef("ch.0", STRING, loc)), - type: F64, - loc, - }, - loc, - }, - { - kind: "exprStmt", - expr: { - kind: "arrIntrinsic", - method: "push", - receiver: varRef("out.0", outT, loc), - args: [varRef("ch.0", STRING, loc)], - type: F64, - loc, - }, - loc, - }, - ], - loc, - }, - { kind: "return", value: varRef("out.0", outT, loc), loc }, - ]; - return { - name, - params: [{ localId: "s.0", name: "s", type: STRING }], - returnType: outT, - locals: [ - { id: "s.0", name: "s", type: STRING, mutable: true }, - { id: "out.0", name: "out", type: outT, mutable: false }, - { id: "i.0", name: "i", type: F64, mutable: true }, - { id: "ch.0", name: "ch", type: STRING, mutable: false }, - ], - body, - loc, - }; + * fresh string[] (scr_str_code_points). */ +export function strCharsCall(src: IrExpr, loc: SrcLoc): IrExpr { + return { kind: "libCall", fn: "str.codePoints", args: [src], type: arrayOf(STRING), loc }; } /** The lowered `length` value of the one-property source literal, or null diff --git a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts index ddd63da3c..6307d871f 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts @@ -157,7 +157,8 @@ const PURE_STRING = new Set([ "toUpperCase", "isWellFormed", "toWellFormed", - "cpAt", + "byteLength", + "cpAtByte", ]); function primitive(t: IrType): boolean { diff --git a/packages/compiler/src/frontend/lowering/containers/collection-methods.ts b/packages/compiler/src/frontend/lowering/containers/collection-methods.ts index f18291a63..bfaca6f96 100644 --- a/packages/compiler/src/frontend/lowering/containers/collection-methods.ts +++ b/packages/compiler/src/frontend/lowering/containers/collection-methods.ts @@ -1135,7 +1135,7 @@ function setFromSeedValue( if (source.type.kind === "set" && typeEquals(source.type, setT)) { return { kind: "setIntrinsic", method: "clone", receiver: source, args: [], type: setT, loc }; } else if (source.type.kind === "string" && setT.elem.kind === "string") { - seed = strCharsCall(lowerer, source, loc); + seed = strCharsCall(source, loc); } else if (source.type.kind === "array" && typeEquals(source.type.elem, setT.elem)) { seed = source; } else if ( diff --git a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts index 903f0884d..77c16594f 100644 --- a/packages/compiler/src/frontend/lowering/expressions/array-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/array-literals.ts @@ -504,7 +504,7 @@ export function lowerArrayLiteral( // interned helper as Array.from(s); the result rides the array // machinery below like any string[] source. if (src.type.kind === "string") { - src = strCharsCall(lowerer, src, locOf(el)); + src = strCharsCall(src, locOf(el)); } // `[...typedArray]`: represented typed arrays are dense numeric // iterables, so drain their elements into the fresh number[] that diff --git a/packages/compiler/src/frontend/lowering/generator-delegation.ts b/packages/compiler/src/frontend/lowering/generator-delegation.ts index a606c177e..81ecc8353 100644 --- a/packages/compiler/src/frontend/lowering/generator-delegation.ts +++ b/packages/compiler/src/frontend/lowering/generator-delegation.ts @@ -577,10 +577,11 @@ export function lowerBuiltinDelegation( loc, }); } else if (sourceT.kind === "string") { + // A byte cursor over the WTF-8 data, as for-of over a string walks. const ch = lowerer.declareHiddenLocal("%delegateElement", STRING); const length = (receiver: IrExpr): IrExpr => ({ kind: "strIntrinsic", - method: "length", + method: "byteLength", receiver, args: [], type: F64, @@ -595,7 +596,7 @@ export function lowerBuiltinDelegation( localId: ch.id, init: { kind: "strIntrinsic", - method: "cpAt", + method: "cpAtByte", receiver: src(), args: [at()], type: STRING, diff --git a/packages/compiler/src/frontend/lowering/iterator-adapters.ts b/packages/compiler/src/frontend/lowering/iterator-adapters.ts index 4909e7f45..78fcff84d 100644 --- a/packages/compiler/src/frontend/lowering/iterator-adapters.ts +++ b/packages/compiler/src/frontend/lowering/iterator-adapters.ts @@ -743,7 +743,7 @@ export function lowerStringIteratorValue( left: index, right: { kind: "strIntrinsic", - method: "length", + method: "byteLength", receiver: text, args: [], type: F64, @@ -758,7 +758,7 @@ export function lowerStringIteratorValue( localId: "point.0", init: { kind: "strIntrinsic", - method: "cpAt", + method: "cpAtByte", receiver: text, args: [index], type: STRING, @@ -775,7 +775,7 @@ export function lowerStringIteratorValue( left: index, right: { kind: "strIntrinsic", - method: "length", + method: "byteLength", receiver: point, args: [], type: F64, diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index d6d8c1cc7..9586727b3 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -2668,7 +2668,7 @@ export function lowerBindingPattern( lowerer.unsupported("SC1031", el, "rest elements over a union of array or tuple layouts"); const loc = locOf(el); let value = unionArrayValueRead(lowerer, srcRef(), numLit(i, loc), loc, (text) => - strCharsCall(lowerer, text, loc), + strCharsCall(text, loc), ); if (!value) lowerer.unsupported( @@ -2798,7 +2798,7 @@ export function lowerBindingPattern( } // STRING sources (`const [a, b] = str`): array destructuring walks // the STRING ITERATOR — code points, astral characters whole — - // which is Array.from(string)'s own machinery (%str.chars), so the + // which is Array.from(string)'s own machinery (str.codePoints), so the // source splits once into a hidden chars array and the pattern // lowers as an array pattern over string[]. Positions, holes, // defaults (with the bounds test), rest (the remaining code points @@ -2817,7 +2817,7 @@ export function lowerBindingPattern( out.push({ kind: "varDecl", localId: chars.id, - init: strCharsCall(lowerer, srcRef(), loc), + init: strCharsCall(srcRef(), loc), loc, }); lowerer.lowerBindingPattern( @@ -9826,7 +9826,7 @@ function lowerArrayAssignInto( let srcType = tmpRef().type; // A STRING source (`[a, b] = s`): the declaration path's rule in // assignment position — the string iterator's code-point split - // (%str.chars, astral characters whole) into a hidden chars array, + // (str.codePoints, astral characters whole) into a hidden chars array, // then the pattern proceeds as an array pattern over string[] // (defaults with the bounds test, rest packing the remaining code // points, elisions; positions past the last code point inherit the @@ -9839,7 +9839,7 @@ function lowerArrayAssignInto( out.push({ kind: "varDecl", localId: chars.id, - init: strCharsCall(lowerer, tmpRef(), loc), + init: strCharsCall(tmpRef(), loc), loc, }); tmpRef = () => ({ kind: "varRef", localId: chars.id, type: charsT, loc }); @@ -11383,16 +11383,18 @@ function lowerForOfMatchAll( /** `for (const ch of s)` over a STRING: JS's string iterator, which walks * code POINTS — each pass binds the whole character (a two-unit string * for astral chars, where charAt/slice would return one surrogate half). - * Desugars over the UTF-16 cursor machinery: a hidden unit - * index reads the character with the cpAt intrinsic and advances by ITS - * length before the body runs (so `continue` never skips the advance): + * Desugars over a byte cursor into the WTF-8 data: a hidden byte offset + * reads the character that starts there with the cpAtByte intrinsic and + * advances by ITS byte count before the body runs (so `continue` never + * skips the advance): * * { const %s = ; let %i = 0; - * while (%i < %s.length) { const ch = cpAt(%s, %i); %i += ch.length; } } + * while (%i < byteLength(%s)) { const ch = cpAtByte(%s, %i); %i += byteLength(ch); } } * - * Strings are immutable, so the length is stable and there is no + * Strings are immutable, so the byte count is stable and there is no * iterator state to unwind on throw; a lone surrogate is a one-unit - * character, so every step lands on a character start. */ + * character with its own three-byte encoding, so every step lands on a + * character start. */ /** for-of over a CLASS ITERABLE (classIteratorOf's shape) — the * iterator protocol as ordinary method calls: * @@ -12087,7 +12089,7 @@ function lowerForOfString( try { const s = lowerer.declareHiddenLocal("%strof", STRING); const i = lowerer.declareHiddenLocal("%iterof", F64); - i.mutable = true; // the unit index reassigns (hidden locals default const) + i.mutable = true; // the byte cursor reassigns (hidden locals default const) const sRef = (): IrExpr => ({ kind: "varRef", localId: s.id, type: STRING, loc }); const iRef = (): IrExpr => ({ kind: "varRef", localId: i.id, type: F64, loc }); // `for (var c of str)`: the character mechanics stay on a hidden @@ -12118,7 +12120,7 @@ function lowerForOfString( localId: ch.id, init: { kind: "strIntrinsic", - method: "cpAt", + method: "cpAtByte", receiver: sRef(), args: [iRef()], type: STRING, @@ -12145,7 +12147,7 @@ function lowerForOfString( left: iRef(), right: { kind: "strIntrinsic", - method: "length", + method: "byteLength", receiver: chRef, args: [], type: F64, @@ -12177,7 +12179,7 @@ function lowerForOfString( left: iRef(), right: { kind: "strIntrinsic", - method: "length", + method: "byteLength", receiver: sRef(), args: [], type: F64, diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 4ccf39aab..091504e65 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -848,6 +848,7 @@ function buildLibFnSignatures() { // (subject, characters, NUL-separated replacement texts): the fused // chain of global one-character replacements. "str.replaceChars": { argTypes: [STRING, STRING, STRING], result: STRING }, + "str.codePoints": { argTypes: [STRING], result: arrayOf(STRING) }, /* ToNumber(string) — ECMA-262 7.1.4.1 StringToNumber (scr_string.c): * trim the JS StrWhiteSpace set, empty/whitespace-only → +0, then the * whole span must be one StrNumericLiteral — signed decimal (Infinity diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 1909040c4..51c0a929b 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -867,6 +867,8 @@ export function analyzeIntegerRangeFacts( ); if (e.type.kind === "f64") { if (e.method === "length") range = { min: 0, max: STRING_MAX_UNITS }; + // A string holds at most SCR_STR_MAX_CAP (0x7fffffff) bytes. + else if (e.method === "byteLength") range = { min: 0, max: STRING_MAX_UNITS }; // An empty needle is found at the clamped position, up to the length. else if (e.method === "indexOf") range = { min: -1, max: STRING_MAX_UNITS }; } diff --git a/packages/compiler/src/ir/intrinsic-signatures.ts b/packages/compiler/src/ir/intrinsic-signatures.ts index eda569883..3e26e77f2 100644 --- a/packages/compiler/src/ir/intrinsic-signatures.ts +++ b/packages/compiler/src/ir/intrinsic-signatures.ts @@ -31,7 +31,8 @@ export const STR_INTRINSIC_SIGS: Record< normalize: { argTypes: [STRING], minArgs: 1, result: STRING }, isWellFormed: { argTypes: [], minArgs: 0, result: BOOL }, toWellFormed: { argTypes: [], minArgs: 0, result: STRING }, - cpAt: { argTypes: [F64], minArgs: 1, result: STRING }, + byteLength: { argTypes: [], minArgs: 0, result: F64 }, + cpAtByte: { argTypes: [F64], minArgs: 1, result: STRING }, }; /** Per-method signature for regexIntrinsic. test/source/flags take the diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 120f4db15..c946d799f 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -2451,10 +2451,13 @@ export type IrStrIntrinsicMethod = // surrogate, and the copy with each one replaced by U+FFFD. | "isWellFormed" | "toWellFormed" - // The string iterator's step: the full code-POINT character at a UTF-16 - // index (astral chars come back whole where charAt truncates) — for-of - // over strings desugars onto it, advancing by the result's length. - | "cpAt"; + // The string iterator over the WTF-8 bytes: the byte count of a string, + // and the code-point character that starts at a byte offset (a lone + // surrogate is a one-unit character). for-of, spread and Array.from of a + // string walk a byte cursor with them, advancing by the character's + // byte count; the cursor is always a character start below the count. + | "byteLength" + | "cpAtByte"; /** The typed-array/Buffer method surface (bytesIntrinsic). Receiver/arg * conventions (validated): `length`/`byteLength` are property reads → f64; diff --git a/packages/compiler/test/string-iteration-emission.test.ts b/packages/compiler/test/string-iteration-emission.test.ts new file mode 100644 index 000000000..7cda3da55 --- /dev/null +++ b/packages/compiler/test/string-iteration-emission.test.ts @@ -0,0 +1,47 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function emit(source: string): Promise { + const outDir = await mkdtemp(join(tmpdir(), "scriptc-string-iteration-")); + directories.push(outDir); + const entry = join(outDir, "main.ts"); + await writeFile(entry, source); + const outPath = join(outDir, "program.ll"); + const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result)).toBe(true); + return readFile(outPath, "utf8"); +} + +const calls = (llvm: string, symbol: string): string[] => + llvm.split("\n").filter((line) => line.includes("call ") && line.includes(`@${symbol}(`)); + +test("for-of over a string walks a byte cursor", async () => { + const llvm = await emit(` +function count(s: string): number { + let n = 0; + for (const c of s) n += c.length; + return n; +} +console.log(count(process.argv[1] ?? "")); +`); + expect(calls(llvm, "sc_str_cp_at_byte")).toHaveLength(1); + expect(calls(llvm, "scr_str_cp_at")).toHaveLength(0); + // The helper answers ASCII bytes from the one-character table. + expect(llvm).toContain("define internal ptr @sc_str_cp_at_byte(ptr %s, i64 %b)"); +}); + +test("spread and Array.from of a string build the array in one call", async () => { + const llvm = await emit(` +const s = process.argv[1] ?? ""; +console.log([...s].length, Array.from(s).length); +`); + expect(calls(llvm, "scr_str_code_points")).toHaveLength(2); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8347-string-iteration-code-points.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8347-string-iteration-code-points.ts.json new file mode 100644 index 000000000..1e83f7b6f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8347-string-iteration-code-points.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8347-string-iteration-code-points.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8348-string-iteration-worker-characters.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8348-string-iteration-worker-characters.ts.json new file mode 100644 index 000000000..e767eaf65 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8348-string-iteration-worker-characters.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8348-string-iteration-worker-characters.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 1e7802fbb..64a4772e5 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -3514,6 +3514,8 @@ ScrStr *scr_bool_to_scrstr(bool b) { * decode is a whole char; an i addressing the LOW half of an astral pair * (unreachable from the iterator, which only lands on char starts) answers * the containing char. Out of range → empty string. Borrows s; +1. */ +static ScrStr *scr_str_char_of(const char *bytes, size_t adv); + ScrStr *scr_str_cp_at(ScrStr *s, double i) { double idx = scr_to_integer_or_infinity(i); if (!(idx >= 0)) return scr_str_empty(); @@ -3528,7 +3530,80 @@ ScrStr *scr_str_cp_at(ScrStr *s, double i) { if (off >= s->len) return scr_str_empty(); /* out of range */ size_t adv; (void)scr_utf8_decode(s->data + off, &adv); - return scr_str_from_span(s->data + off, adv); + return scr_str_char_of(s->data + off, adv); +} + +/* The one-character strings of U+0080..U+07FF (two-byte sequences, lead + * bytes 0xC2..0xDF), made immortal on first use and shared by every + * thread, as the ASCII ones are static: the string iterator returns them + * instead of a new string per character. A string is immortal before its + * slot publishes it; a thread that loses the race to fill a slot keeps its + * own immortal copy, so at most one extra string per slot and thread. The + * strings come from the system allocator, as other state that outlives a + * thread does: a worker's own memory goes back when it exits without + * sharing (scr_alloc.c), and the table outlives the worker that filled a + * slot. */ +static ScrStr *scr_two_byte_chars[0x800 - 0x80]; + +static ScrStr *scr_two_byte_char(const char *bytes) { + uint32_t cp = ((uint32_t)((unsigned char)bytes[0] & 0x1F) << 6) | ((unsigned char)bytes[1] & 0x3F); + ScrStr **slot = &scr_two_byte_chars[cp - 0x80]; + ScrStr *known = __atomic_load_n(slot, __ATOMIC_ACQUIRE); + if (known) return known; + ScrStr *made = calloc(1, sizeof(ScrStr) + 3); + if (!made) scr_oom(); + made->len = 2; + made->cap = 2; + memcpy(made->data, bytes, 2); + scr_str_hash_prime(made); + made->rc = SIZE_MAX; + scr_lsan_immortal(made); + ScrStr *expected = NULL; + if (__atomic_compare_exchange_n(slot, &expected, made, false, __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE)) + return made; + return expected; +} + +/* The character whose encoding starts at bytes and is adv bytes long (+1, + * or immortal). */ +static ScrStr *scr_str_char_of(const char *bytes, size_t adv) { + unsigned char lead = (unsigned char)bytes[0]; + if (adv == 1 && lead < 0x80) return (ScrStr *)&scr_ascii1[lead]; + if (adv == 2) return scr_two_byte_char(bytes); + return scr_str_from_span(bytes, adv); +} + +ScrStr *scr_str_cp_at_byte(ScrStr *s, uint64_t b) { + if (b >= s->len) return scr_str_empty(); + size_t adv; + (void)scr_utf8_decode(s->data + b, &adv); + return scr_str_char_of(s->data + b, adv); +} + +ScrArr *scr_str_code_points(ScrStr *s) { + /* Every byte but a continuation byte starts one character; a lone + * surrogate's three bytes are one character. */ + size_t count = s->len; + if (!s->ascii) + for (size_t b = 0; b < s->len; b++) count -= ((unsigned char)s->data[b] & 0xC0) == 0x80; + ScrArr *out = scr_arr_new_inline(SCR_ELEM_STR, count, false); + size_t k = 0; + for (size_t b = 0; b < s->len;) { + unsigned char c = (unsigned char)s->data[b]; + if (c < 0x80) { + out->data[k++] = (uint64_t)(uintptr_t)&scr_ascii1[c]; + b++; + continue; + } + size_t adv; + (void)scr_utf8_decode(s->data + b, &adv); + out->data[k++] = (uint64_t)(uintptr_t)scr_str_char_of(s->data + b, adv); + b += adv; + } + out->len = (uint32_t)count; + out->filled = (uint32_t)count; + return out; } /* ── the URI component codecs (encodeURIComponent/decodeURIComponent) ── diff --git a/packages/runtime/src/scr_string_methods.h b/packages/runtime/src/scr_string_methods.h index 01ccdd8ae..2e69ee6ac 100644 --- a/packages/runtime/src/scr_string_methods.h +++ b/packages/runtime/src/scr_string_methods.h @@ -150,6 +150,15 @@ ScrStr *scr_str_char_at(ScrStr *s, double i); * (two units); the consuming loop advances by the result's length. * Out of range → empty string. Returns +1. */ ScrStr *scr_str_cp_at(ScrStr *s, double i); +/* The string iterator's step over the WTF-8 bytes: the code-point + * character whose encoding starts at byte offset b, a character start + * below s->len (a lone surrogate is its own three-byte character). The + * compiler answers ASCII bytes inline and calls this for the rest. + * Returns +1. */ +ScrStr *scr_str_cp_at_byte(ScrStr *s, uint64_t b); +/* [...s] and Array.from(s): the code-point characters of s in order, as a + * fresh string array of exactly that length. Returns +1. */ +struct ScrArr *scr_str_code_points(ScrStr *s); /* trimStart()/trimEnd(): the one-sided halves of trim. Returns +1. */ ScrStr *scr_str_trim_start(ScrStr *s); diff --git a/tests/corpus/1561-forof-strings.ts b/tests/corpus/1561-forof-strings.ts index 9d16efd9a..0ea09e755 100644 --- a/tests/corpus/1561-forof-strings.ts +++ b/tests/corpus/1561-forof-strings.ts @@ -1,7 +1,7 @@ // for-of over STRINGS: JS's string iterator walks code POINTS — each pass // binds the whole character, so astral chars arrive as one two-unit string -// (where charAt/slice would truncate the halves). Desugars over the UTF-16 -// cursor with the cpAt step intrinsic. +// (where charAt/slice would truncate the halves). Desugars over a byte +// cursor with the cpAtByte step intrinsic. for (const ch of "héllo 🎉x") console.log(ch, ch.length); // The empty string iterates zero times. diff --git a/tests/corpus/8347-string-iteration-code-points.ts b/tests/corpus/8347-string-iteration-code-points.ts new file mode 100644 index 000000000..3a1286140 --- /dev/null +++ b/tests/corpus/8347-string-iteration-code-points.ts @@ -0,0 +1,96 @@ +// The string iterator: for-of, spread, Array.from, yield* and +// [Symbol.iterator]() walk code points. Astral characters come back whole, +// a lone surrogate is a one-unit character, and halves joined by +// concatenation form one character, as in Node. +function unitCodes(c: string): string { + const codes: string[] = []; + for (let k = 0; k < c.length; k++) codes.push(c.charCodeAt(k).toString(16)); + return `${c.length}:${codes.join(".")}`; +} +const show = (chars: string[]): string => chars.map(unitCodes).join(" "); + +const samples = [ + "", + "abc", + "aé😀b", + "\u0000\u007f\u0080߿ࠀ￿", + "\u{10000}\u{10ffff}", + "é ñ", + "\ud800", + "\udc00", + "x\ud83d", + "\ude00x", + "\udc00\ud800", + "\ud83d" + "\ude00", + "a\ud83d" + "\ude00b\ud83d", + "日本語テキスト", +]; +for (const s of samples) { + const looped: string[] = []; + for (const c of s) looped.push(c); + console.log(JSON.stringify(s), show(looped), show([...s]), show(Array.from(s)), [...s].length, s.length); +} + +// Loop control, heads and bindings. +const text = "a😀b\ud800c"; +let out = ""; +outer: for (const c of text) { + if (c === "b") continue; + for (const d of "xy") { + if (d === "y" && c === "c") continue outer; + out += c + d; + } + if (c === "\ud800") out += "|"; +} +console.log(out); +for (var v of "hé") out += v; +console.log(out); +let last = ""; +for (last of "q😀") out += last.length; +console.log(out, last); +for (const [first] of ["😀z", "ab"].join("")) out += first; +console.log(out); +let source = "abc"; +for (const c of source) { + source += c; + out += source.length; +} +console.log(out, source); +let n = 0; +for (const c of "stop😀here") { + if (c === "😀") break; + n += c.length; +} +console.log(n); + +// Characters as values: lengths, Map keys and code points. +const counts = new Map(); +for (const c of "abracadabra😀😀é") counts.set(c, (counts.get(c) ?? 0) + 1); +console.log([...counts].map(([k, v]) => `${k}=${v}`).join(",")); +let points = 0; +for (const c of "😀é\ud800a") points += c.codePointAt(0)!; +console.log(points); + +// Generators and explicit iterators. +function* letters(s: string): Generator { + yield* s; +} +console.log(show([...letters("x😀\udc00y")])); +const it = "o😀"[Symbol.iterator](); +let step = it.next(); +while (!step.done) { + out += step.value.length; + step = it.next(); +} +console.log(out); + +// A long string built from mixed pieces. +let long = ""; +for (let i = 0; i < 2000; i++) long += i % 3 === 0 ? "a" : i % 3 === 1 ? "é" : "😀"; +let units = 0; +let chars = 0; +for (const c of long) { + units += c.length; + chars++; +} +console.log(units, chars, long.length, [...long].length); diff --git a/tests/corpus/8348-string-iteration-worker-characters.ts b/tests/corpus/8348-string-iteration-worker-characters.ts new file mode 100644 index 000000000..52a38fb78 --- /dev/null +++ b/tests/corpus/8348-string-iteration-worker-characters.ts @@ -0,0 +1,44 @@ +import { Worker, isMainThread, parentPort } from "node:worker_threads"; + +// The one-character strings of U+0080..U+07FF that string iteration returns +// are shared by every thread. A worker that is the first to iterate them +// and exits without sharing anything else leaves them in use: the main +// thread iterates the same characters after the worker's memory goes back. + +const text = "aé ñ ß ü Ω ж א ع ߿"; + +function codes(s: string): string { + const out: string[] = []; + for (const c of s) out.push(`${c.length}:${c.charCodeAt(0).toString(16)}`); + return out.join(" "); +} + +function runWorker(): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL(import.meta.url)); + let count = -1; + worker.on("message", (value: number) => { + count = value; + }); + worker.on("error", reject); + worker.on("exit", () => resolve(count)); + }); +} + +async function main(): Promise { + console.log("worker", await runWorker()); + // The worker's memory goes back shortly after it exits. + await new Promise((resolve) => setTimeout(resolve, 300)); + console.log("main", codes(text)); + console.log("spread", [...text].join("|")); + console.log("from", Array.from("ñandú").map((c) => c.length).join(",")); +} + +if (isMainThread) { + void main(); +} else { + let count = 0; + for (const c of text) count += c.length; + count += [...text].length + Array.from("ñandú").length; + parentPort!.postMessage(count); +} diff --git a/tests/harness/corpus-costs/8347-string-iteration-code-points.ts.json b/tests/harness/corpus-costs/8347-string-iteration-code-points.ts.json new file mode 100644 index 000000000..3e2ae0706 --- /dev/null +++ b/tests/harness/corpus-costs/8347-string-iteration-code-points.ts.json @@ -0,0 +1 @@ +{ "plain": 2770, "san": 3750 } diff --git a/tests/harness/corpus-costs/8348-string-iteration-worker-characters.ts.json b/tests/harness/corpus-costs/8348-string-iteration-worker-characters.ts.json new file mode 100644 index 000000000..4c96af583 --- /dev/null +++ b/tests/harness/corpus-costs/8348-string-iteration-worker-characters.ts.json @@ -0,0 +1 @@ +{ "plain": 3960, "san": 3900 } diff --git a/tests/harness/string-iteration-workers.test.ts b/tests/harness/string-iteration-workers.test.ts new file mode 100644 index 000000000..bc1feca95 --- /dev/null +++ b/tests/harness/string-iteration-workers.test.ts @@ -0,0 +1,44 @@ +/* The one-character strings that string iteration shares between threads + * outlive the worker that first made them. On Linux a worker that exits + * without sharing anything returns its memory; with SCR_SA_DISCARD=poison + * that memory becomes inaccessible instead, so a cached character + * allocated from the worker's memory would fault when the main thread + * reads it. The corpus program pins Node.js parity; this suite runs it + * with the discard poisoned and immediate. Under SCRIPTC_SAN=1 the runtime + * keeps every worker's memory, so the check does not apply. */ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const program = join(import.meta.dirname, "../corpus/8348-string-iteration-worker-characters.ts"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +test.skipIf(process.platform !== "linux" || sanitize)( + "characters a discarded worker cached stay readable", + { timeout: 240_000 }, + async () => { + const outDir = await mkdtemp(join(tmpdir(), "scriptc-string-iteration-workers-")); + try { + const result = await compile(program, { + backend: "llvm", + outDir, + outPath: join(outDir, "program"), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const { stdout } = await exec(result.binaryPath, [], { + env: { ...process.env, SCR_SA_DISCARD: "poison", SCR_SA_DISCARD_DELAY_MS: "0" }, + timeout: 120_000, + }); + expect(stdout).toContain("main 1:61 1:e9 1:20 1:f1"); + expect(stdout).toContain("spread a|é| |ñ"); + } finally { + await rm(outDir, { recursive: true, force: true }); + } + }, +);