From 763541b433ea9e7b9c6201a7555bc0200699aa84 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 6 Oct 2026 19:01:30 -0500 Subject: [PATCH] Stream string split loops - Stream eligible split loops while preserving snapshots, bindings, and cleanup. - Reuse unescaped piece storage and batch materialized results. - Cover limits, Unicode, debugging, and native/WASI behavior. --- packages/compiler/src/backend/llvm/emitter.ts | 99 +++++++++--- .../compiler/src/backend/llvm/split-loops.ts | 153 ++++++++++++++++++ .../src/frontend/lowering/lower-stmts.ts | 16 +- .../compiler/src/frontend/lowering/lowerer.ts | 1 + .../src/frontend/lowering/split-consumers.ts | 66 ++++++++ .../compiler/test/split-loop-emission.test.ts | 103 ++++++++++++ .../test/ts7/baselines/order-parity.json | 6 + packages/runtime/src/scr_runtime.h | 14 ++ packages/runtime/src/scr_string.c | 134 ++++++++++----- packages/runtime/test/gen-string-cases.mjs | 20 +++ packages/runtime/test/string-cases.txt | 55 +++++++ packages/runtime/test/test_string.c | 67 +++++++- tests/corpus/string-split-traversal.ts | 131 +++++++++++++++ .../harness/self-hosting-llvm-emitter.test.ts | 1 + tests/harness/wasi-differential.test.ts | 1 + 15 files changed, 801 insertions(+), 66 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/split-loops.ts create mode 100644 packages/compiler/src/frontend/lowering/split-consumers.ts create mode 100644 packages/compiler/test/split-loop-emission.test.ts create mode 100644 tests/corpus/string-split-traversal.ts diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 1d10330d3..6b6735a6d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -48,6 +48,7 @@ import { RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, typeKey, + STRING, VOID, } from "../../ir/ir.js"; import { @@ -64,6 +65,18 @@ import { import { emitCountedLoopLimit } from "./counted-loops.js"; import { emitLiteralSwitch } from "./switch-dispatch.js"; import { findBytesBounds } from "./bytes-bounds.js"; +import { + emitSplitCursor, + emitSplitNext, + emitSplitScratch, + emitSplitSnapshot, + findPrivateSplitLocals, + loadSplitSnapshot, + storeSplitSnapshot, + stringSplit, + type StoredSplitSnapshot, + type StringSplit, +} from "./split-loops.js"; import { scalarizeNumericRecords } from "../../ir/scalar-records.js"; import { everyStmtList } from "../../ir/traverse.js"; import { analyzeIntegerRanges, type IntegerRanges } from "../../ir/integer-ranges.js"; @@ -534,6 +547,9 @@ export class LlEmitter { finallyDepth: number; }[] = []; private currentLocals = new Map(); + private privateSplitLocals = new Map(); + private storedSplits = new Map(); + private streamingSplitsEnabled = false; private readonly initializerBindings: ReturnType; private numericLocals = new Map(); private captureIds = new Set(); @@ -3731,6 +3747,11 @@ export class LlEmitter { this.unwindCleanups.clear(); this.jumpTargets = []; this.currentLocals = new Map(fn.locals.map((l) => [l.id, l])); + // Preserve concrete source bindings for debugger inspection. Suspended + // functions keep the established array lifetime across continuations. + this.streamingSplitsEnabled = this.debug === null && !fn.async && !fn.generator; + this.privateSplitLocals = this.streamingSplitsEnabled ? findPrivateSplitLocals(fn) : new Map(); + this.storedSplits.clear(); const initializerBindings = this.initializerBindings.get(fn.name) ?? []; const numericFn = withInitializerBindings(fn, initializerBindings); this.numericLocals = new Map(numericFn.locals.map((l) => [l.id, l])); @@ -4061,6 +4082,16 @@ export class LlEmitter { switch (s.kind) { case "varDecl": { const b = this.binding(s.localId); + const split = this.privateSplitLocals.get(s.localId); + if (split) { + const snapshot = storeSplitSnapshot(this, emitSplitSnapshot(this, split)); + this.storedSplits.set(s.localId, snapshot); + this.scopes[this.scopes.length - 1]!.push( + { slot: snapshot.sourceSlot, type: STRING }, + { slot: snapshot.separatorSlot, type: STRING }, + ); + break; + } const localUnion = this.localStackUnions.get(s.localId); if (localUnion) { const union = emitStackUnion(this, localUnion); @@ -4723,23 +4754,42 @@ export class LlEmitter { if (s.iterable.type.kind !== "array") throw new LlvmUnsupportedError(`forOf:${s.iterable.type.kind}`, s.loc); const elem = s.iterable.type.elem; - const arr = this.emitExpr(s.iterable); - const idxSlot = B.slot(); - B.entryAllocas.push(`${idxSlot} = alloca double`); - B.line(`store double ${f64Lit(0)}, ptr ${idxSlot}`); + const stored = + s.iterable.kind === "varRef" ? this.storedSplits.get(s.iterable.localId) : undefined; + const direct = this.streamingSplitsEnabled ? stringSplit(s.iterable) : null; + const snapshot = stored + ? loadSplitSnapshot(this, stored) + : direct + ? emitSplitSnapshot(this, direct) + : null; + const arr = snapshot ? null : this.emitExpr(s.iterable); + const cursor = snapshot ? emitSplitCursor(this, snapshot) : null; + const scratch = snapshot ? emitSplitScratch(this) : null; + const idxSlot = snapshot ? null : B.slot(); + if (idxSlot) { + B.entryAllocas.push(`${idxSlot} = alloca double`); + B.line(`store double ${f64Lit(0)}, ptr ${idxSlot}`); + } const lc = B.newLabel("fof.c"); const lb = B.newLabel("fof.b"); const lu = B.newLabel("fof.u"); const le = B.newLabel("fof.e"); B.br(lc); B.startBlock(lc); - const i = B.tmp(); - const len = B.tmp(); const inBounds = B.tmp(); - this.declare(`declare double @scr_arr_len(ptr)`); - B.line(`${i} = load double, ptr ${idxSlot}`); - B.line(`${len} = call double @scr_arr_len(ptr ${arr.name})`); - B.line(`${inBounds} = fcmp olt double ${i}, ${len}`); + let cur: string; + if (snapshot && cursor && scratch) { + cur = emitSplitNext(this, snapshot, cursor, scratch); + B.line(`${inBounds} = icmp ne ptr ${cur}, null`); + } else { + const i = B.tmp(), + len = B.tmp(); + this.declare(`declare double @scr_arr_len(ptr)`); + B.line(`${i} = load double, ptr ${idxSlot}`); + B.line(`${len} = call double @scr_arr_len(ptr ${arr!.name})`); + B.line(`${inBounds} = fcmp olt double ${i}, ${len}`); + cur = i; + } B.condBr(inBounds, lb, le); B.startBlock(lb); this.jumpTargets.push({ @@ -4757,13 +4807,16 @@ export class LlEmitter { this.scopes.push([]); const localInfo = this.currentLocals.get(s.localId); const slot = `%${mangleLocal(s.localId)}`; - const acc = elemAccess(elem); - const accTy = acc === "f64" ? "double" : acc === "bool" ? "i1" : "ptr"; - this.declare( - `declare ${acc === "bool" ? "zeroext i1" : accTy} @scr_arr_get_${acc}(ptr, double)`, - ); - const cur = B.tmp(); - B.line(`${cur} = call ${accTy} @scr_arr_get_${acc}(ptr ${arr.name}, double ${i})`); + if (!snapshot) { + const acc = elemAccess(elem); + const accTy = acc === "f64" ? "double" : acc === "bool" ? "i1" : "ptr"; + this.declare( + `declare ${acc === "bool" ? "zeroext i1" : accTy} @scr_arr_get_${acc}(ptr, double)`, + ); + const value = B.tmp(); + B.line(`${value} = call ${accTy} @scr_arr_get_${acc}(ptr ${arr!.name}, double ${cur})`); + cur = value; + } if (localInfo?.boxed) { // Captured loop variable: a fresh box per iteration, matching the // fresh const binding. The box takes ownership of a ref element's @@ -4784,11 +4837,13 @@ export class LlEmitter { this.jumpTargets.pop(); B.br(lu); B.startBlock(lu); - const i2 = B.tmp(); - const i3 = B.tmp(); - B.line(`${i2} = load double, ptr ${idxSlot}`); - B.line(`${i3} = fadd double ${i2}, ${f64Lit(1)}`); - B.line(`store double ${i3}, ptr ${idxSlot}`); + if (idxSlot) { + const i2 = B.tmp(), + i3 = B.tmp(); + B.line(`${i2} = load double, ptr ${idxSlot}`); + B.line(`${i3} = fadd double ${i2}, ${f64Lit(1)}`); + B.line(`store double ${i3}, ptr ${idxSlot}`); + } B.br(lc); B.startBlock(le); break; diff --git a/packages/compiler/src/backend/llvm/split-loops.ts b/packages/compiler/src/backend/llvm/split-loops.ts new file mode 100644 index 000000000..b7af76de8 --- /dev/null +++ b/packages/compiler/src/backend/llvm/split-loops.ts @@ -0,0 +1,153 @@ +import { STRING, type IrExpr, type IrFunction } from "../../ir/ir.js"; +import { everyStmtList } from "../../ir/traverse.js"; +import type { LlvmEmitterContext } from "./expr-context.js"; + +export type StringSplit = IrExpr & { kind: "strIntrinsic" }; + +export function stringSplit(value: IrExpr): StringSplit | null { + return value.kind === "strIntrinsic" && value.method === "split" ? value : null; +} + +/** Stored arrays can disappear only when one loop is their entire use. + * Count every executable reference and explicit write; captured, TDZ and + * suspended bindings retain their ordinary representation. Frontend + * density proofs alone are not an escape or lifetime proof. */ +export function findPrivateSplitLocals(fn: IrFunction): Map { + const splits = new Map(); + if (fn.async || fn.generator) return splits; + everyStmtList(fn.body, { + expr: () => true, + stmt: (stmt) => { + if (stmt.kind !== "varDecl" || !stmt.init) return true; + const split = stringSplit(stmt.init); + if (!split) return true; + const local = fn.locals.find((local) => local.id === stmt.localId); + if (local && !local.mutable && !local.boxed && !local.tdz) splits.set(stmt.localId, split); + return true; + }, + }); + if (splits.size === 0) return splits; + const definitions = new Map(); + const references = new Map(); + const consumers = new Set(); + const writes = new Set(); + everyStmtList(fn.body, { + expr: (value) => { + if (value.kind === "varRef" && splits.has(value.localId)) + references.set(value.localId, (references.get(value.localId) ?? 0) + 1); + if (value.kind === "closure" || value.kind === "classRef") + for (const id of value.captures ?? []) if (splits.has(id)) writes.add(id); + return true; + }, + stmt: (stmt) => { + if (stmt.kind === "varDecl" && splits.has(stmt.localId)) { + definitions.set(stmt.localId, (definitions.get(stmt.localId) ?? 0) + 1); + } + if (stmt.kind === "assign" && splits.has(stmt.localId)) writes.add(stmt.localId); + if (stmt.kind === "forOf" && stmt.iterable.kind === "varRef") + consumers.add(stmt.iterable.localId); + return true; + }, + }); + for (const id of splits.keys()) + if ( + definitions.get(id) !== 1 || + references.get(id) !== 1 || + !consumers.has(id) || + writes.has(id) + ) + splits.delete(id); + return splits; +} + +export interface SplitSnapshot { + source: string; + separator: string; + limit: string; +} + +/** Evaluate in source order, retaining even stable bindings: body updates + * and unique-string concatenation must not alter either snapshot. Pending + * exceptions still see frame-owned inputs until every argument succeeds. */ +export function emitSplitSnapshot(host: LlvmEmitterContext, split: StringSplit): SplitSnapshot { + const source = host.emitExpr(split.receiver); + const separator = host.emitExpr(split.args[0]!); + const limit = host.emitExpr(split.args[1]!); + host.declare("declare i32 @scr_to_uint32(double)"); + const converted = host.B.tmp(); + host.B.line(`${converted} = call i32 @scr_to_uint32(double ${limit.name})`); + return { source: source.name, separator: separator.name, limit: converted }; +} + +export interface StoredSplitSnapshot { + sourceSlot: string; + separatorSlot: string; + limitSlot: string; +} + +/** Move snapshots into the declaration's lexical scope. Their ownership + * then participates in normal, jumping and exceptional scope cleanup. */ +export function storeSplitSnapshot( + host: LlvmEmitterContext, + snapshot: SplitSnapshot, +): StoredSplitSnapshot { + const sourceSlot = host.B.slot(), + separatorSlot = host.B.slot(), + limitSlot = host.B.slot(); + host.B.entryAllocas.push( + `${sourceSlot} = alloca ptr`, + `${separatorSlot} = alloca ptr`, + `${limitSlot} = alloca i32`, + ); + host.B.line(`store ptr ${snapshot.source}, ptr ${sourceSlot}`); + host.B.line(`store ptr ${snapshot.separator}, ptr ${separatorSlot}`); + host.B.line(`store i32 ${snapshot.limit}, ptr ${limitSlot}`); + host.moveTemp({ name: snapshot.source, type: STRING }); + host.moveTemp({ name: snapshot.separator, type: STRING }); + return { sourceSlot, separatorSlot, limitSlot }; +} + +export function loadSplitSnapshot( + host: LlvmEmitterContext, + snapshot: StoredSplitSnapshot, +): SplitSnapshot { + const source = host.B.tmp(), + separator = host.B.tmp(), + limit = host.B.tmp(); + host.B.line(`${source} = load ptr, ptr ${snapshot.sourceSlot}`); + host.B.line(`${separator} = load ptr, ptr ${snapshot.separatorSlot}`); + host.B.line(`${limit} = load i32, ptr ${snapshot.limitSlot}`); + return { source, separator, limit }; +} + +/** Each loop entry restarts a split result, including repeated execution + * of the single syntactic consumer. No cursor owns heap memory. */ +export function emitSplitCursor(host: LlvmEmitterContext, snapshot: SplitSnapshot): string { + const cursor = host.B.slot(); + host.B.entryAllocas.push(`${cursor} = alloca { ${host.sizeType}, i32, i32 }`); + host.declare("declare void @scr_str_split_cursor_init(ptr, i32)"); + host.B.line(`call void @scr_str_split_cursor_init(ptr ${cursor}, i32 ${snapshot.limit})`); + return cursor; +} + +export function emitSplitNext( + host: LlvmEmitterContext, + snapshot: SplitSnapshot, + cursor: string, + scratch: string, +): string { + const piece = host.B.tmp(); + host.declare("declare ptr @scr_str_split_cursor_next(ptr, ptr, ptr, ptr)"); + host.B.line( + `${piece} = call ptr @scr_str_split_cursor_next(ptr ${snapshot.source}, ptr ${snapshot.separator}, ptr ${cursor}, ptr ${scratch})`, + ); + return piece; +} + +export function emitSplitScratch(host: LlvmEmitterContext): string { + const scratch = host.B.slot(); + host.B.entryAllocas.push(`${scratch} = alloca ptr`); + host.B.line(`store ptr null, ptr ${scratch}`); + host.ownSlot(scratch, STRING); + return scratch; +} diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 29e333ecf..8698f4ff3 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -2,6 +2,7 @@ import { dynUndefinedExpr, numLit, varRef } from "../../ir/build.js"; import { lowerDynObjectLiteral } from "./expressions/object-literals.js"; import { importMetaBindingSource } from "./import-meta.js"; import { iteratorCanStep, iteratorValue } from "./iterator-consumption.js"; +import { isPrivateSplitConsumer } from "./split-consumers.js"; import { InternalCompilerError } from "../../errors.js"; /* Statement lowering: the statement dispatch (lowerStmt), variable * declarations including destructuring patterns, scoped blocks, control @@ -8923,7 +8924,9 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { } } const elementType = iterable.type.kind === "array" ? iterable.type.elem : tupleSource!.elem; - const yieldedT = tupleSource === null ? arrayValueType(lowerer, elementType) : elementType; + const privateSplit = !awaitArray && isPrivateSplitConsumer(lowerer, iterSrc, iterable); + const yieldedT = + tupleSource === null && !privateSplit ? arrayValueType(lowerer, elementType) : elementType; let elemValueT = yieldedT; let awaitPromiseTag: number | null = null; if (awaitArray) { @@ -8978,12 +8981,15 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { } // Native arrays retain their payload ABI; the loop binds the value yielded // by Get, which is undefined for both holes and present undefined slots. - const sourceT = iterable.type; - const source = lowerer.declareHiddenLocal("%arrayIterator", sourceT); - const cursor = lowerer.declareHiddenLocal("%arrayCursor", F64); - cursor.mutable = true; const forValues = (localId: string, body: IrStmt[]): IrStmt => { const loc = locOf(stmt); + if (privateSplit) { + return { kind: "forOf", localId, iterable, body, ...(labels && { labels }), loc }; + } + const sourceT = iterable.type; + const source = lowerer.declareHiddenLocal("%arrayIterator", sourceT); + const cursor = lowerer.declareHiddenLocal("%arrayCursor", F64); + cursor.mutable = true; const needsClose = awaitArray ? lowerer.declareHiddenLocal("%arrayIteratorNeedsClose", BOOL) : null; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index e19fe690e..38c2b71f1 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2201,6 +2201,7 @@ export class Lowerer { * (the ctx guard keeps hidden locals out of closures — a cross-function * walk falls back to the plain array walk and the fence). */ readonly matchAllDrainIndexes = new Map(); + readonly splitIdentifierUses = new Map>(); /** STORED numeric value iterators: `const it = numbers.values()` (and * the equivalent `[Symbol.iterator]()` spelling) over number[] or a * represented typed array has no first-class IR value, so its statically diff --git a/packages/compiler/src/frontend/lowering/split-consumers.ts b/packages/compiler/src/frontend/lowering/split-consumers.ts new file mode 100644 index 000000000..7d8a6d8bd --- /dev/null +++ b/packages/compiler/src/frontend/lowering/split-consumers.ts @@ -0,0 +1,66 @@ +import * as ts from "../ts7/adapter.js"; +import type { IrExpr } from "../../ir/ir.js"; +import type { Lowerer } from "./lowerer.js"; +import { stripValueWrappers } from "./binding-analysis.js"; + +/** A fresh split array is dense. A stored result has the same guarantee + * only when this loop is its sole observation: even a const array can be + * mutated through an alias. Keep the ordinary array protocol otherwise. */ +export function isPrivateSplitConsumer( + lowerer: Lowerer, + source: ts.Expression, + iterable: IrExpr, +): boolean { + if (iterable.type.kind !== "array" || iterable.type.elem.kind !== "string") return false; + if (iterable.kind === "strIntrinsic" && iterable.method === "split") return true; + const name = stripValueWrappers(source); + if (!ts.isIdentifier(name) || iterable.kind !== "varRef") return false; + const symbol = lowerer.resolveValueSymbol(name); + if (!symbol) return false; + const local = lowerer.bindingIn(lowerer.ctx, symbol); + if (!local || local.boxed || local.tdz || local.mutable || local.id !== iterable.localId) + return false; + const decl = lowerer.checker.valueDeclarationOf(symbol); + if ( + !decl || + !ts.isVariableDeclaration(decl) || + !ts.isIdentifier(decl.name) || + !decl.initializer || + !ts.isVariableDeclarationList(decl.parent) || + (decl.parent.flags & ts.NodeFlags.Const) === 0 + ) + return false; + const init = stripValueWrappers(decl.initializer); + if ( + !ts.isCallExpression(init) || + init.questionDotToken || + !ts.isPropertyAccessExpression(init.expression) || + init.expression.questionDotToken || + init.expression.name.text !== "split" || + !lowerer.isStdlibMember(init.expression) || + lowerer.mapTypeOf(lowerer.typeOf(init.expression.expression))?.kind !== "string" || + init.arguments.length === 0 || + lowerer.mapTypeOf(lowerer.typeOf(init.arguments[0]!))?.kind !== "string" + ) + return false; + + // Index spelling once per file; resolve only this binding's names. Type + // positions and shorthand properties conservatively count as uses too. + const file = decl.getSourceFile(); + let names = lowerer.splitIdentifierUses.get(file); + if (!names) { + names = new Map(); + ts.walkPreorder(file, (node) => { + if (!ts.isIdentifier(node)) return; + const uses = names!.get(node.text); + if (uses) uses.push(node); + else names!.set(node.text, [node]); + }); + lowerer.splitIdentifierUses.set(file, names); + } + for (const use of names.get(name.text) ?? []) { + if (use === decl.name || use === name) continue; + if (lowerer.resolveValueSymbol(use) === symbol) return false; + } + return true; +} diff --git a/packages/compiler/test/split-loop-emission.test.ts b/packages/compiler/test/split-loop-emission.test.ts new file mode 100644 index 000000000..ec7415fa8 --- /dev/null +++ b/packages/compiler/test/split-loop-emission.test.ts @@ -0,0 +1,103 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule, serializeModule, validateModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; + +test("private split loops keep dense bindings and target-sized snapshot cursors", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-split-loops-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile( + entry, + ` +function direct(input: string, limit: number): string { + let result = ""; + for (const piece of input.split(",", limit)) result += piece; + return result; +} +function stored(input: string, limit: number): string { + const pieces = input.split(",", limit); + input += ",changed"; + let result = ""; + for (let pass = 0; pass < 2; pass++) { + for (const piece of pieces) result += piece; + } + return result; +} +function observed(input: string): string { + const pieces = input.split(","); + pieces.push("extra"); + let result = ""; + for (const piece of pieces) result += piece; + return result; +} +function captured(input: string): string { + const pieces = input.split(","); + const use = (): number => pieces.length; + let result = ""; + for (const piece of pieces) result += piece; + return result + String(use()); +} +async function suspended(input: string): Promise { + let result = ""; + for (const piece of input.split(",")) { await Promise.resolve(); result += piece; } + return result; +} +console.log(direct("a,b", 2), stored("a,b", 2), observed("a,b"), captured("a,b")); +suspended("a,b").then(console.log); +`, + ); + const result = await compile(entry, { + outDir: dir, + outPath: output, + outputKind: "ir", + dynamic: false, + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const module = deserializeModule(await readFile(output, "utf8")); + expect(validateModule(module)).toEqual([]); + const body = (llvm: string, name: string): string => { + const definition = new RegExp( + `^define internal [^\\n]*@sc_(?:bf|f)_${name}\\([^]*?^}`, + "m", + ).exec(llvm); + expect(definition, name).not.toBeNull(); + return definition![0]; + }; + for (const pointerBits of [32, 64] as const) { + const llvm = emitLlvmModule(deserializeModule(serializeModule(module)), { pointerBits }); + for (const name of ["direct", "stored"]) { + const work = body(llvm, name); + expect(work).toContain(`alloca { i${pointerBits}, i32, i32 }`); + expect(work).toContain("@scr_str_split_cursor_next"); + expect(work).not.toContain("@scr_str_split_limit"); + expect(work).not.toContain("@scr_arr_get_ref"); + expect(work).not.toContain("@scr_union_wrap"); + } + const stored = body(llvm, "stored"); + expect(stored.indexOf("@scr_to_uint32")).toBeLessThan(stored.indexOf("@scr_str_concat")); + expect(stored.indexOf("@scr_str_concat")).toBeLessThan( + stored.indexOf("@scr_str_split_cursor_init"), + ); + for (const name of ["observed", "captured", "suspended"]) { + const work = body(llvm, name); + expect(work).toContain("@scr_str_split_limit"); + expect(work).not.toContain("@scr_str_split_cursor_next"); + } + const debug = emitLlvmModule(deserializeModule(serializeModule(module)), { + pointerBits, + debugSources: new Map([[entry, await readFile(entry, "utf8")]]), + }); + for (const name of ["direct", "stored"]) { + expect(body(debug, name)).toContain("@scr_str_split_limit"); + expect(body(debug, name)).not.toContain("@scr_str_split_cursor_next"); + } + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index bf4746414..a527d6f64 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -11176,6 +11176,12 @@ ], "diags": [] }, + "/tests/corpus/string-split-traversal.ts": { + "order": [ + "/tests/corpus/string-split-traversal.ts" + ], + "diags": [] + }, "/tests/corpus/structural-class-properties.ts": { "order": [ "/tests/corpus/structural-class-properties.ts" diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index fb562cb65..e063374a1 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -805,6 +805,20 @@ ScrStr *scr_str_trim_end(ScrStr *s); struct ScrArr; struct ScrArr *scr_str_split(ScrStr *s, ScrStr *sep); struct ScrArr *scr_str_split_limit(ScrStr *s, ScrStr *sep, double limit); +/* Internal dense split traversal. The cursor borrows input strings, returns + * one owned piece per step, and uses NULL only for exhaustion. Its layout + * is shared with generated stack storage, including wasm32. */ +typedef struct ScrSplitCursor { + size_t offset; + uint32_t remaining; + uint32_t pending_unit; +} ScrSplitCursor; +void scr_str_split_cursor_init(ScrSplitCursor *cursor, uint32_t limit); +/* Optional scratch is a separately owned, NULL-initialized string slot. + * A piece that has not escaped can reuse it after the preceding yield is + * released. The caller releases the slot on every exit; NULL disables reuse. */ +ScrStr *scr_str_split_cursor_next(ScrStr *s, ScrStr *sep, ScrSplitCursor *cursor, + ScrStr **scratch); /* padStart(maxLength, fill)/padEnd — ECMA StringPad, UTF-16 unit counts. * Target at or below the length (or empty fill) returns the receiver diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index a3a51162d..6509729e7 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -1317,15 +1317,6 @@ static const struct { size_t rc; size_t len; size_t cap; char data[4]; } /* Split owns a fresh array and appends only present string values. Fill an * available dense slot directly; ordinary push handles growth and the sparse * boundary. The array remains valid after every append, including RC audit. */ -static void scr_str_split_append(ScrArr *out, ScrStr *value) { - if (out->len < out->cap) { - out->data[out->len] = (uint64_t)(uintptr_t)value; - out->present[out->len++] = SCR_ARR_VALUE; - } else { - scr_arr_push_ref(out, value); - } -} - /* split(separator, limit) with a STRING separator (ECMA-262 22.1.3.23): * limit is ToUint32'd; zero returns [] and reaching the limit stops before * any later separator probes. The no-limit wrapper supplies 2^32-1. @@ -1338,38 +1329,107 @@ static void scr_str_split_append(ScrArr *out, ScrStr *value) { * yield the two lone surrogate halves, each half is U+FFFD here * (divergence 2 — the same substitution the island's boundary marshal * applied). Borrows both; returns a +1 string[]. */ -ScrArr *scr_str_split_limit(ScrStr *s, ScrStr *sep, double limit_num) { - ScrArr *out = scr_arr_new(SCR_ELEM_STR, 0); - uint32_t limit = scr_to_uint32(limit_num); - if (limit == 0) return out; - if (sep->len == 0) { - size_t i = 0; - while (i < s->len) { - size_t adv; - uint32_t cp = scr_utf8_decode(s->data + i, &adv); - if (cp >= 0x10000) { /* two units in JS: both halves become U+FFFD */ - scr_str_split_append(out, scr_str_retain((ScrStr *)&scr_lit_fffd)); - if (out->len == limit) return out; - scr_str_split_append(out, scr_str_retain((ScrStr *)&scr_lit_fffd)); - } else { - scr_str_split_append(out, scr_str_from_span(s->data + i, adv)); - } - if (out->len == limit) return out; - i += adv; +void scr_str_split_cursor_init(ScrSplitCursor *cursor, uint32_t limit) { + cursor->offset = 0; + cursor->remaining = limit; + cursor->pending_unit = 0; +} + +static ScrStr *scr_str_split_piece(const char *bytes, size_t len, ScrStr **scratch) { + if (!scratch || len <= 4) return scr_str_from_span(bytes, len); + ScrStr *piece = *scratch; + if (piece && piece->rc == 1 && piece->cap >= len) { + /* The scratch owner is the only remaining reference. Invalidate all + * metadata before replacing bytes, just as unique concatenation does. */ + scr_short_forget(piece); + scr_sidx_purge(piece); + memcpy(piece->data, bytes, len); + piece->len = len; + piece->data[len] = '\0'; + } else { + ScrStr *fresh = scr_str_alloc_raw(len, len < 64 ? 64 : len); + memcpy(fresh->data, bytes, len); + fresh->data[len] = '\0'; + *scratch = fresh; + scr_str_release(piece); + piece = fresh; + } + return scr_str_retain(piece); +} + +/* Both eager and loop consumers use the same boundary state machine. + * Eager consumers select the unit/substring path once before their loop. */ +static const char *scr_str_split_span(ScrStr *s, ScrStr *sep, + ScrSplitCursor *cursor, bool units, + size_t *len) { + if (cursor->remaining == 0 || cursor->offset > s->len) return NULL; + if (units) { + if (cursor->pending_unit) { + cursor->pending_unit = 0; + cursor->remaining--; + *len = 3; + return scr_lit_fffd.data; } - return out; + if (cursor->offset == s->len) return NULL; + size_t start = cursor->offset, adv; + uint32_t cp = scr_utf8_decode(s->data + start, &adv); + cursor->offset += adv; + cursor->remaining--; + if (cp >= 0x10000) { + cursor->pending_unit = 1; + *len = 3; + return scr_lit_fffd.data; + } + *len = adv; + return s->data + start; } - size_t start = 0; + size_t start = cursor->offset; + const char *found = scr_byte_find(s->data + start, s->len - start, + sep->data, sep->len); + size_t end = found ? (size_t)(found - s->data) : s->len; + /* String allocations reserve the header and terminator, so len + 1 + * cannot overflow. This sentinel includes a final empty trailing piece. */ + cursor->offset = found ? end + sep->len : s->len + 1; + cursor->remaining--; + *len = end - start; + return s->data + start; +} + +ScrStr *scr_str_split_cursor_next(ScrStr *s, ScrStr *sep, ScrSplitCursor *cursor, + ScrStr **scratch) { + size_t len; + const char *bytes = scr_str_split_span(s, sep, cursor, sep->len == 0, &len); + if (!bytes) return NULL; + if (bytes == scr_lit_fffd.data) return scr_str_retain((ScrStr *)&scr_lit_fffd); + return scr_str_split_piece(bytes, len, scratch); +} + +/* Move a bounded batch of pieces into array storage in one operation. + * This avoids a second separator scan and preserves sparse overflow. */ +static void scr_str_split_fill(ScrArr *out, ScrStr *s, ScrStr *sep, + ScrSplitCursor *cursor, bool units) { + uint64_t slots[64]; for (;;) { - const char *found = scr_byte_find(s->data + start, s->len - start, - sep->data, sep->len); - if (!found) break; - size_t at = (size_t)(found - s->data); - scr_str_split_append(out, scr_str_from_span(s->data + start, at - start)); - if (out->len == limit) return out; - start = at + sep->len; + size_t count = 0, len; + const char *bytes; + while (count < 64 && (bytes = scr_str_split_span(s, sep, cursor, units, &len))) { + ScrStr *piece = bytes == scr_lit_fffd.data + ? scr_str_retain((ScrStr *)&scr_lit_fffd) + : scr_str_from_span(bytes, len); + slots[count++] = (uint64_t)(uintptr_t)piece; + } + if (count) scr_arr_push_many(out, count, slots); + if (count < 64) return; } - scr_str_split_append(out, scr_str_from_span(s->data + start, s->len - start)); +} + +ScrArr *scr_str_split_limit(ScrStr *s, ScrStr *sep, double limit_num) { + uint32_t limit = scr_to_uint32(limit_num); + ScrArr *out = scr_arr_new(SCR_ELEM_STR, 0); + ScrSplitCursor cursor; + scr_str_split_cursor_init(&cursor, limit); + if (sep->len == 0) scr_str_split_fill(out, s, sep, &cursor, true); + else scr_str_split_fill(out, s, sep, &cursor, false); return out; } diff --git a/packages/runtime/test/gen-string-cases.mjs b/packages/runtime/test/gen-string-cases.mjs index 490dbd2af..4d74c63b3 100644 --- a/packages/runtime/test/gen-string-cases.mjs +++ b/packages/runtime/test/gen-string-cases.mjs @@ -78,6 +78,26 @@ function emit(op, input, args, expected) { lines.add(`${op}\t${hex(input)}\t${args}\t${expected}`); } +// Pin conversion and early exhaustion separately from the broad method +// oracle: limits can stop between the surrogate halves of an astral unit. +for (const [input, separator] of [ + ["", ","], + [",a,,b,", ","], + ["ababa", "aba"], + ["a\0b\0", "\0"], + ["a😀b", ""], +]) { + for (const limit of [0, 1, 2, 3, 7, -1, 1.9, NaN, Infinity, 4294967296, 4294967297]) { + const pieces = input.split(separator, limit); + emit( + "splitLimit", + input, + `${hex(separator)},${num(limit)}`, + hex(`${pieces.length}:${pieces.join("\x01")}`), + ); + } +} + for (const s of strings) { const L = s.length; const idxs = [ diff --git a/packages/runtime/test/string-cases.txt b/packages/runtime/test/string-cases.txt index 74c17ff05..651d581e1 100644 --- a/packages/runtime/test/string-cases.txt +++ b/packages/runtime/test/string-cases.txt @@ -1,3 +1,58 @@ +splitLimit - 2c,0 303a +splitLimit - 2c,1 313a +splitLimit - 2c,2 313a +splitLimit - 2c,3 313a +splitLimit - 2c,7 313a +splitLimit - 2c,-1 313a +splitLimit - 2c,1.9 313a +splitLimit - 2c,NaN 303a +splitLimit - 2c,Infinity 303a +splitLimit - 2c,4294967296 303a +splitLimit - 2c,4294967297 313a +splitLimit 2c612c2c622c 2c,0 303a +splitLimit 2c612c2c622c 2c,1 313a +splitLimit 2c612c2c622c 2c,2 323a0161 +splitLimit 2c612c2c622c 2c,3 333a016101 +splitLimit 2c612c2c622c 2c,7 353a016101016201 +splitLimit 2c612c2c622c 2c,-1 353a016101016201 +splitLimit 2c612c2c622c 2c,1.9 313a +splitLimit 2c612c2c622c 2c,NaN 303a +splitLimit 2c612c2c622c 2c,Infinity 303a +splitLimit 2c612c2c622c 2c,4294967296 303a +splitLimit 2c612c2c622c 2c,4294967297 313a +splitLimit 6162616261 616261,0 303a +splitLimit 6162616261 616261,1 313a +splitLimit 6162616261 616261,2 323a016261 +splitLimit 6162616261 616261,3 323a016261 +splitLimit 6162616261 616261,7 323a016261 +splitLimit 6162616261 616261,-1 323a016261 +splitLimit 6162616261 616261,1.9 313a +splitLimit 6162616261 616261,NaN 303a +splitLimit 6162616261 616261,Infinity 303a +splitLimit 6162616261 616261,4294967296 303a +splitLimit 6162616261 616261,4294967297 313a +splitLimit 61006200 00,0 303a +splitLimit 61006200 00,1 313a61 +splitLimit 61006200 00,2 323a610162 +splitLimit 61006200 00,3 333a61016201 +splitLimit 61006200 00,7 333a61016201 +splitLimit 61006200 00,-1 333a61016201 +splitLimit 61006200 00,1.9 313a61 +splitLimit 61006200 00,NaN 303a +splitLimit 61006200 00,Infinity 303a +splitLimit 61006200 00,4294967296 303a +splitLimit 61006200 00,4294967297 313a61 +splitLimit 61f09f988062 -,0 303a +splitLimit 61f09f988062 -,1 313a61 +splitLimit 61f09f988062 -,2 323a6101efbfbd +splitLimit 61f09f988062 -,3 333a6101efbfbd01efbfbd +splitLimit 61f09f988062 -,7 343a6101efbfbd01efbfbd0162 +splitLimit 61f09f988062 -,-1 343a6101efbfbd01efbfbd0162 +splitLimit 61f09f988062 -,1.9 313a61 +splitLimit 61f09f988062 -,NaN 303a +splitLimit 61f09f988062 -,Infinity 303a +splitLimit 61f09f988062 -,4294967296 303a +splitLimit 61f09f988062 -,4294967297 313a61 len - - 30 trim - - - trimStart - - - diff --git a/packages/runtime/test/test_string.c b/packages/runtime/test/test_string.c index 609ac34f7..5bbeec841 100644 --- a/packages/runtime/test/test_string.c +++ b/packages/runtime/test/test_string.c @@ -200,6 +200,40 @@ static void split_storage_asserts(void) { scr_str_release(input); } +static void split_scratch_asserts(void) { + _Static_assert(offsetof(ScrSplitCursor, offset) == 0, "split cursor offset"); + _Static_assert(offsetof(ScrSplitCursor, remaining) == sizeof(size_t), "split cursor limit"); + _Static_assert(offsetof(ScrSplitCursor, pending_unit) == sizeof(size_t) + 4, "split cursor pending unit"); + _Static_assert(sizeof(ScrSplitCursor) == sizeof(size_t) + 8, "split cursor storage"); + ScrStr *input = scr_str_new("alpha,beta-long,\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8C,test-tail,,last", 44); + ScrStr *separator = scr_str_new(",", 1), *scratch = NULL; + ScrSplitCursor cursor; + scr_str_split_cursor_init(&cursor, UINT32_MAX); + ScrStr *first = scr_str_split_cursor_next(input, separator, &cursor, &scratch); + total++; + if (first != scratch || first->rc != 2 || scr_str_utf16_len(first) != 5) failed++; + scr_str_release(first); + ScrStr *saved = scr_str_split_cursor_next(input, separator, &cursor, &scratch); + total++; + if (saved != first || scr_str_utf16_len(saved) != 9) failed++; + /* Keep this yield alive: the next piece must allocate independent storage. */ + ScrStr *mixed = scr_str_split_cursor_next(input, separator, &cursor, &scratch); + total++; + if (mixed == saved || scr_str_utf16_len(mixed) != 4 || + saved->len != 9 || memcmp(saved->data, "beta-long", 9) != 0) failed++; + scr_str_release(mixed); + ScrStr *tail = scr_str_split_cursor_next(input, separator, &cursor, &scratch); + total++; + if (tail != mixed || scr_str_utf16_len(tail) != 9) failed++; + scr_str_release(tail); + while ((tail = scr_str_split_cursor_next(input, separator, &cursor, &scratch))) + scr_str_release(tail); + scr_str_release(saved); + scr_str_release(scratch); + scr_str_release(separator); + scr_str_release(input); +} + static void construction_asserts(void) { ScrStr *empty = scr_str_new("", 0); ScrStr *value = scr_str_new("a\0\xE6\x97\xA5", 5); @@ -802,12 +836,16 @@ int main(int argc, char **argv) { } else if (strcmp(op, "parseInt") == 0) { check_f64(op, args, input, scr_parse_int(input, strtod(args, NULL)), expected_bytes, exp_len); - } else if (strcmp(op, "split") == 0) { + } else if (strcmp(op, "split") == 0 || strcmp(op, "splitLimit") == 0) { /* args = separator hex; expected = ":" */ + double limit = 4294967295.0; + char *comma = strcmp(op, "splitLimit") == 0 ? strchr(args, ',') : NULL; + if (comma) { *comma = '\0'; limit = strtod(comma + 1, NULL); } size_t sep_len = hex_decode(args, needle_bytes); + if (comma) *comma = ','; if (sep_len == (size_t)-1) goto badline_release; ScrStr *sep = scr_str_new(needle_bytes, sep_len); - ScrArr *pieces = scr_str_split(input, sep); + ScrArr *pieces = scr_str_split_limit(input, sep, limit); size_t count = (size_t)scr_arr_len(pieces); size_t cap = 32; for (size_t i = 0; i < count; i++) { @@ -825,6 +863,30 @@ int main(int argc, char **argv) { scr_str_release(p); } check(op, args, input, joined, o, expected_bytes, exp_len); + /* Consume and release one piece at a time. Empty pieces must remain + * distinct from exhaustion, and resetting must restart the walk. */ + ScrSplitCursor cursor; + for (int pass = 0; pass < 2; pass++) { + scr_str_split_cursor_init(&cursor, scr_to_uint32(limit)); + ScrStr *scratch = NULL; + size_t n = 0, used = 32; + ScrStr *piece; + while ((piece = scr_str_split_cursor_next(input, sep, &cursor, &scratch))) { + if (n) joined[used++] = '\x01'; + memcpy(joined + used, piece->data, piece->len); + used += piece->len; + n++; + scr_str_release(piece); + } + size_t prefix = (size_t)snprintf(joined, 32, "%zu:", n); + total++; + if (n != count || prefix > exp_len || used - 32 != exp_len - prefix || + memcmp(joined + 32, expected_bytes + prefix, used - 32) != 0) + failed++; + total++; + if (scr_str_split_cursor_next(input, sep, &cursor, &scratch) != NULL) failed++; + scr_str_release(scratch); + } free(joined); scr_arr_release(pieces); scr_str_release(sep); @@ -889,6 +951,7 @@ int main(int argc, char **argv) { divergence_asserts(); construction_asserts(); split_storage_asserts(); + split_scratch_asserts(); accumulation_asserts(); short_string_asserts(); #ifdef SCR_SIDX_TEST diff --git a/tests/corpus/string-split-traversal.ts b/tests/corpus/string-split-traversal.ts new file mode 100644 index 000000000..eebc620f9 --- /dev/null +++ b/tests/corpus/string-split-traversal.ts @@ -0,0 +1,131 @@ +function collect(input: string, separator: string, limit: number): string { + let result = ""; + for (const piece of input.split(separator, limit)) result += "[" + piece + "]"; + return result; +} + +for (const limit of [0, 1, 2, 7, -1, 1.9, NaN, Infinity, 4294967296, 4294967297]) { + console.log("limit", limit, collect(",a,,b,", ",", limit)); +} +console.log("empty", collect("", "", 9), collect("", ",", 9)); +console.log("overlap", collect("ababa", "aba", 9)); +console.log("unicode", collect("é世界é中é", "é", 9), collect("世界", "", 9)); +console.log("zero", collect("a\0b\0", "\0", 9)); + +function snapshot(): string { + let input = "left,right"; + let separator = ","; + let limit = 2; + const pieces = input.split(separator, limit); + input += ",changed"; + separator += "changed"; + limit = 0; + let result = ""; + for (let pass = 0; pass < 3; pass++) { + for (const piece of pieces) result += "[" + piece + "]"; + } + return result + ":" + input + ":" + separator + ":" + limit; +} +console.log("snapshot", snapshot()); + +let receiver = "first,second"; +let effects = ""; +function source(): string { effects += "s"; return receiver; } +function separator(): string { effects += "p"; receiver += ",late"; return ","; } +function limit(): number { effects += "l"; return 2; } +let evaluated = ""; +for (const piece of source().split(separator(), limit())) { + receiver += ",body"; + evaluated += "[" + piece + "]"; +} +console.log("order", effects, evaluated, receiver); + +function failingSeparator(): string { throw new Error("separator"); } +function failingLimit(): number { throw new Error("limit"); } +try { + for (const piece of (receiver + ",temporary").split(failingSeparator())) console.log(piece); +} catch (error) { console.log("argument", (error as Error).message); } +try { + for (const piece of (receiver + ",temporary").split(receiver + ",separator", failingLimit())) console.log(piece); +} catch (error) { console.log("argument", (error as Error).message); } + +function bindings(): string { + const readers: (() => string)[] = []; + for (let piece of "first-word,second-word,third-word".split(",")) { + piece += "!"; + readers.push(() => piece); + } + let result = ""; + for (const read of readers) result += read(); + const shared: (() => string)[] = []; + for (var piece of "four,five".split(",")) shared.push(() => piece); + for (const read of shared) result += read(); + let existing = "unchanged"; + for (existing of "six,seven".split(",")) result += existing; + result += existing; + for (const [first, second = "_"] of "ab,c".split(",")) result += first + second; + return result; +} +console.log("bindings", bindings()); + +function exits(input: string): string { + const pieces = input.split(","); + let result = ""; + outer: for (const piece of pieces) { + try { + for (const inner of "a,b,c".split(",")) { + if (inner === "a") continue; + if (piece === "break") break outer; + if (piece === "return") return result; + if (piece === "throw") throw new Error("stop"); + result += piece + inner; + break; + } + } finally { result += "!"; } + } + return result; +} +console.log("break", exits("one,break,three")); +console.log("return", exits("one,return,three")); +try { console.log(exits("one,throw,three")); } catch (error) { console.log("throw", (error as Error).message); } + +function observable(): string { + const pieces = "one,two".split(","); + const alias = pieces; + alias.push("three"); + let result = ""; + for (const piece of pieces) { + result += piece; + if (piece === "one") pieces.push("four"); + } + return result + ":" + pieces.length; +} +console.log("observable", observable()); + +function escapes(): string { + const pieces = "one,two".split(","); + const read = (): string => pieces.join("/"); + let result = ""; + for (const piece of pieces) result += piece; + return result + ":" + read(); +} +console.log("escapes", escapes()); + +async function suspended(): Promise { + let input = "one,two"; + const pieces = input.split(","); + input += ",late"; + let result = ""; + for (const piece of pieces) { + await Promise.resolve(); + result += piece; + } + for await (const piece of "three,four".split(",")) result += piece; + return result; +} +suspended().then((value) => console.log("async", value)); + +function* generator(): Generator { + for (const piece of "one,two".split(",")) yield piece; +} +for (const piece of generator()) console.log("generator", piece); diff --git a/tests/harness/self-hosting-llvm-emitter.test.ts b/tests/harness/self-hosting-llvm-emitter.test.ts index e38fd601f..99f9f4aaf 100644 --- a/tests/harness/self-hosting-llvm-emitter.test.ts +++ b/tests/harness/self-hosting-llvm-emitter.test.ts @@ -101,6 +101,7 @@ const programs = [ "3120-ir-nonfinite-numbers.ts", "4031-event-emitter-long-tuples.ts", "text-codec-values/main.ts", + "string-split-traversal.ts", "heap-input-aliases.ts", "heap-input-collections.ts", "input-lifetime-intervals.ts", diff --git a/tests/harness/wasi-differential.test.ts b/tests/harness/wasi-differential.test.ts index 42863d888..33471a2e7 100644 --- a/tests/harness/wasi-differential.test.ts +++ b/tests/harness/wasi-differential.test.ts @@ -157,6 +157,7 @@ describe.skipIf(!zigOnPath())("wasm32-wasi differential", () => { "1002-json-parse-cast.ts", "1010-json-stringify-space.ts", "json-output-bulk.ts", + "string-split-traversal.ts", "1565-union-element-join.ts", "1020-async-basics.ts", "1021-async-ordering.ts",