diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 520964cf3..af3d3d239 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -30,6 +30,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | | `records-sort` | cpu | comparator sorts, group-by, `reduce`/`filter` | | `regex-logs` | cpu | regex `exec` with groups, `test`, global `replace` | +| `regex-replacers` | cpu | regex `replace`/`replaceAll` with inline callbacks whose parameters have no annotations: dictionary placeholders, snake_case to camelCase, date reordering, optional captures, capitalization, and a string search that reads the offset | | `region-builds` | cpu | a watch-mode build loop over `LifetimeRegion`: each build binds a program in a new region and releases the previous one, which a cache keeps referenced for two builds | | `template-render` | cpu | template literals, escaping, `map`/`join` | | `throw-chains` | cpu | exceptions caught 2, 5 and 20 recursive calls above the throw, through direct calls and through a callback, and a backtracking parser that throws on a failed alternative | diff --git a/benchmarks/runtime/workloads/regex-replacers.json b/benchmarks/runtime/workloads/regex-replacers.json new file mode 100644 index 000000000..c01c6c272 --- /dev/null +++ b/benchmarks/runtime/workloads/regex-replacers.json @@ -0,0 +1,6 @@ +{ + "entry": "workloads/regex-replacers.ts", + "args": [ + "1" + ] +} diff --git a/benchmarks/runtime/workloads/regex-replacers.ts b/benchmarks/runtime/workloads/regex-replacers.ts new file mode 100644 index 000000000..be3820b89 --- /dev/null +++ b/benchmarks/runtime/workloads/regex-replacers.ts @@ -0,0 +1,60 @@ +// Regex replace and replaceAll with inline callbacks, written as they +// usually are, without parameter annotations: template placeholders looked +// up in a dictionary, snake_case to camelCase, date reordering, optional +// version parts, capitalization and a string search with its offset. +let seed = 7; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function pick(items: readonly T[]): T { + return items[Math.floor(random() * items.length)]!; +} + +const words = ["user", "order", "item", "cart", "price", "total", "status", "created", "updated", "id"]; +const names = ["Ada", "Grace", "Linus", "Barbara", "Ken", "Margaret"]; +const vars: Record = { + name: "world", + city: "Lisbon", + day: "Tuesday", + plan: "pro", + count: "12", +}; +const keys = [...Object.keys(vars), "missing"]; + +const scale = Number(process.argv[2] ?? "1"); +const count = Math.floor(20000 * scale); +const templates: string[] = []; +const identifiers: string[] = []; +const dates: string[] = []; +const versions: string[] = []; +const sentences: string[] = []; +for (let i = 0; i < count; i++) { + templates.push(`Hello {${pick(keys)}}, your {${pick(keys)}} plan renews on {${pick(keys)}} in {${pick(keys)}}.`); + identifiers.push(`${pick(words)}_${pick(words)}_${pick(words)}${random() < 0.5 ? "_" + pick(words) : ""}`); + const year = 1990 + Math.floor(random() * 40); + const month = String(1 + Math.floor(random() * 12)).padStart(2, "0"); + const day = String(1 + Math.floor(random() * 28)).padStart(2, "0"); + dates.push(`from ${year}-${month}-${day} to ${year + 1}-${month}-${day} (${pick(names)})`); + versions.push(`v${i % 9}.${i % 13}${random() < 0.5 ? "." + (i % 7) : ""} and v${i % 5}.${i % 3}`); + sentences.push(`${pick(names)} said the ${pick(words)} and the ${pick(words)} were ready for ${pick(names)}`); +} + +let length = 0; +let checksum = 0; +for (let round = 0; round < 4; round++) { + for (let i = 0; i < count; i++) { + const filled = templates[i]!.replace(/\{(\w+)\}/g, (_m, key) => vars[key] ?? `<${key}>`); + const camel = identifiers[i]!.replace(/_([a-z])/g, (_m, c) => c.toUpperCase()); + const reordered = dates[i]!.replace(/(\d{4})-(\d{2})-(\d{2})/g, (_m, y, m, d) => `${d}/${m}/${y}`); + const padded = versions[i]!.replace(/v(\d+)\.(\d+)(?:\.(\d+))?/g, (_m, major, minor, patch) => `v${major}.${minor}.${patch ?? "0"}`); + const titled = sentences[i]!.replace(/\b[a-z]/g, (c) => c.toUpperCase()); + const marked = sentences[i]!.replaceAll("the", (m, at) => (at % 2 === 0 ? m.toUpperCase() : m)); + length += filled.length + camel.length + reordered.length + padded.length + titled.length + marked.length; + checksum = (checksum * 31 + camel.charCodeAt(camel.length - 1) + padded.charCodeAt(padded.length - 1)) >>> 0; + } +} +console.log(templates[0]!.replace(/\{(\w+)\}/g, (_m, key) => vars[key] ?? `<${key}>`)); +console.log(identifiers[1]!.replace(/_([a-z])/g, (_m, c) => c.toUpperCase())); +console.log(versions[2]!.replace(/v(\d+)\.(\d+)(?:\.(\d+))?/g, (_m, major, minor, patch) => `v${major}.${minor}.${patch ?? "0"}`)); +console.log("length", length, "checksum", checksum); diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 8a98fb154..4af8182a7 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -843,35 +843,68 @@ function emitAssertFailing( * `a.name.localeCompare(b.name)` reads both fields without retaining). */ const READ_ONLY_STRING_LIB_FNS: ReadonlySet = new Set(["str.localeCompare"]); -/** The parameter count of a replace callback the runtime can call through - * the closure ABI (scr_regex_replace_fn), or null when it needs the dyn - * box. Each declared parameter must receive a value of its declared kind - * in every match: the match string at 0; with more than one parameter the +/** The most parameters scr_regex_replace_fn calls a replacer with + * (SCR_REPLACER_MAX_ARITY), the first bit of its string-or-undefined + * parameters (SCR_REPLACER_OPTIONAL), and its flag for a checked-dynamic + * result (SCR_REPLACER_BOXED_RESULT). */ +const REPLACER_MAX_ARITY = 6; +const REPLACER_OPTIONAL = 2 ** 8; +const REPLACER_BOXED_RESULT = 2 ** 31; + +/** `string | undefined` held as the string pointer, NULL for undefined. */ +function optionalStringType(host: LlvmEmitterContext, type: IrType): boolean { + if (type.kind !== "union") return false; + const nullable = host.nullableUnions.get(type.unionId); + if (nullable?.repr !== "single" || nullable.arm.kind !== "string" || nullable.nullTag >= 0) + return false; + return host.unionsById.get(type.unionId)?.arms[nullable.unitTag]?.kind === "undefinedT"; +} + +/** The parameter count and kinds (scr_regex_replace_fn's `kinds`) of a + * replace callback the runtime can call through the closure ABI, or null + * when it needs the dyn box. A checked-dynamic parameter receives whatever + * Node passes at its position. Each typed parameter must receive a value of + * its declared kind in every match: the match string at 0; past 0 the * pattern must be a literal whose captures are known, so position i is a - * capture that every match sets (a string), the offset (a number) right - * after the captures, or the subject (a string) after that. */ -function typedReplacerArity(callback: IrType, pattern: IrExpr): number | null { + * capture (a string when every match sets it, otherwise `string | + * undefined`), the offset (a number) right after the captures, or the + * subject (a string) after that. The result is a string or a + * checked-dynamic value. */ +function typedReplacerShape( + host: LlvmEmitterContext, + callback: IrType, + pattern: IrExpr, +): { arity: number; kinds: number } | null { if (callback.kind !== "func" || callback.rest || callback.argumentsAll) return null; - if (callback.ret.kind !== "string") return null; + if (callback.ret.kind !== "string" && callback.ret.kind !== "dyn") return null; const params = callback.params; - if (params.length > 4) return null; - if (params.length === 0) return 0; - if (params[0]!.kind !== "string") return null; - if (params.length === 1) return 1; - if (pattern.kind !== "regexLit") return null; - const captures = regexLiteralCaptures(pattern.pattern, pattern.flags); - if (!captures) return null; - for (let i = 1; i < params.length; i++) { + if (params.length > REPLACER_MAX_ARITY) return null; + let kinds = callback.ret.kind === "dyn" ? REPLACER_BOXED_RESULT : 0; + let captures: { count: number; always: boolean[] } | null | undefined; + for (let i = 0; i < params.length; i++) { const kind = params[i]!.kind; + if (kind === "dyn") { + kinds += 2 ** i; + continue; + } + const optional = optionalStringType(host, params[i]!); + if (i === 0) { + if (kind !== "string") return null; + continue; + } + if (pattern.kind !== "regexLit") return null; + captures ??= regexLiteralCaptures(pattern.pattern, pattern.flags); + if (!captures) return null; const ok = i < captures.count - ? kind === "string" && captures.always[i] === true + ? (kind === "string" && captures.always[i] === true) || optional : i === captures.count ? kind === "f64" : i === captures.count + 1 && kind === "string"; if (!ok) return null; + if (optional) kinds += REPLACER_OPTIONAL * 2 ** i; } - return params.length; + return { arity: params.length, kinds }; } /** `s.replace(re, fn)` / `replaceAll` with a typed callback the runtime can @@ -881,16 +914,16 @@ function typedReplacerArity(callback: IrType, pattern: IrExpr): number | null { function emitTypedRegexReplace(host: LlvmEmitterContext, e: LibCallExpr): LlValue | null { const [subject, pattern, callback, all] = e.args; if (!subject || !pattern || callback?.kind !== "dynFrom" || all?.kind !== "boolLit") return null; - const arity = typedReplacerArity(callback.value.type, pattern); - if (arity === null) return null; + const shape = typedReplacerShape(host, callback.value.type, pattern); + if (shape === null) return null; const B = host.B; const s = host.emitExpr(subject); const re = host.emitExpr(pattern); const fn = host.emitExpr(callback.value); - host.declare(`declare ptr @scr_regex_replace_fn(ptr, ptr, ptr, i32, i1 zeroext)`); + host.declare(`declare ptr @scr_regex_replace_fn(ptr, ptr, ptr, i32, i32, i1 zeroext)`); const result = B.tmp(); B.line( - `${result} = call ptr @scr_regex_replace_fn(ptr ${s.name}, ptr ${re.name}, ptr ${fn.name}, i32 ${arity}, i1 zeroext ${all.value})`, + `${result} = call ptr @scr_regex_replace_fn(ptr ${s.name}, ptr ${re.name}, ptr ${fn.name}, i32 ${shape.arity}, i32 ${shape.kinds | 0}, i1 zeroext ${all.value})`, ); const output = host.own({ name: result, type: e.type }); host.emitPendingCheck(); diff --git a/packages/compiler/src/frontend/lowering/containers/replacer-params.test.ts b/packages/compiler/src/frontend/lowering/containers/replacer-params.test.ts new file mode 100644 index 000000000..1d5d376e9 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/containers/replacer-params.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "vitest"; +import { regexCaptureParticipation } from "./replacer-params.js"; + +describe("regex capture participation", () => { + test("counts captures and marks the ones every match sets", () => { + expect(regexCaptureParticipation("\\{(\\w+)\\}", "g")).toEqual({ + count: 2, + always: [true, true], + }); + expect(regexCaptureParticipation("(\\d{4})-(\\d{2})(?:-(\\d{2}))?", "g")).toEqual({ + count: 4, + always: [true, true, true, false], + }); + expect(regexCaptureParticipation("(a)|(b)", "")).toEqual({ + count: 3, + always: [true, false, false], + }); + expect(regexCaptureParticipation("((a)|b)", "")).toEqual({ + count: 3, + always: [true, true, false], + }); + expect(regexCaptureParticipation("(?:(a)|b)(c)", "")).toEqual({ + count: 3, + always: [true, false, true], + }); + expect(regexCaptureParticipation("(a)*(b)+(c){0,2}(d){1,3}(e){00}", "")).toEqual({ + count: 6, + always: [true, false, true, false, true, false], + }); + expect(regexCaptureParticipation("(?=(a))(?!(b))(?<=(c))(?e)", "")).toEqual({ + count: 6, + always: [true, false, false, false, false, true], + }); + }); + + test("skips escapes, classes and non-capturing groups", () => { + expect(regexCaptureParticipation("\\((a)\\)[(|)\\]](?:b)(?i:c)", "")).toEqual({ + count: 2, + always: [true, true], + }); + expect(regexCaptureParticipation("[](a)", "")).toEqual({ count: 2, always: [true, true] }); + expect(regexCaptureParticipation("(a)[", "")).toBeNull(); + expect(regexCaptureParticipation("(a", "")).toBeNull(); + expect(regexCaptureParticipation("a)", "")).toBeNull(); + expect(regexCaptureParticipation("(a)", "v")).toBeNull(); + }); + test("marks captures in alternations, optional groups and lookarounds", () => { + const always = (pattern: string): boolean[] | null => + regexCaptureParticipation(pattern, "")?.always ?? null; + expect(always("tag(\\d+)")).toEqual([true, true]); + expect(always("((a)b)+c")).toEqual([true, true, true]); + expect(always("(?\\d)(x)")).toEqual([true, true, true]); + expect(always("(a){2,3}")).toEqual([true, true]); + expect(always("(?:x|y)(a)")).toEqual([true, true]); + expect(always("(?:(a)|b)c")).toEqual([true, false]); + expect(always("(x)?(\\d)")).toEqual([true, false, true]); + expect(always("(?:(t)a)*g")).toEqual([true, false]); + expect(always("((a)?)")).toEqual([true, true, false]); + expect(always("(?<=(-))(\\w)")).toEqual([true, false, true]); + expect(always("[\\]|(](b)")).toEqual([true, true]); + expect(always("[^)](c)\\|")).toEqual([true, true]); + }); +}); diff --git a/packages/compiler/src/frontend/lowering/containers/replacer-params.ts b/packages/compiler/src/frontend/lowering/containers/replacer-params.ts new file mode 100644 index 000000000..e84ca1461 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/containers/replacer-params.ts @@ -0,0 +1,292 @@ +import * as ts from "../../ts7/adapter.js"; +import { F64, STRING, type IrExpr, type IrType } from "../../../ir/ir.js"; +import type { Lowerer } from "../lowerer.js"; +import { bindingNeverReassigned } from "../binding-analysis.js"; + +/** The captures of a regex literal's pattern (the whole match included) + * and, per capture, whether every successful match sets it. A capture may + * be unset when an enclosing group has alternatives, when an enclosing + * group or the capture itself is quantified with a minimum of zero, or + * when it sits inside a lookaround. Null for the `v` flag (nested + * classes) and for a pattern the scan does not finish. */ +export function regexCaptureParticipation( + pattern: string, + flags: string, +): { count: number; always: boolean[] } | null { + if (flags.includes("v")) return null; + const always: boolean[] = [true]; + /** Open groups: the group's own capture (-1 when it does not capture), + * the captures nested in it, whether it has alternatives, and whether it + * is a lookaround. */ + const frames: { own: number; nested: number[]; alternatives: boolean; lookaround: boolean }[] = [ + { own: -1, nested: [], alternatives: false, lookaround: false }, + ]; + let i = 0; + while (i < pattern.length) { + const c = pattern[i]!; + if (c === "\\") { + i += 2; + continue; + } + if (c === "[") { + // The first unescaped `]` ends the class (`[]` is an empty class). + i++; + while (i < pattern.length && pattern[i] !== "]") i += pattern[i] === "\\" ? 2 : 1; + if (i >= pattern.length) return null; + i++; + continue; + } + if (c === "|") { + frames[frames.length - 1]!.alternatives = true; + i++; + continue; + } + if (c === "(") { + let capture = false; + let lookaround = false; + if (pattern[i + 1] !== "?") { + capture = true; + i++; + } else if (pattern[i + 2] === "<" && pattern[i + 3] !== "=" && pattern[i + 3] !== "!") { + const gt = pattern.indexOf(">", i + 3); + if (gt < 0) return null; + capture = true; + i = gt + 1; + } else { + const kind = pattern[i + 2] === "<" ? pattern[i + 3] : pattern[i + 2]; + lookaround = kind === "=" || kind === "!"; + i += 2; + } + frames.push({ + own: capture ? always.length : -1, + nested: [], + alternatives: false, + lookaround, + }); + if (capture) always.push(true); + continue; + } + if (c === ")") { + if (frames.length === 1) return null; + const frame = frames.pop()!; + i++; + const next = pattern[i]; + const optional = + next === "?" || next === "*" || (next === "{" && /^\{0+[,}]/.test(pattern.slice(i))); + const parent = frames[frames.length - 1]!; + // Alternatives inside the group leave its nested captures unset, but + // not the group's own capture. + for (const index of frame.nested) { + if (optional || frame.lookaround || frame.alternatives) always[index] = false; + parent.nested.push(index); + } + if (frame.own >= 0) { + if (optional || frame.lookaround) always[frame.own] = false; + parent.nested.push(frame.own); + } + continue; + } + i++; + } + if (frames.length !== 1) return null; + if (frames[0]!.alternatives) for (const index of frames[0]!.nested) always[index] = false; + return { count: always.length, always }; +} + +/** String members a refined parameter may use: these lower from the + * receiver's type whatever the checker resolved for an `any` receiver. */ +const STRING_MEMBERS: ReadonlySet = new Set([ + "length", + "toUpperCase", + "toLowerCase", + "trim", + "charCodeAt", + "codePointAt", + "slice", + "substring", + "repeat", + "padStart", + "padEnd", + "includes", + "startsWith", + "endsWith", + "indexOf", +]); + +/** Whether every read of `symbol` in `body` is one that means the same + * for an `any` value and for a `kind` value, and lowers for both: a + * template span, a Number/String/parseInt/parseFloat argument, a listed + * string member, `+`, strict equality, arithmetic and comparisons of + * numbers, a returned value, and a string key of an element access on a + * string-indexed record or an `any` value. Anything else keeps the + * parameter `any`. */ +function readsFit( + lowerer: Lowerer, + body: ts.Node, + symbol: ts.Symbol, + kind: "string" | "f64", +): boolean { + const name = ts.symbolName(symbol); + const primitive = (e: ts.Expression): boolean => { + const flags = lowerer.checker.getTypeAtLocation(e).flags; + return (flags & (ts.TypeFlags.NumberLike | ts.TypeFlags.StringLike)) !== 0; + }; + const fits = (id: ts.Identifier): boolean => { + let node: ts.Node = id; + while (ts.isParenthesizedExpression(node.parent)) node = node.parent; + const parent = node.parent; + if (ts.isTemplateSpan(parent) || ts.isReturnStatement(parent)) return true; + if (ts.isElementAccessExpression(parent)) { + if (kind !== "string" || parent.argumentExpression !== node) return false; + const receiver = lowerer.mapTypeOf(lowerer.checker.getTypeAtLocation(parent.expression)); + if (receiver?.kind === "dyn") return true; + const shape = receiver?.kind === "record" ? lowerer.shapes.get(receiver.shapeId) : undefined; + return shape?.indexValue !== undefined && !shape.tuple; + } + if (ts.isArrowFunction(parent)) return parent.body === node; + if (ts.isCallExpression(parent)) + return ( + parent.expression !== node && + ["Number", "String", "parseInt", "parseFloat"].some((global) => + lowerer.isStdlibGlobal(parent.expression, global), + ) + ); + if (ts.isPropertyAccessExpression(parent)) { + const member = parent.name.text; + if (kind !== "string" || parent.expression !== node || !STRING_MEMBERS.has(member)) + return false; + return ( + member === "length" || + (ts.isCallExpression(parent.parent) && parent.parent.expression === parent) + ); + } + if (ts.isBinaryExpression(parent)) { + const op = parent.operatorToken.kind; + if ( + op === ts.SyntaxKind.EqualsEqualsEqualsToken || + op === ts.SyntaxKind.ExclamationEqualsEqualsToken + ) + return true; + const other = parent.left === node ? parent.right : parent.left; + if (op === ts.SyntaxKind.PlusToken) return kind === "string" || primitive(other); + return ( + kind === "f64" && + (op === ts.SyntaxKind.MinusToken || + op === ts.SyntaxKind.AsteriskToken || + op === ts.SyntaxKind.SlashToken || + op === ts.SyntaxKind.PercentToken || + op === ts.SyntaxKind.LessThanToken || + op === ts.SyntaxKind.GreaterThanToken || + op === ts.SyntaxKind.LessThanEqualsToken || + op === ts.SyntaxKind.GreaterThanEqualsToken) && + (lowerer.checker.getTypeAtLocation(other).flags & ts.TypeFlags.NumberLike) !== 0 + ); + } + return false; + }; + let ok = true; + const visit = (n: ts.Node): void => { + if (!ok) return; + if (ts.isIdentifier(n) && n.text === name && lowerer.resolveValueSymbol(n) === symbol) { + if (!fits(n)) ok = false; + return; + } + ts.forEachChild(n, visit); + }; + visit(body); + return ok; +} + +/** The capture layout of a string search: the match, then the offset and + * the subject. */ +export const PLAIN_SEARCH_CAPTURES: { count: number; always: boolean[] } = { + count: 1, + always: [true], +}; + +/** The captures of a regex literal argument (regexCaptureParticipation), + * or null for any other pattern expression. */ +export function regexLiteralCapturesOf( + pattern: ts.Expression, +): { count: number; always: boolean[] } | null { + let regex = pattern; + while (ts.isParenthesizedExpression(regex)) regex = regex.expression; + if (!ts.isRegularExpressionLiteral(regex)) return null; + const text = regex.text; + const slash = text.lastIndexOf("/"); + return regexCaptureParticipation(text.slice(1, slash), text.slice(slash + 1)); +} + +/** Lowers an inline replace callback. JavaScript passes it the match, the + * captures, the match offset and the subject (`captures` gives their + * layout; null lowers the callback as written). TypeScript types the + * parameters after the match `any`, so an unannotated parameter that is + * never reassigned, whose every read fits its value (readsFit), and that + * receives a capture every match sets, the offset or the subject takes + * that value's type: a string or a number. The closure can then be called + * directly (typed replacers in lib-dispatch.ts) with fewer boxed values. */ +export function lowerReplacerCallback( + lowerer: Lowerer, + callback: ts.Expression, + captures: { count: number; always: boolean[] } | null, +): IrExpr { + let node = callback; + while (ts.isParenthesizedExpression(node)) node = node.expression; + // A named function expression can call itself with other arguments. + const literal = + ts.isArrowFunction(node) || (ts.isFunctionExpression(node) && !node.name) ? node : null; + if (!literal || !captures) return lowerer.lowerExpr(callback); + const bound: { symbol: ts.Symbol; previous: IrType | undefined }[] = []; + const previousImplicit = lowerer.implicitParamTypes; + let implicit: Map | null = null; + const params = literal.parameters.nodes; + // A declared `this` parameter takes no argument position. + const first = + params[0] && ts.isIdentifier(params[0].name) && params[0].name.text === "this" ? 1 : 0; + params.slice(first).forEach((param, i) => { + const type = + i === 0 || i > captures.count + 1 + ? null + : i < captures.count + ? captures.always[i] === true + ? STRING + : null + : i === captures.count + ? F64 + : STRING; + if ( + type === null || + !ts.isIdentifier(param.name) || + param.type || + param.initializer || + param.dotDotDotToken || + param.questionToken + ) + return; + if ((lowerer.checker.getTypeAtLocation(param.name).flags & ts.TypeFlags.Any) === 0) return; + const symbol = lowerer.checker.getSymbolAtLocation(param.name); + if ( + !symbol || + !bindingNeverReassigned(lowerer, symbol, param) || + !readsFit(lowerer, literal.body, symbol, type === STRING ? "string" : "f64") + ) + return; + bound.push({ symbol, previous: lowerer.runtimeOptionalBindingTypes.get(symbol) }); + lowerer.runtimeOptionalBindingTypes.set(symbol, type); + implicit ??= new Map(previousImplicit ?? []); + implicit.set( + symbol, + type === STRING ? lowerer.checker.getStringType() : lowerer.checker.getNumberType(), + ); + }); + if (implicit !== null) lowerer.implicitParamTypes = implicit; + try { + return lowerer.lowerExpr(callback); + } finally { + for (const { symbol, previous } of bound) { + if (previous === undefined) lowerer.runtimeOptionalBindingTypes.delete(symbol); + else lowerer.runtimeOptionalBindingTypes.set(symbol, previous); + } + lowerer.implicitParamTypes = previousImplicit; + } +} diff --git a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts index d55f71ff0..12885418b 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts @@ -33,6 +33,7 @@ import { isSafeToDiscard } from "../expressions/evaluation-safety.js"; import { STRING_INDEX_METHODS, STRING_REPLACE_METHODS, STR_METHODS } from "../surfaces.js"; import { execResultCaptureMask } from "./exec-result-uses.js"; import { lowerStringReplacement } from "./string-replacement.js"; +import { lowerReplacerCallback, regexLiteralCapturesOf } from "./replacer-params.js"; import { coerceStringSearchValue, defaultAfterUndefined, @@ -1025,7 +1026,11 @@ export function lowerRegexMethodCall( : [ lowerer.lowerExprExpecting(arg0, { kind: "regex" }), call.arguments.nodes[1] - ? lowerer.lowerExpr(call.arguments.nodes[1]) + ? lowerReplacerCallback( + lowerer, + call.arguments.nodes[1], + regexLiteralCapturesOf(arg0), + ) : dynUndefinedExpr(loc), ]; if (name !== "split") { diff --git a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts index 156f74869..c21808464 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts @@ -18,6 +18,7 @@ import { import { dynUndefinedExpr, numLit, strLit, varRef } from "../../../ir/build.js"; import { InternalCompilerError } from "../../../errors.js"; import { locOf } from "../../program.js"; +import { PLAIN_SEARCH_CAPTURES, lowerReplacerCallback } from "./replacer-params.js"; import type { Lowerer } from "../lowerer.js"; import { coerceStringSearchValue, @@ -48,11 +49,18 @@ export function lowerStringReplacement( type.kind === "bool" || type.kind === "bigint" || isUnitType(type); - const argument = (node: ts.Expression | undefined, callback: boolean): IrExpr => { + const argument = ( + node: ts.Expression | undefined, + callback: boolean, + plainSearch = false, + ): IrExpr => { if (!node) return strLit("undefined", loc); const absent = lowerStaticallyUndefinedArgument(lowerer, node); if (absent) return defaultAfterUndefined(absent, strLit("undefined", loc)); - const value = lowerer.lowerExpr(node); + // A plain search passes (match, offset, subject). + const value = plainSearch + ? lowerReplacerCallback(lowerer, node, PLAIN_SEARCH_CAPTURES) + : lowerer.lowerExpr(node); if (value.type.kind === "dyn" || (callback && value.type.kind === "func")) return value; if (!primitive(value.type)) { lowerer.noLowering( @@ -64,7 +72,7 @@ export function lowerStringReplacement( return coerceStringSearchValue(lowerer, value, node, loc); }; const search = argument(args[0], false); - const replacement = argument(args[1], true); + const replacement = argument(args[1], true, search.type.kind !== "dyn"); if (search.type.kind === "dyn" || replacement.type.kind === "dyn") { // Evaluate all inputs once before choosing the search protocol. Regexes // retain their native pattern; plain values use the string-search helper. diff --git a/packages/compiler/test/replacer-params.test.ts b/packages/compiler/test/replacer-params.test.ts new file mode 100644 index 000000000..bb5549a86 --- /dev/null +++ b/packages/compiler/test/replacer-params.test.ts @@ -0,0 +1,84 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { compile } from "../src/index.js"; +import { regexLiteralCaptures } from "../src/backend/llvm/regex-matchers.js"; +import { regexCaptureParticipation } from "../src/frontend/lowering/containers/replacer-params.js"; + +// The frontend binds a replace callback parameter as a string only for a +// capture every match sets; the backend then calls the callback directly +// only when its own parse agrees. +describe("replace callback captures", () => { + test("never claims a capture the backend's parse can leave unset", () => { + const patterns = [ + "(\\w+)=(\\d)+", + "(?a)(b)?|(c)", + "x(?:(a)|(b))y(c)*(d){0,2}(e){1,3}", + "((a)(b)?)+", + "(a(b|c))d", + "(?:a|(b))(c)", + ]; + for (const pattern of patterns) { + const frontend = regexCaptureParticipation(pattern, ""); + const backend = regexLiteralCaptures(pattern, ""); + expect(frontend?.count).toBe(backend?.count); + frontend!.always.forEach((always, index) => { + if (always) expect(backend!.always[index]).toBe(true); + }); + } + }); +}); + +const directories: string[] = []; +afterEach(async () => { + await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function emit(source: string): Promise { + const outDir = await mkdtemp(join(tmpdir(), "scriptc-replacer-")); + directories.push(outDir); + const entry = join(outDir, "main.ts"); + await writeFile(entry, source); + const outPath = join(outDir, "program.ll"); + const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result)).toBe(true); + return readFile(outPath, "utf8"); +} + +/** The (arity, kinds) arguments of each direct replacer call, and the + * number of calls through the boxed argument array. */ +function replacerCalls(llvm: string): { direct: string[]; boxed: number } { + const lines = llvm.split("\n").filter((line) => line.includes("call ")); + return { + direct: lines + .filter((line) => line.includes("@scr_regex_replace_fn(")) + .map((line) => /i32 (-?\d+), i32 (-?\d+), i1/.exec(line)!.slice(1, 3).join(":")), + boxed: lines.filter((line) => line.includes("@scr_regex_replace_callback(")).length, + }; +} + +describe("replace callback calls", () => { + test("unannotated parameters take the kinds the replace passes", async () => { + const llvm = await emit(` +const vars: Record = { a: "1" }; +const text = String(Date.now() > 0 ? "{a} {b}" : ""); +console.log(text.replace(/\\{(\\w+)\\}/g, (_m, key) => vars[key] ?? "")); +console.log(text.replace(/\\{(\\w)/g, (_m, c) => c.toUpperCase())); +console.log(text.replace(/(a)|(b)/g, (m, a, b) => \`\${m}\${a}\${b}\`)); +console.log(text.replace(/(a)(b)?/g, (m: string, a: string, b: string | undefined, at: number) => m + a + (b ?? "") + at)); +const half = (n: number): number => n / 2; +console.log(text.replace(/b/g, (m, at) => String(half(at)))); +console.log(text.replace(/(a)(b)(c)/g, (m, a, b, c, at, all, extra) => m + a + b + c + at + all + extra)); +`); + expect(replacerCalls(llvm)).toEqual({ + // Typed parameters and result (0); a checked-dynamic result (bit 31); + // two captures a match can leave unset (bits 1 and 2); + // \`string | undefined\` at capture 2 (bit 8 + 2); an offset passed to a + // number parameter keeps its checked-dynamic box (bit 1). + direct: ["2:0", "2:-2147483648", "3:6", "4:1024", "2:2"], + // Seven parameters exceed the direct calls' limit. + boxed: 1, + }); + }); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8118-replacer-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8118-replacer-params.ts.json new file mode 100644 index 000000000..4de6ead6b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8118-replacer-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8118-replacer-params.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8345-replace-callback-untyped-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8345-replace-callback-untyped-params.ts.json new file mode 100644 index 000000000..9a4c2b510 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8345-replace-callback-untyped-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8345-replace-callback-untyped-params.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8346-replace-callback-untyped-params.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8346-replace-callback-untyped-params.mjs.json new file mode 100644 index 000000000..301203783 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8346-replace-callback-untyped-params.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8346-replace-callback-untyped-params.mjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 158697a14..ab323c288 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -1529,49 +1529,101 @@ ScrStr *scr_regex_replace_callback(ScrStr *s, ScrRegex *re, ScrDyn *callback, bo return scr_jb_finish(&output); } -/* One call of a typed replacer (scr_regex_replace_fn's contract): string - * arguments are passed +1 (the closure ABI's owned parameters), the offset - * as a number, and the result is the closure's +1 string. */ -static ScrStr *scr_regex_call_replacer(ScrClosure *f, uint32_t arity, int ncap, - ScrStr *const *caps, double offset, ScrStr *subject) { - void *p[4] = { NULL, NULL, NULL, NULL }; - for (uint32_t i = 0; i < arity && i < 4; i++) { - if ((int)i < ncap) { - if (!caps[i]) scr_trap("scriptc: internal error: typed replacer capture is unset\n"); - p[i] = scr_str_retain(caps[i]); - } else if ((int)i == ncap + 1) { - p[i] = scr_str_retain(subject); - } - } +/* The closure signature scr_regex_replace_fn calls: `arity` pointer + * parameters, except a number at `number_at` (-1 when there is none). A + * string and a checked-dynamic value are both pointers, and so are both + * results. */ +#define SCR_REPLACER_CALL(...) return ((void *(*)(ScrClosure *, __VA_ARGS__))fn) +static inline __attribute__((always_inline)) void *scr_regex_call_replacer( + ScrClosure *f, uint32_t arity, int number_at, void *const *p, double offset) { void *fn = f->fn; - typedef ScrStr *R; - switch (arity) { - case 0: return ((R (*)(ScrClosure *))fn)(f); - case 1: return ((R (*)(ScrClosure *, void *))fn)(f, p[0]); - case 2: - if (ncap == 1) return ((R (*)(ScrClosure *, void *, double))fn)(f, p[0], offset); - return ((R (*)(ScrClosure *, void *, void *))fn)(f, p[0], p[1]); - case 3: - if (ncap == 1) return ((R (*)(ScrClosure *, void *, double, void *))fn)(f, p[0], offset, p[2]); - if (ncap == 2) return ((R (*)(ScrClosure *, void *, void *, double))fn)(f, p[0], p[1], offset); - return ((R (*)(ScrClosure *, void *, void *, void *))fn)(f, p[0], p[1], p[2]); - case 4: - if (ncap == 2) - return ((R (*)(ScrClosure *, void *, void *, double, void *))fn)(f, p[0], p[1], offset, p[3]); - if (ncap == 3) - return ((R (*)(ScrClosure *, void *, void *, void *, double))fn)(f, p[0], p[1], p[2], offset); - if (ncap >= 4) - return ((R (*)(ScrClosure *, void *, void *, void *, void *))fn)(f, p[0], p[1], p[2], p[3]); - break; + typedef void *P; + typedef double D; + switch (arity * 8 + (uint32_t)(number_at < 0 ? 7 : number_at)) { + case 0 * 8 + 7: return ((void *(*)(ScrClosure *))fn)(f); + case 1 * 8 + 7: SCR_REPLACER_CALL(P)(f, p[0]); + case 2 * 8 + 7: SCR_REPLACER_CALL(P, P)(f, p[0], p[1]); + case 2 * 8 + 1: SCR_REPLACER_CALL(P, D)(f, p[0], offset); + case 3 * 8 + 7: SCR_REPLACER_CALL(P, P, P)(f, p[0], p[1], p[2]); + case 3 * 8 + 1: SCR_REPLACER_CALL(P, D, P)(f, p[0], offset, p[2]); + case 3 * 8 + 2: SCR_REPLACER_CALL(P, P, D)(f, p[0], p[1], offset); + case 4 * 8 + 7: SCR_REPLACER_CALL(P, P, P, P)(f, p[0], p[1], p[2], p[3]); + case 4 * 8 + 1: SCR_REPLACER_CALL(P, D, P, P)(f, p[0], offset, p[2], p[3]); + case 4 * 8 + 2: SCR_REPLACER_CALL(P, P, D, P)(f, p[0], p[1], offset, p[3]); + case 4 * 8 + 3: SCR_REPLACER_CALL(P, P, P, D)(f, p[0], p[1], p[2], offset); + case 5 * 8 + 7: SCR_REPLACER_CALL(P, P, P, P, P)(f, p[0], p[1], p[2], p[3], p[4]); + case 5 * 8 + 1: SCR_REPLACER_CALL(P, D, P, P, P)(f, p[0], offset, p[2], p[3], p[4]); + case 5 * 8 + 2: SCR_REPLACER_CALL(P, P, D, P, P)(f, p[0], p[1], offset, p[3], p[4]); + case 5 * 8 + 3: SCR_REPLACER_CALL(P, P, P, D, P)(f, p[0], p[1], p[2], offset, p[4]); + case 5 * 8 + 4: SCR_REPLACER_CALL(P, P, P, P, D)(f, p[0], p[1], p[2], p[3], offset); + case 6 * 8 + 7: SCR_REPLACER_CALL(P, P, P, P, P, P)(f, p[0], p[1], p[2], p[3], p[4], p[5]); + case 6 * 8 + 1: SCR_REPLACER_CALL(P, D, P, P, P, P)(f, p[0], offset, p[2], p[3], p[4], p[5]); + case 6 * 8 + 2: SCR_REPLACER_CALL(P, P, D, P, P, P)(f, p[0], p[1], offset, p[3], p[4], p[5]); + case 6 * 8 + 3: SCR_REPLACER_CALL(P, P, P, D, P, P)(f, p[0], p[1], p[2], offset, p[4], p[5]); + case 6 * 8 + 4: SCR_REPLACER_CALL(P, P, P, P, D, P)(f, p[0], p[1], p[2], p[3], offset, p[5]); + case 6 * 8 + 5: SCR_REPLACER_CALL(P, P, P, P, P, D)(f, p[0], p[1], p[2], p[3], p[4], offset); default: break; } scr_trap("scriptc: internal error: typed replacer signature\n"); } +#undef SCR_REPLACER_CALL + +/* Typed replacer argument k: a capture (NULL for an unset optional one) + * or the subject; the offset is passed separately. The compiler admits a + * typed parameter only at those positions. Inlined with each constant k. */ +static inline __attribute__((always_inline)) void scr_regex_typed_arg( + void **p, uint32_t k, uint32_t kinds, int capture_count, ScrStr *const *caps, ScrStr *subject) { + if ((int)k < capture_count) { + if (!caps[k] && !(kinds & (SCR_REPLACER_OPTIONAL << k))) + scr_trap("scriptc: internal error: typed replacer capture is unset\n"); + p[k] = caps[k] ? scr_str_retain(caps[k]) : NULL; + } else if ((int)k == capture_count + 1) { + p[k] = scr_str_retain(subject); + } +} + +/* One match's replacer arguments when some parameter is checked-dynamic + * (scr_regex_replace_fn's `kinds`): typed ones +1, the rest new boxes. + * `boxed_subject` is the subject's box; `groups` the match's groups + * object, or NULL when no parameter reads it. */ +__attribute__((noinline)) static void scr_regex_replacer_args( + void **p, uint32_t arity, uint32_t kinds, int capture_count, ScrStr *const *caps, double offset, + ScrStr *subject, ScrDyn *boxed_subject, ScrDyn *groups) { + for (uint32_t k = 0; k < arity; k++) { + int at = (int)k; + if (kinds & (1u << k)) { + if (at < capture_count) + p[k] = caps[k] ? scr_dyn_new_str(caps[k]) : scr_dyn_retain(scr_dyn_undefined()); + else if (at == capture_count) + p[k] = scr_dyn_new_num(offset); + else if (at == capture_count + 1) + p[k] = scr_dyn_retain(boxed_subject); + else if (at == capture_count + 2 && groups) + p[k] = scr_dyn_retain(groups); + else + p[k] = scr_dyn_retain(scr_dyn_undefined()); + } else if (at < capture_count) { + if (!caps[k] && !(kinds & (SCR_REPLACER_OPTIONAL << k))) + scr_trap("scriptc: internal error: typed replacer capture is unset\n"); + p[k] = caps[k] ? scr_str_retain(caps[k]) : NULL; + } else if (at == capture_count + 1) { + p[k] = scr_str_retain(subject); + } else if (at != capture_count) { + scr_trap("scriptc: internal error: typed replacer signature\n"); + } + } +} /* scr_regex_replace_callback's algorithm for a typed replacer: the matches - * are collected first, keeping only the captures the replacer reads, then - * the replacer runs once per match in order. */ -ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t arity, bool all) { + * are collected first, keeping only the captures and groups objects the + * replacer reads, then the replacer runs once per match in order. Typed + * parameters receive their values +1 (the closure ABI's owned + * parameters) and an unset optional capture NULL; checked-dynamic ones + * receive new boxes. Inlined into one copy for kinds 0, where the boxed + * handling folds away, and one for the other kinds. */ +static inline __attribute__((always_inline)) ScrStr * +scr_regex_replace_fn_impl(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t arity, uint32_t kinds, + bool all) { uint8_t *bc = scr_regex_bc(re); int flags = lre_get_flags(bc); bool global = (flags & LRE_FLAG_GLOBAL) != 0; @@ -1580,15 +1632,28 @@ ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t ar scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; } + if (arity > SCR_REPLACER_MAX_ARITY) scr_trap("scriptc: internal error: typed replacer arity\n"); bool sticky = (flags & LRE_FLAG_STICKY) != 0; bool unicode = (flags & (LRE_FLAG_UNICODE | LRE_FLAG_UNICODE_SETS)) != 0; int capture_count = lre_get_capture_count(bc); size_t keep = arity < (uint32_t)capture_count ? arity : (size_t)capture_count; + /* Node passes the groups object after the subject when the pattern has + * named groups; only a checked-dynamic parameter can read it, and no + * typed parameter follows the subject. */ + bool past_subject = (uint32_t)capture_count + 2 < arity; + if (past_subject && !(kinds & SCR_REPLACER_BOXED_PARAMS)) + scr_trap("scriptc: internal error: typed replacer signature\n"); + const char *groupnames = past_subject ? lre_get_groupnames(bc) : NULL; + bool groups_read = groupnames != NULL; + if (groups_read && !(kinds & (1u << (capture_count + 2)))) + scr_trap("scriptc: internal error: typed replacer groups parameter\n"); + int number_at = capture_count < (int)arity && !(kinds & (1u << capture_count)) ? capture_count : -1; ScrRegexInput input; scr_regex_input_init(&input, re, bc, s); int len = input.len; struct { int start, end; } *matches = NULL; ScrStr **caps = NULL; + ScrDyn **groups = NULL; size_t count = 0, capacity = 0; if (global) re->last_index = 0; int position = sticky ? scr_regex_start(re, len) : 0; @@ -1610,6 +1675,11 @@ ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t ar if (!grown) scr_regex_oom(); caps = grown; } + if (groups_read) { + grown = realloc(groups, next * sizeof(*groups)); + if (!grown) scr_regex_oom(); + groups = grown; + } capacity = next; } for (size_t group = 0; group < keep; group++) { @@ -1618,6 +1688,18 @@ ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t ar cs && ce ? scr_regex_span(&input, scr_capture_index(&input, cs), scr_capture_index(&input, ce)) : NULL; } + if (groups_read) { + ScrDyn *object = scr_dyn_new_obj_null_proto(); + const char *name = groupnames; + for (int group = 1; group < capture_count; group++) { + size_t length = strlen(name); + if (length && (capture[group * 2] || !scr_dyn_obj_get(object, name, length))) + scr_dyn_obj_set(object, name, length, + scr_regex_capture_value(&input, capture[group * 2], capture[group * 2 + 1])); + name += length + LRE_GROUP_NAME_TRAILER_LEN; + } + groups[count] = object; + } matches[count].start = start; matches[count].end = end; count++; @@ -1627,18 +1709,43 @@ ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t ar ScrJsonBuf output; scr_jb_init(&output); int next = 0; + /* One box of the subject serves every match. */ + ScrDyn *subject = NULL; + if (kinds & SCR_REPLACER_BOXED_PARAMS) subject = scr_dyn_new_str(s); scr_dyn_this_push_dyn(NULL); for (size_t i = 0; i < count; i++) { - ScrStr *replacement = scr_regex_call_replacer(f, arity, capture_count, caps + i * keep, - (double)matches[i].start, s); - if (scr_exc_pending()) break; + void *p[SCR_REPLACER_MAX_ARITY] = { NULL }; + double offset = (double)matches[i].start; + ScrStr *const *match_caps = caps + i * keep; + if (kinds & SCR_REPLACER_BOXED_PARAMS) { + scr_regex_replacer_args(p, arity, kinds, capture_count, match_caps, offset, s, subject, + groups_read ? groups[i] : NULL); + } else { + for (uint32_t k = 0; k < SCR_REPLACER_MAX_ARITY; k++) + if (k < arity) scr_regex_typed_arg(p, k, kinds, capture_count, match_caps, s); + } + void *result = scr_regex_call_replacer(f, arity, number_at, p, offset); + ScrStr *replacement = (ScrStr *)result; + if (kinds & SCR_REPLACER_BOXED_RESULT) { + replacement = scr_exc_pending() ? NULL : scr_dyn_string_coerce_js((ScrDyn *)result); + scr_dyn_release((ScrDyn *)result); + } + if (scr_exc_pending()) { + scr_str_release(replacement); + break; + } scr_regex_put_span(&output, &input, next, matches[i].start); scr_jb_put_str(&output, replacement); scr_str_release(replacement); next = matches[i].end; } scr_dyn_this_pop(); + if (subject) scr_dyn_release(subject); for (size_t i = 0; i < count * keep; i++) scr_str_release(caps[i]); + if (groups_read) { + for (size_t i = 0; i < count; i++) scr_dyn_release(groups[i]); + free(groups); + } free(caps); free(matches); if (scr_exc_pending()) { @@ -1651,6 +1758,24 @@ ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t ar return scr_jb_finish(&output); } +__attribute__((noinline)) static ScrStr *scr_regex_replace_typed(ScrStr *s, ScrRegex *re, + ScrClosure *f, uint32_t arity, + bool all) { + return scr_regex_replace_fn_impl(s, re, f, arity, 0, all); +} + +__attribute__((noinline)) static ScrStr *scr_regex_replace_kinds(ScrStr *s, ScrRegex *re, + ScrClosure *f, uint32_t arity, + uint32_t kinds, bool all) { + return scr_regex_replace_fn_impl(s, re, f, arity, kinds, all); +} + +ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, ScrClosure *f, uint32_t arity, + uint32_t kinds, bool all) { + if (kinds == 0) return scr_regex_replace_typed(s, re, f, arity, all); + return scr_regex_replace_kinds(s, re, f, arity, kinds, all); +} + /* The pinned Node ANSI matcher derives from ansi-regex, copyright (c) * Sindre Sorhus , used under the MIT license: * Permission is hereby granted, free of charge, to any person obtaining a diff --git a/packages/runtime/src/scr_regex.h b/packages/runtime/src/scr_regex.h index 01c545bc5..cb61e49f3 100644 --- a/packages/runtime/src/scr_regex.h +++ b/packages/runtime/src/scr_regex.h @@ -80,16 +80,26 @@ ScrStr *scr_regex_flags(ScrRegex *re); /* +1 */ ScrStr *scr_regex_to_string(ScrRegex *re); /* +1 */ /* replace: first match without /g, every match with it. */ ScrStr *scr_regex_replace(ScrStr *s, ScrRegex *re, ScrStr *rep); -/* replace/replaceAll with a typed replacer closure of `arity` (at most 4) - * parameters returning a string. Parameter i receives what Node passes at - * position i: the match and captures (strings) below the capture count n, - * the match offset (a number) at n, the subject (a string) at n + 1. The - * compiler admits a closure only when every position it declares is one of - * those with the declared kind, and every capture it reads is set in every - * match (scr_regex_replace_callback handles the rest). Borrows all. */ +/* replace/replaceAll with a replacer closure of `arity` (at most + * SCR_REPLACER_MAX_ARITY) parameters. Parameter i receives what Node passes + * at position i: the match and captures below the capture count n, the + * match offset at n, the subject at n + 1, the groups object at n + 2 when + * the pattern has named groups, and undefined after that. Bit i of + * `kinds` makes parameter i a checked-dynamic value, and bit + * SCR_REPLACER_OPTIONAL << i a capture held as a string or NULL for + * undefined; otherwise it is a string, or a number at n. + * SCR_REPLACER_BOXED_RESULT makes the result a checked-dynamic value, + * converted with ToString. The compiler admits a closure only when every + * typed position it declares is one of those with the declared kind, and + * every capture a string parameter reads is set in every match + * (scr_regex_replace_callback handles the rest). Borrows all. */ +#define SCR_REPLACER_MAX_ARITY 6 +#define SCR_REPLACER_BOXED_PARAMS 0xffu +#define SCR_REPLACER_OPTIONAL (1u << 8) +#define SCR_REPLACER_BOXED_RESULT (1u << 31) struct ScrClosure; ScrStr *scr_regex_replace_fn(ScrStr *s, ScrRegex *re, struct ScrClosure *f, uint32_t arity, - bool all); + uint32_t kinds, bool all); /* replaceAll: throws Node's TypeError when /g is missing (may-throw). */ ScrStr *scr_regex_replace_all(ScrStr *s, ScrRegex *re, ScrStr *rep); /* split: captured substrings are interleaved with the split pieces; diff --git a/tests/corpus/8118-replacer-params.ts b/tests/corpus/8118-replacer-params.ts new file mode 100644 index 000000000..14579ec84 --- /dev/null +++ b/tests/corpus/8118-replacer-params.ts @@ -0,0 +1,43 @@ +// Inline replace callbacks whose parameters after the match are left +// untyped: captures every match sets, captures that can be unset (in an +// alternation, an optional or zero-repeat group, a lookahead), the offset, +// the subject and the groups object, reassigned parameters, named groups, +// patterns with classes, escapes and lookbehind, function expressions and +// replaceAll. +const lines = ["id7,tag12;tag3", "x9 y10", "", "naïve 😀 42", "a-b-c"]; +for (const line of lines) { + console.log(line.replace(/tag(\d+)/g, (_m, d) => String(Number(d) + 1))); + console.log(line.replace(/(\w)(\d+)/g, (m, letter, digits, offset, subject) => + `[${letter}:${digits.length}@${offset}/${subject.length}]`)); + console.log(line.replace(/(a)|(\d)/g, (m, a, digit) => `<${typeof a},${typeof digit}>`)); + console.log(line.replace(/(x)?(\d)/g, (m, x, d) => `${x === undefined ? "-" : x}${d}`)); + console.log(line.replace(/(?:(t)a)*g/g, (m, t, offset) => `{${t}|${offset}}`)); + console.log(line.replace(/(?=(\d))\d/g, (m, ahead, offset) => `${ahead}${offset}`)); + console.log(line.replace(/(?\d)(\d)?/g, (m, n, rest, offset, subject, groups) => + `${n}${rest ?? "_"}${offset}${typeof subject}${JSON.stringify(groups)}`)); + console.log(line.replace(/([\(\)\[\]]|[a-c])(-)/g, (m, c, dash) => c.toUpperCase() + dash.length)); + console.log(line.replace(/(?<=-)(\w)/g, (m, c, offset) => c + offset.toFixed(1))); + console.log(line.replaceAll(/(\d)/g, function (m, d, offset) { + return `${d * 2}:${offset + 1}`; + })); + console.log(line.replace(/(\d)/g, (m, d) => { + d = d.length > 0 ? 5 : 6; + return String(d + 1); + })); + console.log(line.replace(/(\d)\1?/g, (m, d, offset, subject) => `${d}${offset}${subject === line}`)); + console.log(line.replace(/(.)/u, (m, c, offset, subject, extra) => `${c.codePointAt(0)}${offset}${extra}`)); + console.log(line.replace(/(\d){0,2}x/g, (m, d) => `${d}`)); + console.log(line.replace(/(\w)(\w)/g, (m, a, b, offset) => + [a.toUpperCase(), b.toLowerCase(), a.trim(), a.charCodeAt(0), b.codePointAt(0), a.slice(0, 1), + b.substring(0), a.repeat(2), b.padStart(3, "."), a.padEnd(2, "!"), b.includes("1"), + a.startsWith("t"), b.endsWith("g"), a.indexOf("a"), a.length, parseInt(b, 36), + parseFloat(b), Number(b), String(offset), offset - 1, offset * 2, offset % 3, offset < 4, + a === b, a !== "x", a + offset, offset + 1, `${offset}`].join(","))); + console.log(line.replace(/(\d+)/g, (m, d) => (d))); + console.log(line.replace(/(\d+)/g, function (m, d) { return d + d; })); +} +let count = 0; +console.log("tag1 tag2".replace(/tag(\d)/g, (m, d) => { + count += Number(d); + return d.repeat(2); +}), count); diff --git a/tests/corpus/8345-replace-callback-untyped-params.ts b/tests/corpus/8345-replace-callback-untyped-params.ts new file mode 100644 index 000000000..31d3a038b --- /dev/null +++ b/tests/corpus/8345-replace-callback-untyped-params.ts @@ -0,0 +1,85 @@ +// String replace/replaceAll with inline callbacks whose parameters have no +// annotations, so the lib types every parameter after the match `any`. +// Each such parameter holds what Node passes at its position: the captures, +// the offset, the subject, the groups object and then undefined. Captures a +// match can leave unset hold undefined, and a result that is not a string +// converts with ToString. +const vars: Record = { name: "world", place: "earth" }; +const template = "Hello {name}, welcome to {place}! {missing} {name}."; +console.log(template.replace(/\{(\w+)\}/g, (_m, key) => vars[key] ?? `<${key}>`)); +console.log(template.replace(/\{(\w+)\}/, (_m, key) => key.toUpperCase())); +console.log("some_snake_case_name".replace(/_([a-z])/g, (_m, c) => c.toUpperCase())); +console.log("2024-01-15 and 1999-07-04".replace(/(\d{4})-(\d{2})-(\d{2})/g, (_m, y, mo, d) => `${d}/${mo}/${y}`)); +console.log("a1b22".replace(/\d+/g, (m, offset, subject) => `[${m}@${offset}/${subject.length}]`)); +console.log("x1y2".replace(/(\d)/g, (m, digit, offset, subject) => `${m}${digit}${offset}${subject}`)); +console.log("k=v".replace(/(\w)=(\w)/, function (m, k, v, at, all) { + return `${v}=${k}:${at}:${all}:${m}`; +})); + +console.log("k=v".replace(/(\w)=(\w)/, function (this: void, m, k, v, at) { + return `${typeof this}:${v}=${k}:${at}:${m}`; +})); + +// A parameter the body assigns keeps `any`. +console.log("a1".replace(/(\d)/, (_m, d) => { + d = Number(d) + 1; + return String(d * 2); +})); + +// Captures that a match can leave unset. +console.log("ab".replace(/(a)|(b)/g, (m, a, b) => `${m}:${a}:${b};`)); +console.log("b xb".replace(/(x)?b/g, (m, x, at) => `${m}:${x}:${at}`)); +console.log("ab".replace(/((a)|b)/g, (m, outer, inner) => `${outer}:${inner};`)); +console.log("ab".replace(/(?:(a)|b)/g, (m, a, at) => `${m}:${a}:${at};`)); +console.log("aab".replace(/(a){0,2}b/, (m, a, at) => `${m}:${a}:${at}`)); +console.log("ab".replace(/(?=(a))a|b/g, (m, look, at) => `${m}:${look}:${at};`)); +console.log("ab".replace(/(?!(x))a/g, (m, x, at) => `${m}:${x}:${at};`)); +console.log("ab".replace(/(a)*b/, (m, a, at) => `${m}:${a}:${at}`)); +console.log("a(b)".replace(/\((b)\)/, (m, b, at) => `${m}:${b}:${at}`)); +console.log("a|b".replace(/[|(](b)?/, (m, b, at) => `${m}:${b}:${at}`)); + +// Annotated `string | undefined` captures hold undefined when unset. +console.log("2024-01-15 1999-07".replace(/(\d{4})-(\d{2})(-(\d{2}))?/g, (_m, y: string, mo: string, _x: string | undefined, d: string | undefined) => `${d ?? "??"}/${mo}/${y}`)); +console.log("ab".replace(/(a)|(b)/g, (m: string, a: string | undefined, b: string | undefined, at: number) => `${m}:${a ?? "-"}:${b ?? "-"}:${at};`)); +console.log("abc".replace(/(a)(b)(c)/, (m: string, a: string, b: string, c: string, at: number, all: string) => `${c}${b}${a}${m}${at}${all}`)); +console.log("abcd".replace(/(a)(b)(c)(d)/, (m, a, b, c, d, at, all) => `${d}${c}${b}${a}${m}${at}${all}`)); + +// Named groups pass the groups object after the subject; later parameters +// are undefined. +console.log("2024-01".replace(/(?\d{4})-(?\d{2})/, (m, y, mo, at, all, groups) => `${groups.m}.${groups.y}:${y}${mo}:${at}:${all}`)); +console.log("ab".replace(/(?a)|(?b)/g, (m, a, b, at, all, groups) => `${m}:${JSON.stringify(groups)};`)); +console.log("ab".replace(/b/, (m, at, all, extra, more) => `${m}:${at}:${all}:${extra}:${more}`)); +console.log("ab".replace(/(a)(b)/, (m, a, b, at, all, extra, more) => `${m}${a}${b}${at}${all}${extra}${more}`)); + +// Results that are not strings convert with ToString. +const counts: Record = { a: 1, b: true, c: null, d: undefined, e: { toString: () => "E" } }; +console.log("a b c d e".replace(/\w/g, (m) => counts[m])); +console.log("abc".replace(/(\w)/g, (m, c) => counts[c])); + +// String search patterns pass (match, offset, subject). +console.log("one two one".replaceAll("one", (m, at, subject) => `${m.length}@${at}/${subject.length}`)); +console.log("one two one".replace("two", (m, at, subject, extra) => `${m}${at}${subject.length}${extra}`)); + +// A throw stops the replacement. +try { + "a1b2c3".replace(/(\d)/g, (_m, d) => { + if (d === "2") throw new Error("stop at " + d); + return "#"; + }); +} catch (error) { + console.log((error as Error).message); +} +try { + "a1".replace(/(\d)/g, (_m, d) => counts.e.missing.field + d); +} catch (error) { + console.log(error instanceof TypeError); +} +try { + "ab".replace(/(a)|b/g, (m, a) => a.toUpperCase()); +} catch (error) { + console.log(error instanceof TypeError, (error as Error).message); +} + +let total = 0; +for (let i = 0; i < 2000; i++) total += template.replace(/\{(\w+)\}/g, (_m, key) => vars[key] ?? String(i)).length; +console.log(total); diff --git a/tests/corpus/8346-replace-callback-untyped-params.mjs b/tests/corpus/8346-replace-callback-untyped-params.mjs new file mode 100644 index 000000000..2552113f3 --- /dev/null +++ b/tests/corpus/8346-replace-callback-untyped-params.mjs @@ -0,0 +1,32 @@ +// JavaScript string replace/replaceAll with inline callbacks. Each +// parameter holds what Node passes at its position: the captures, the +// offset, the subject, the groups object and then undefined. Captures a +// match can leave unset hold undefined, and a result that is not a string +// converts with ToString. +const vars = { name: "world", place: "earth" }; +const template = "Hello {name}, welcome to {place}! {missing} {name}."; +console.log(template.replace(/\{(\w+)\}/g, (_m, key) => vars[key] ?? `<${key}>`)); +console.log("some_snake_case_name".replace(/_([a-z])/g, (_m, c) => c.toUpperCase())); +console.log("2024-01-15 and 1999-07-04".replace(/(\d{4})-(\d{2})-(\d{2})/g, (_m, y, mo, d) => `${d}/${mo}/${y}`)); +console.log("a1b22".replace(/\d+/g, (m, offset, subject) => `[${m}@${offset}/${subject.length}]`)); +console.log("k=v".replace(/(\w)=(\w)/, function (m, k, v, at, all) { + return `${this === undefined}:${v}=${k}:${at}:${all}:${m}`; +})); +console.log("ab".replace(/(a)|(b)/g, (m, a, b) => `${m}:${a}:${b};`)); +console.log("b xb".replace(/(x)?b/g, (m, x, at) => `${m}:${x}:${at}`)); +console.log("2024-01".replace(/(?\d{4})-(?\d{2})/, (m, y, mo, at, all, groups) => `${groups.m}.${groups.y}:${y}${mo}:${at}:${all}`)); +console.log("ab".replace(/b/, (m, at, all, extra, more) => `${m}:${at}:${all}:${extra}:${more}`)); +const counts = { a: 1, b: true, c: null, d: undefined, e: { toString: () => "E" } }; +console.log("a b c d e".replace(/\w/g, (m) => counts[m])); +console.log("1-2".replace(/(\d)/g, (_m, d) => Number(d) * 10)); +console.log("one two one".replaceAll("one", (m, at, subject) => `${m.length}@${at}/${subject.length}`)); +let seen = ""; +console.log("xyz".replace(/(y)/, (m, y, at, subject) => { + seen = typeof m + typeof y + typeof at + typeof subject; + return y + at; +}), seen); +try { + "ab".replace(/(a)|b/g, (m, a) => a.toUpperCase()); +} catch (error) { + console.log(error instanceof TypeError, error.message); +} diff --git a/tests/harness/corpus-costs/8118-replacer-params.ts.json b/tests/harness/corpus-costs/8118-replacer-params.ts.json new file mode 100644 index 000000000..9b0bcec50 --- /dev/null +++ b/tests/harness/corpus-costs/8118-replacer-params.ts.json @@ -0,0 +1 @@ +{ "plain": 3670, "san": 4410 } diff --git a/tests/harness/corpus-costs/8345-replace-callback-untyped-params.ts.json b/tests/harness/corpus-costs/8345-replace-callback-untyped-params.ts.json new file mode 100644 index 000000000..5d5bbcefe --- /dev/null +++ b/tests/harness/corpus-costs/8345-replace-callback-untyped-params.ts.json @@ -0,0 +1 @@ +{ "plain": 3630, "san": 4420 } diff --git a/tests/harness/corpus-costs/8346-replace-callback-untyped-params.mjs.json b/tests/harness/corpus-costs/8346-replace-callback-untyped-params.mjs.json new file mode 100644 index 000000000..cdddf6d76 --- /dev/null +++ b/tests/harness/corpus-costs/8346-replace-callback-untyped-params.mjs.json @@ -0,0 +1 @@ +{ "plain": 2880, "san": 5080 }