From 7f4f2a76fcdf5a55350e38bc43e4070352bc6df2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 13 Oct 2026 20:30:01 -0500 Subject: [PATCH] Speed up regex replace, match and capture reads --- .../compiler/src/backend/llvm/expr-records.ts | 13 +- .../compiler/src/backend/llvm/lib-shared.ts | 4 + .../src/backend/llvm/reference-effects.ts | 2 + .../src/backend/llvm/reference-writes.ts | 3 + .../lowering/containers/string-and-regexp.ts | 90 ++++++ .../src/frontend/lowering/lower-calls.ts | 18 +- .../compiler/src/frontend/lowering/lowerer.ts | 5 + .../src/frontend/lowering/regex-span-reads.ts | 195 +++++++++++++ .../compiler/src/ir/builtin-signatures.ts | 9 + .../compiler/src/ir/intrinsic-signatures.ts | 2 + packages/compiler/src/ir/ir.ts | 9 + packages/compiler/src/ir/validate.ts | 15 +- .../test/regex-capture-emission.test.ts | 90 ++++++ .../5115-regex-replace-compiled.ts.json | 6 + .../corpus/5116-regex-capture-reads.ts.json | 6 + .../5117-regex-log-report-spans.ts.json | 6 + .../5118-regex-replace-char-chains.ts.json | 6 + packages/runtime/src/scr_regex.c | 272 +++++++++++++++++- packages/runtime/src/scr_runtime.h | 21 ++ packages/runtime/src/scr_string.c | 148 ++++++++-- packages/runtime/test/test_regex.c | 95 ++++++ tests/corpus/5115-regex-replace-compiled.ts | 42 +++ tests/corpus/5116-regex-capture-reads.ts | 78 +++++ tests/corpus/5117-regex-log-report-spans.ts | 73 +++++ .../corpus/5118-regex-replace-char-chains.ts | 18 ++ 25 files changed, 1187 insertions(+), 39 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/regex-span-reads.ts create mode 100644 packages/compiler/test/regex-capture-emission.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5115-regex-replace-compiled.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5116-regex-capture-reads.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5117-regex-log-report-spans.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5118-regex-replace-char-chains.ts.json create mode 100644 tests/corpus/5115-regex-replace-compiled.ts create mode 100644 tests/corpus/5116-regex-capture-reads.ts create mode 100644 tests/corpus/5117-regex-log-report-spans.ts create mode 100644 tests/corpus/5118-regex-replace-char-chains.ts diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index 202cb9af1..6820d4752 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -134,6 +134,8 @@ function directRegexOperandMethod(method: string): boolean { switch (method) { case "exec": case "execGroups": + case "execSpans": + case "matchGroups": case "match": case "search": case "matchAll": @@ -237,7 +239,9 @@ export function emitRegexIntrinsic( } case "match": case "exec": - case "execGroups": { + case "execGroups": + case "execSpans": + case "matchGroups": { // +1 string[] or NULL from the runtime; the `string[] | null` // union wraps type-directedly, the envGet convention. if (e.type.kind !== "union") @@ -255,6 +259,13 @@ export function emitRegexIntrinsic( B.line( `${raw} = call ptr @scr_regex_exec_groups(ptr ${r.name}, ptr ${args[0]!.name}, double ${args[1]!.name})`, ); + } else if (e.method === "execSpans" || e.method === "matchGroups") { + // args[1] selects the read captures, args[2] the lazily read ones. + const symbol = e.method === "execSpans" ? "scr_regex_exec_spans" : "scr_regex_match_groups"; + host.declare(`declare ptr @${symbol}(ptr, ptr, double, double)`); + B.line( + `${raw} = call ptr @${symbol}(ptr ${r.name}, ptr ${args[0]!.name}, double ${args[1]!.name}, double ${args[2]!.name})`, + ); } else { const symbol = e.method === "exec" ? "scr_regex_exec" : "scr_regex_match"; host.declare(`declare ptr @${symbol}(ptr, ptr)`); diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index a695cead0..451d63259 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -78,6 +78,7 @@ export const LIB_FN_SYMS: Record = { "num.parseFloat": "scr_parse_float", "str.localeCompare": "scr_str_locale_compare", "str.replaceLiteral": "scr_str_replace_literal", + "str.replaceChars": "scr_str_replace_chars", "num.fromString": "scr_string_to_number", "math.round": "scr_math_round", "math.sin": "sin", @@ -1060,6 +1061,9 @@ export const LIB_FN_SYMS: Record = { "regex.resetLastIndex": "scr_regex_reset_last_index", "regex.newChecked": "scr_regex_new_checked", "regex.replaceCallback": "scr_regex_replace_callback", + "regex.groupEquals": "scr_regex_group_eq", + "regex.groupNumber": "scr_regex_group_number", + "regex.groupParseInt": "scr_regex_group_parse_int", // queueMicrotask's checked-dynamic form (borrowed dyn; a non-function // throws synchronously), the minted setImmediate value, and the // timers/promises immediate — all mark the loop live. diff --git a/packages/compiler/src/backend/llvm/reference-effects.ts b/packages/compiler/src/backend/llvm/reference-effects.ts index c2a9a0afa..ecee62e8e 100644 --- a/packages/compiler/src/backend/llvm/reference-effects.ts +++ b/packages/compiler/src/backend/llvm/reference-effects.ts @@ -24,6 +24,8 @@ export function preservesRegexInputs(method: string): boolean { case "test": case "exec": case "execGroups": + case "execSpans": + case "matchGroups": case "match": case "search": case "matchAll": diff --git a/packages/compiler/src/backend/llvm/reference-writes.ts b/packages/compiler/src/backend/llvm/reference-writes.ts index 0eb5544a8..3a05ce0dd 100644 --- a/packages/compiler/src/backend/llvm/reference-writes.ts +++ b/packages/compiler/src/backend/llvm/reference-writes.ts @@ -100,6 +100,9 @@ const INERT_LIB_CALLS = new Set([ "process.stdoutWrite", "process.envGet", "process.envPairs", + "regex.groupEquals", + "regex.groupNumber", + "regex.groupParseInt", "process.cwd", "process.argv", "process.platform", diff --git a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts index fea422068..361441636 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-and-regexp.ts @@ -771,6 +771,7 @@ export function lowerRegexMethodCall( // A result read only through null tests and constant indices skips // the captures and properties no reader uses. const mask = execResultCaptureMask(lowerer, call); + if (mask !== null) lowerer.regexGroupResults = true; const execCall = (regex: IrExpr): IrExpr => ({ kind: "regexIntrinsic", method: mask === null ? "exec" : "execGroups", @@ -835,6 +836,20 @@ export function lowerRegexMethodCall( }; const mapped = lowerer.mapTypeOf(lowerer.typeOf(call)); const resultT: IrType = mapped && typeEquals(mapped, exactT) ? mapped : exactT; + // A result read like execGroups' skips the same captures when the + // regex is not global (a global regex still answers every match). + const mask = execResultCaptureMask(lowerer, call); + if (mask !== null) { + lowerer.regexGroupResults = true; + return { + kind: "regexIntrinsic", + method: "matchGroups", + receiver, + args: [re, numLit(mask, loc), numLit(0, loc)], + type: resultT, + loc, + }; + } return { kind: "regexIntrinsic", method: "match", receiver, args: [re], type: resultT, loc }; } // `s.matchAll(re)` — the every-match iterator drained EAGERLY into a @@ -873,6 +888,10 @@ export function lowerRegexMethodCall( const re = lowerer.lowerExprExpecting(arg0, { kind: "regex" }); return { kind: "regexIntrinsic", method: "search", receiver, args: [re], type: F64, loc }; } + if (receiverKind === "string" && (name === "replace" || name === "replaceAll")) { + const chain = lowerReplaceCharChain(lowerer, call); + if (chain) return chain; + } if ( receiverKind === "string" && (name === "replace" || name === "replaceAll" || name === "split") @@ -947,6 +966,77 @@ export function lowerRegexMethodCall( return null; } +/** `s.replace(/&/g, "&").replace(/ 16) return null; + steps.reverse(); + for (let i = 0; i < steps.length; i++) { + for (let j = i + 1; j < steps.length; j++) { + if (steps[j]!.char === steps[i]!.char || steps[i]!.text.includes(steps[j]!.char)) return null; + } + } + const loc = locOf(call); + return { + kind: "libCall", + fn: "str.replaceChars", + args: [ + lowerer.lowerExprExpecting(node, STRING), + strLit(steps.map((step) => step.char).join(""), loc), + strLit(steps.map((step) => step.text).join("\0"), loc), + ], + type: STRING, + loc, + }; +} + +/** The character a `/c/g` literal matches, for one printable ASCII + * character or an escaped syntax character; null otherwise. */ +function globalCharPattern(text: string): string | null { + if (!text.endsWith("/g")) return null; + const body = text.slice(1, -2); + const syntax = "\\^$.|?*+()[]{}/-"; + if (body.length === 1) { + const code = body.charCodeAt(0); + return code >= 0x20 && code < 0x7f && !syntax.includes(body) ? body : null; + } + if (body.length === 2 && body[0] === "\\" && syntax.includes(body[1]!)) return body[1]!; + return null; +} + /** `s.slice(1, 4)` and friends → strIntrinsic. Null when this isn't an * ambient string method call (caller keeps its generic rejection). * Complete position defaults and conversions here before code generation. */ diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 3d2d4c624..208885d5d 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -9151,10 +9151,26 @@ export function lowerStaticNumberGlobal( type: BOOL, loc, }, + // parseInt(undefined, radix) parses the text + // "undefined" (a number from radix 31 up). then: [ { kind: "return", - value: { kind: "numLit", value: NaN, type: F64, loc }, + value: { + kind: "libCall", + fn: "num.parseInt", + args: [ + { + kind: "strLit", + value: arm.kind === "nullT" ? "null" : "undefined", + type: STRING, + loc, + }, + radixRef, + ], + type: F64, + loc, + }, loc, }, ], diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index d96a702ad..388341d68 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -62,6 +62,7 @@ import { utilTypesModuleValue, } from "./lower-builtin-values.js"; import { devirtualizeVirtualCalls } from "./devirtualize.js"; +import { foldRegexSpanReads } from "./regex-span-reads.js"; import { RuntimeOptionalProvenance } from "./runtime-optional-provenance.js"; import { presentAtUse, reassignedBindings } from "./runtime-optional-guards.js"; import { NarrowingStability } from "./narrowing-stability.js"; @@ -1750,6 +1751,9 @@ export class Lowerer { * prepass. This includes explicit annotations: unchecked TS annotations do * not prove that an array property read produced a value. */ readonly runtimeOptionalBindingTypes = new Map(); + /** Set when a match result lowers to execGroups/matchGroups, the inputs + * of regex span folding. */ + regexGroupResults = false; /** Bindings that receive an unchecked indexed read (or a value derived * from one) through an initializer or assignment. The checker narrows a * declared optional binding to that read's bare type, so presence tests @@ -6342,6 +6346,7 @@ export class Lowerer { ...this.liftedFns, ...this.implicitFns, ]; + if (this.regexGroupResults) for (const fn of functions) foldRegexSpanReads(this, fn); // The worklist can leave a virtual site with a single implementation. // Devirtualize after construction/override reachability has settled. devirtualizeVirtualCalls(this, functions, { diff --git a/packages/compiler/src/frontend/lowering/regex-span-reads.ts b/packages/compiler/src/frontend/lowering/regex-span-reads.ts new file mode 100644 index 000000000..a196e2299 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/regex-span-reads.ts @@ -0,0 +1,195 @@ +import { numLit } from "../../ir/build.js"; +import { + BOOL, + F64, + STRING, + type IrExpr, + type IrFunction, + type IrStmt, + typeKey, +} from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild, transformStmtList } from "../../ir/traverse.js"; +import type { Lowerer } from "./lowerer.js"; + +type Intrinsic = IrExpr & { kind: "regexIntrinsic" }; + +/** One read of capture `index` of a candidate result: the IR node that + * consumes it and the narrowed result it reads from. */ +interface CaptureUse { + index: number; + result: IrExpr; +} + +interface Candidate { + match: Intrinsic; + folds: Map; + strings: Set; + rejected: boolean; +} + +/** Captures that need no string. An execGroups or matchGroups result + * (execResultCaptureMask's const binding) whose capture k is read only by + * strict comparisons with a string literal, Number() and parseInt() lets + * the runtime keep capture k as a span of the subject: those reads become + * regex.groupEquals/groupNumber/groupParseInt and the match becomes + * execSpans (or matchGroups) with the lazy mask. The pass reads the + * lowered IR, so it sees every read: any other read of capture k keeps k a + * string, and any use of the binding it cannot attribute to a capture + * leaves the whole result unchanged. */ +export function foldRegexSpanReads(lowerer: Lowerer, fn: IrFunction): void { + const candidates = new Map(); + const assigned = new Set(); + const findStmt = (stmt: IrStmt): boolean => { + // A non-literal receiver is saved first: the call is a sequence result. + const init = + stmt.kind !== "varDecl" ? null : stmt.init?.kind === "seqExpr" ? stmt.init.result : stmt.init; + if ( + stmt.kind === "varDecl" && + init?.kind === "regexIntrinsic" && + (init.method === "execGroups" || init.method === "matchGroups") + ) { + candidates.set(stmt.localId, { + match: init, + folds: new Map(), + strings: new Set(), + rejected: candidates.has(stmt.localId), + }); + } + if (stmt.kind === "assign") assigned.add(stmt.localId); + return everyStmtChild(stmt, findExpr, findStmt); + }; + const findExpr = (expr: IrExpr): boolean => everyExprChild(expr, findExpr, findStmt); + fn.body.every(findStmt); + if (candidates.size === 0) return; + for (const id of assigned) candidates.delete(id); + + const stringIndexHelper = lowerer.arrHofHelpers.get(`idxOr:${typeKey(STRING)}`); + /** `m[k]` as lowered: the optional element read of the narrowed result + * (or, for `m[k]!`, the dense read). */ + const captureRead = (expr: IrExpr): { local: string; use: CaptureUse } | null => { + let result: IrExpr | undefined; + let index: IrExpr | undefined; + if (expr.kind === "call" && expr.callee === stringIndexHelper && expr.args.length === 2) + [result, index] = expr.args; + else if (expr.kind === "arrayGet") ({ arr: result, index } = expr); + else return null; + if ( + result?.kind !== "unionNarrow" || + result.value.kind !== "varRef" || + index?.kind !== "numLit" || + !Number.isInteger(index.value) || + index.value < 0 || + index.value >= 53 + ) + return null; + return { local: result.value.localId, use: { index: index.value, result } }; + }; + /** The capture read a foldable consumer takes, and its other operands. */ + const foldable = (expr: IrExpr): { read: IrExpr; rest: IrExpr[] } | null => { + if (expr.kind === "unionEq" && !expr.sameValue) { + if (isLiteral(expr.right)) return { read: expr.left, rest: [] }; + if (isLiteral(expr.left)) return { read: expr.right, rest: [] }; + return null; + } + if ( + expr.kind === "libCall" && + expr.fn === "dyn.numberConstructor" && + expr.args[0]?.kind === "dynFrom" + ) + return { read: expr.args[0].value, rest: [] }; + if ( + expr.kind === "call" && + expr.args.length === 2 && + expr.args[0]!.type.kind === "union" && + expr.callee === lowerer.valueHelpers.get(`parseInt.optional:${expr.args[0]!.type.unionId}`) + ) + return { read: expr.args[0]!, rest: [expr.args[1]!] }; + return null; + }; + + const visitExpr = (expr: IrExpr, parent: IrExpr | null): boolean => { + const fold = foldable(expr); + const folded = fold && fold.read.kind === "call" ? captureRead(fold.read) : null; + const candidate = folded && candidates.get(folded.local); + if (fold && folded && candidate) { + candidate.folds.set(expr, folded.use); + for (const operand of fold.rest) visitExpr(operand, expr); + return true; + } + const read = captureRead(expr); + const reader = read && candidates.get(read.local); + if (read && reader) { + reader.strings.add(read.use.index); + return true; + } + if (expr.kind === "varRef") { + const use = candidates.get(expr.localId); + // Null tests are the only other use the binding may have. + if (use && parent?.kind !== "unionIsTag") use.rejected = true; + return true; + } + return everyExprChild( + expr, + (child) => visitExpr(child, expr), + (stmt) => visitStmt(stmt), + ); + }; + const visitStmt = (stmt: IrStmt): boolean => + everyStmtChild(stmt, (child) => visitExpr(child, null), visitStmt); + fn.body.every(visitStmt); + + const rewrites = new Map(); + for (const candidate of candidates.values()) { + if (candidate.rejected) continue; + let lazy = 0; + const lazyIndices = new Set(); + for (const use of candidate.folds.values()) { + if (!candidate.strings.has(use.index) && !lazyIndices.has(use.index)) { + lazyIndices.add(use.index); + lazy += 2 ** use.index; + } + } + if (lazy === 0) continue; + for (const [consumer, use] of candidate.folds) { + if (lazyIndices.has(use.index)) rewrites.set(consumer, spanRead(consumer, use)); + } + const { match } = candidate; + const mask = match.args[1]!; + rewrites.set(match, { + ...match, + method: match.method === "matchGroups" ? "matchGroups" : "execSpans", + args: [match.args[0]!, mask, numLit(lazy, match.loc)], + }); + } + if (rewrites.size === 0) return; + fn.body = transformStmtList(fn.body, { + stmt: (stmt) => stmt, + expr: (expr) => rewrites.get(expr) ?? expr, + }); +} + +function isLiteral(expr: IrExpr): boolean { + return expr.kind === "unionWrap" && expr.value.kind === "strLit"; +} + +/** The span form of a foldable consumer of `use`. */ +function spanRead(consumer: IrExpr, use: CaptureUse): IrExpr { + const loc = consumer.loc; + const capture = [use.result, numLit(use.index, loc)]; + if (consumer.kind === "unionEq") { + const literal = isLiteral(consumer.right) ? consumer.right : consumer.left; + const value = (literal as IrExpr & { kind: "unionWrap" }).value; + const test: IrExpr = { + kind: "libCall", + fn: "regex.groupEquals", + args: [...capture, value], + type: BOOL, + loc, + }; + return consumer.negated ? { kind: "unary", op: "!", operand: test, type: BOOL, loc } : test; + } + if (consumer.kind === "libCall") + return { kind: "libCall", fn: "regex.groupNumber", args: capture, type: F64, loc }; + const radix = (consumer as IrExpr & { kind: "call" }).args[1]!; + return { kind: "libCall", fn: "regex.groupParseInt", args: [...capture, radix], type: F64, loc }; +} diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 4a21fba91..2f12e176b 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -527,6 +527,9 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * Latin-script text (scr_str_locale_compare). Never throws. */ "str.localeCompare": { argTypes: [STRING, STRING], result: F64 }, "str.replaceLiteral": { argTypes: [STRING, STRING, STRING, BOOL], result: STRING }, + // (subject, characters, NUL-separated replacement texts): the fused + // chain of global one-character replacements. + "str.replaceChars": { argTypes: [STRING, STRING, STRING], result: STRING }, /* ToNumber(string) — ECMA-262 7.1.4.1 StringToNumber (scr_string.c): * trim the JS StrWhiteSpace set, empty/whitespace-only → +0, then the * whole span must be one StrNumericLiteral — signed decimal (Infinity @@ -2707,6 +2710,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "regex.resetLastIndex": { argTypes: [REGEX, F64], result: VOID }, "regex.newChecked": { argTypes: [DYN, DYN], result: REGEX }, "regex.replaceCallback": { argTypes: [STRING, REGEX, DYN, BOOL], result: STRING }, + // Reads of capture args[1] of an execSpans/matchGroups result: + // `capture === args[2]`, Number(capture) and parseInt(capture, + // args[2]). Borrowed; never throw. + "regex.groupEquals": { argTypes: [arrayOf(STRING), F64, STRING], result: BOOL }, + "regex.groupNumber": { argTypes: [arrayOf(STRING), F64], result: F64 }, + "regex.groupParseInt": { argTypes: [arrayOf(STRING), F64, F64], result: F64 }, // node:events EventEmitter: receivers are emitter-hierarchy objects and // the chaining forms (on/off/removeAll/setMax) return the receiver's // own class — program-dependent object types, checked in the libCall diff --git a/packages/compiler/src/ir/intrinsic-signatures.ts b/packages/compiler/src/ir/intrinsic-signatures.ts index c456a75bd..3c224f8bc 100644 --- a/packages/compiler/src/ir/intrinsic-signatures.ts +++ b/packages/compiler/src/ir/intrinsic-signatures.ts @@ -49,6 +49,8 @@ export const REGEX_INTRINSIC_SIGS: Record< match: { receiver: STRING, argTypes: [REGEX], result: VOID }, exec: { receiver: STRING, argTypes: [REGEX], result: VOID }, execGroups: { receiver: STRING, argTypes: [REGEX, F64], result: VOID }, + execSpans: { receiver: STRING, argTypes: [REGEX, F64, F64], result: VOID }, + matchGroups: { receiver: STRING, argTypes: [REGEX, F64, F64], result: VOID }, lastIndex: { receiver: REGEX, argTypes: [], result: F64 }, matchAll: { receiver: STRING, argTypes: [REGEX], result: arrayOf(arrayOf(STRING)) }, matchAllInto: { diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index f36301aa0..095755f1f 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -2198,6 +2198,15 @@ export type IrRegexIntrinsicMethod = * carries no index/input properties; the match, lastIndex transitions * and null result are exec's. */ | "execGroups" + /** execGroups whose args[2] (a numeric literal, a subset of args[1]) + * marks the captures read only through the regex.groupEquals/ + * groupNumber/groupParseInt libCalls; the runtime may keep those as + * spans of the subject rather than strings. */ + | "execSpans" + /** `s.match(re)` read like an execGroups result: a global regex answers + * match's whole-match array; otherwise execSpans with args[1], args[2] + * (args[2] is 0 until regex span folding marks lazy captures). */ + | "matchGroups" | "lastIndex" /** `s.matchAll(re)` — every match as its honest string[] slice (match's * rule), drained EAGERLY into a fresh string[][]: the lazy iterator is diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index e7e244996..09684f5c6 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -1954,10 +1954,19 @@ function validateFunction( const want = sig.argTypes[i]; if (want) expectType(a, want, `regexIntrinsic ${e.method} arg ${i}`); }); - if (e.method === "execGroups" && e.args[1]?.kind !== "numLit") { - err("regexIntrinsic execGroups mask must be a numeric literal", e.loc); + if ( + (e.method === "execGroups" || e.method === "execSpans" || e.method === "matchGroups") && + e.args.slice(1).some((mask) => mask.kind !== "numLit") + ) { + err(`regexIntrinsic ${e.method} masks must be numeric literals`, e.loc); } - if (e.method === "match" || e.method === "exec" || e.method === "execGroups") { + if ( + e.method === "match" || + e.method === "exec" || + e.method === "execGroups" || + e.method === "execSpans" || + e.method === "matchGroups" + ) { // The `string[] | null` union (program-dependent id) — checked // by arms, like the libCall case checks process.envGet. const def = e.type.kind === "union" ? unions.get(e.type.unionId) : undefined; diff --git a/packages/compiler/test/regex-capture-emission.test.ts b/packages/compiler/test/regex-capture-emission.test.ts new file mode 100644 index 000000000..59ff6f748 --- /dev/null +++ b/packages/compiler/test/regex-capture-emission.test.ts @@ -0,0 +1,90 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all(directories.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function emit(source: string): Promise { + const outDir = await mkdtemp(join(tmpdir(), "scriptc-regex-")); + directories.push(outDir); + const entry = join(outDir, "main.ts"); + await writeFile(entry, source); + const outPath = join(outDir, "program.ll"); + const result = await compile(entry, { entry, outDir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result)).toBe(true); + return readFile(outPath, "utf8"); +} + +const calls = (llvm: string, symbol: string): string[] => + llvm.split("\n").filter((line) => line.includes(`call `) && line.includes(`@${symbol}(`)); + +test("captures only compared or converted to numbers stay spans", async () => { + const llvm = await emit(` +const line = /^(\\w+) (\\d+) (\\w+)$/; +let total = 0; +for (const s of ["GET 200 a", "PUT 404 b"]) { + const m = line.exec(s); + if (m === null) continue; + if (m[1] === "GET") total += Number(m[2]) + parseInt(m[2], 16); + console.log(m[3]); +} +console.log(total); +`); + // Captures 1 and 2 are lazy (6); capture 3 is read as a string (mask 14). + const exec = calls(llvm, "scr_regex_exec_spans"); + expect(exec).toHaveLength(1); + expect(exec[0]).toContain("double 0x402C000000000000, double 0x4018000000000000"); + expect(calls(llvm, "scr_regex_group_eq")).toHaveLength(1); + expect(calls(llvm, "scr_regex_group_number")).toHaveLength(1); + expect(calls(llvm, "scr_regex_group_parse_int")).toHaveLength(1); +}); + +test("a capture read as a string, or a result captured by a closure, keeps strings", async () => { + const llvm = await emit(` +const kv = /(\\w+)=(\\w+)/; +const a = kv.exec("x=1"); +if (a !== null) console.log(a[2] === "1", a[2]); +const b = kv.exec("y=2"); +if (b !== null) { + const later = (): boolean => b[1] === "y"; + console.log(later(), Number(b[2])); +} +`); + expect(calls(llvm, "scr_regex_exec_spans")).toHaveLength(0); + expect(calls(llvm, "scr_regex_exec_groups")).toHaveLength(2); + expect(calls(llvm, "scr_regex_group_eq")).toHaveLength(0); +}); + +test("match results read by constant indices take the group masks", async () => { + const llvm = await emit(` +for (const s of ["v1.2", "x"]) { + const m = s.match(/^v(\\d+)\\.(\\d+)$/); + if (m !== null) console.log(Number(m[1]), m[2]); +} +`); + const match = calls(llvm, "scr_regex_match_groups"); + expect(match).toHaveLength(1); + // Read captures 1 and 2 (6); capture 1 is lazy (2). + expect(match[0]).toContain("double 0x4018000000000000, double 0x4000000000000000"); + expect(calls(llvm, "scr_regex_match")).toHaveLength(0); +}); + +test("chains of one-character global replacements fuse into one pass", async () => { + const llvm = await emit(` +function escape(text: string): string { + return text.replace(/&/g, "&").replace(/&'), ordered("a&b")); +`); + expect(calls(llvm, "scr_str_replace_chars")).toHaveLength(1); + // A later step that matches an earlier replacement keeps both steps. + expect(calls(llvm, "scr_regex_replace")).toHaveLength(2); +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5115-regex-replace-compiled.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5115-regex-replace-compiled.ts.json new file mode 100644 index 000000000..e241208db --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5115-regex-replace-compiled.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5115-regex-replace-compiled.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5116-regex-capture-reads.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5116-regex-capture-reads.ts.json new file mode 100644 index 000000000..08bb51767 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5116-regex-capture-reads.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5116-regex-capture-reads.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5117-regex-log-report-spans.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5117-regex-log-report-spans.ts.json new file mode 100644 index 000000000..6e130c3c8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5117-regex-log-report-spans.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5117-regex-log-report-spans.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5118-regex-replace-char-chains.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5118-regex-replace-char-chains.ts.json new file mode 100644 index 000000000..c1772fdf9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5118-regex-replace-char-chains.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5118-regex-replace-char-chains.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 25bd99ed3..37d5222b5 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -640,12 +640,29 @@ bool scr_regex_test(ScrRegex *re, ScrStr *s) { /* ── match ────────────────────────────────────────────────────────────── */ +/* Lazily materialized captures. A capture the compiler proved is only + * compared with a literal or converted to a number stays a span when the + * engine matched the subject's own bytes (an ASCII subject): + * its slot is a present undefined whose payload word packs the capture's + * byte offsets (start << 32 | end), SCR_RE_NO_SPAN when it did not + * participate, and the retained subject is appended after the captures. + * The compiler reads such a result only through constant indices below + * the capture count, so neither the extra element nor the undefined slots + * are observable. Every other case stores lazy captures as strings, and + * non-participating lazy captures keep SCR_RE_NO_SPAN. */ +#define SCR_RE_NO_SPAN UINT64_MAX + +static uint64_t scr_regex_mask_bits(double mask) { + return mask >= 0 && mask < 18446744073709551616.0 ? (uint64_t)mask : UINT64_MAX; +} + /* re.exec(s): a fresh string[] of [whole match, ...captures], or NULL * for no match (the compiler wraps the `string[] | null` union). Global * and sticky regexes start at lastIndex, set it to the match end, and * reset it on failure. Nonparticipating captures are present undefined * elements, distinct from participating empty captures and array holes. */ -static ScrArr *scr_regex_exec_core(ScrStr *s, ScrRegex *re, uint64_t mask, bool metadata) { +static ScrArr *scr_regex_exec_core(ScrStr *s, ScrRegex *re, uint64_t mask, bool metadata, + uint64_t lazy) { uint8_t *bc = scr_regex_bc(re); bool stateful = (lre_get_flags(bc) & (LRE_FLAG_GLOBAL | LRE_FLAG_STICKY)) != 0; ScrRegexInput input; @@ -659,35 +676,98 @@ static ScrArr *scr_regex_exec_core(ScrStr *s, ScrRegex *re, uint64_t mask, bool ScrArr *out = NULL; if (rc == 1) { int count = lre_get_capture_count(bc); - out = scr_arr_new(SCR_ELEM_STR, (size_t)count); + bool spans = lazy != 0 && !input.wide && (count >= 64 || ((mask | lazy) >> count) == 0); + out = scr_arr_new(SCR_ELEM_STR, (size_t)count + spans); if (metadata) { ScrDyn *props = scr_dyn_new_obj(); scr_arr_ext(out)->metadata = props; scr_dyn_obj_set(props, "index", 5, scr_dyn_new_num((double)scr_capture_index(&input, capture[0]))); scr_dyn_obj_set(props, "input", 5, scr_dyn_new_str(s)); } + /* A fresh dense array: every slot is written directly, then the + * hole-free prefix is the run of materialized captures from 0. */ + size_t n = (size_t)count + spans; + bool prefix = true; for (int k = 0; k < count; k++) { const uint8_t *cs = capture[2 * k], *ce = capture[2 * k + 1]; + bool deferred = k < 64 && (lazy >> k & 1); + uint64_t word = 0; + uint8_t state = SCR_ARR_UNDEFINED; if (cs == NULL || ce == NULL || (k < 64 && !(mask >> k & 1))) { - scr_arr_set_undefined(out, (double)out->len); + if (deferred) word = SCR_RE_NO_SPAN; + } else if (deferred && spans) { + word = (uint64_t)(cs - input.data) << 32 | (uint64_t)(ce - input.data); } else { - scr_arr_push_ref( - out, scr_regex_span(&input, scr_capture_index(&input, cs), scr_capture_index(&input, ce))); + ScrStr *text = + scr_regex_span(&input, scr_capture_index(&input, cs), scr_capture_index(&input, ce)); + word = (uint64_t)(uintptr_t)text; + state = SCR_ARR_VALUE; } + out->data[k] = word; + out->present[k] = state; + if (state != SCR_ARR_VALUE) prefix = false; + if (prefix) out->filled = (uint32_t)k + 1; } + if (spans) { + out->data[count] = (uint64_t)(uintptr_t)scr_str_retain(s); + out->present[count] = SCR_ARR_VALUE; + if (prefix) out->filled = (uint32_t)n; + } + out->len = n; } scr_regex_input_dispose(&input); return out; /* +1, or NULL (no match) */ } ScrArr *scr_regex_exec(ScrStr *s, ScrRegex *re) { - return scr_regex_exec_core(s, re, UINT64_MAX, true); + return scr_regex_exec_core(s, re, UINT64_MAX, true, 0); } /* Unselected captures are present undefined elements: nothing reads them. */ ScrArr *scr_regex_exec_groups(ScrStr *s, ScrRegex *re, double mask) { - uint64_t bits = mask >= 0 && mask < 18446744073709551616.0 ? (uint64_t)mask : UINT64_MAX; - return scr_regex_exec_core(s, re, bits, false); + return scr_regex_exec_core(s, re, scr_regex_mask_bits(mask), false, 0); +} + +ScrArr *scr_regex_exec_spans(ScrStr *s, ScrRegex *re, double mask, double lazy) { + return scr_regex_exec_core(s, re, scr_regex_mask_bits(mask), false, scr_regex_mask_bits(lazy)); +} + +/* The bytes of capture k, false when it reads as undefined. */ +static bool scr_regex_group_bytes(const ScrArr *m, double k, const char **p, size_t *n) { + if (!(k >= 0 && k < (double)m->len)) return false; + size_t i = (size_t)k; + if (m->present[i] == SCR_ARR_VALUE) { + const ScrStr *str = (const ScrStr *)(uintptr_t)m->data[i]; + *p = str->data; + *n = str->len; + return true; + } + uint64_t span = m->data[i]; + if (m->present[i] != SCR_ARR_UNDEFINED || span == SCR_RE_NO_SPAN) return false; + const ScrStr *subject = (const ScrStr *)(uintptr_t)m->data[m->len - 1]; + *p = subject->data + (span >> 32); + *n = (size_t)((span & UINT32_MAX) - (span >> 32)); + return true; +} + +bool scr_regex_group_eq(ScrArr *m, double k, ScrStr *lit) { + const char *p; + size_t n; + return scr_regex_group_bytes(m, k, &p, &n) && n == lit->len && memcmp(p, lit->data, n) == 0; +} + +double scr_regex_group_number(ScrArr *m, double k) { + const char *p; + size_t n; + return scr_regex_group_bytes(m, k, &p, &n) ? scr_bytes_to_number(p, n) : NAN; +} + +double scr_regex_group_parse_int(ScrArr *m, double k, double radix) { + const char *p; + size_t n; + /* parseInt(undefined, radix) parses the string "undefined". */ + if (!scr_regex_group_bytes(m, k, &p, &n)) return scr_parse_int_bytes("undefined", 9, radix); + return scr_parse_int_bytes(p, n, radix); } /* Global Symbol.match always starts at zero and drains whole matches, @@ -695,6 +775,11 @@ ScrArr *scr_regex_exec_groups(ScrStr *s, ScrRegex *re, double mask) { * to zero, so no state escapes this internal iteration. Sticky globals * stop at the first gap; empty matches advance by a Unicode code point * under /u and by one UTF-16 unit otherwise. */ +ScrArr *scr_regex_match_groups(ScrStr *s, ScrRegex *re, double mask, double lazy) { + if (lre_get_flags(scr_regex_bc(re)) & LRE_FLAG_GLOBAL) return scr_regex_match(s, re); + return scr_regex_exec_spans(s, re, mask, lazy); +} + ScrArr *scr_regex_match(ScrStr *s, ScrRegex *re) { uint8_t *bc = scr_regex_bc(re); int flags = lre_get_flags(bc); @@ -904,6 +989,163 @@ static void scr_put_substitution(ScrJsonBuf *b, const ScrRegexInput *input, } } +/* ── direct replace over a build-time matcher ───────────────────────── + * An ASCII subject of a regex with a build-time matcher needs no engine + * input and no generic builder: the matcher reports byte boundaries and + * the result string is written in place. The replacement template is + * decoded once per call into literal runs and capture references; the + * position-dependent directives ($`, $', $) keep the general loop. */ + +enum { SCR_RE_TEMPLATE_MAX = 16 }; + +typedef struct { + int32_t capture; /* -1: the literal run rep[off, off + len) */ + int32_t off, len; +} ScrReTemplateOp; + +/* GetSubstitution's scan, restricted to $$, $& and $n/$nn; mirrors + * scr_put_substitution's digit and out-of-range rules exactly. Returns + * the op count, or -1 when the template needs the general loop. */ +static int scr_re_template_ops(const ScrStr *rep, int capture_count, bool named, + ScrReTemplateOp *ops) { + int n = 0; + size_t i = 0, run = 0; +#define SCR_RE_OP(cap, o, l) \ + do { \ + if (n == SCR_RE_TEMPLATE_MAX) return -1; \ + ops[n++] = (ScrReTemplateOp){ (cap), (int32_t)(o), (int32_t)(l) }; \ + } while (0) + while (i < rep->len) { + if (rep->data[i] != '$' || i + 1 >= rep->len) { + i++; + continue; + } + char c1 = rep->data[i + 1]; + if (c1 == '$') { + SCR_RE_OP(-1, run, i + 1 - run); /* keep one '$' */ + i += 2; + run = i; + } else if (c1 == '&') { + if (i > run) SCR_RE_OP(-1, run, i - run); + SCR_RE_OP(0, 0, 0); + i += 2; + run = i; + } else if (c1 >= '0' && c1 <= '9') { + int k = c1 - '0'; + size_t adv = 2; + if (i + 2 < rep->len && rep->data[i + 2] >= '0' && rep->data[i + 2] <= '9') { + int k1 = k * 10 + (rep->data[i + 2] - '0'); + if (k1 >= 1 && k1 < capture_count) { + k = k1; + adv = 3; + } + } + if (k >= 1 && k < capture_count) { + if (i > run) SCR_RE_OP(-1, run, i - run); + SCR_RE_OP(k, 0, 0); + i += adv; + run = i; + } else { + i++; /* a literal '$' */ + } + } else if (c1 == '`' || c1 == '\'' || (c1 == '<' && named)) { + return -1; + } else { + i++; + } + } + if (i > run) SCR_RE_OP(-1, run, i - run); +#undef SCR_RE_OP + return n; +} + +/* The result string of the direct loop, written in place: `cap` caches + * the block's capacity so a piece costs one compare. */ +typedef struct { + ScrStr *s; + char *data; + size_t len; + size_t cap; +} ScrReOut; + +static ScrReOut scr_re_out_grow(ScrReOut o, size_t n) { + if (n > SCR_STR_MAX_CAP - o.len) scr_regex_oom(); + size_t cap = o.cap * 2; + if (cap < o.len + n) cap = o.len + n; + if (cap > SCR_STR_MAX_CAP) cap = SCR_STR_MAX_CAP; + o.s->len = o.len; + o.s = scr_str_regrow(o.s, cap); + o.data = o.s->data; + o.cap = o.s->cap; + return o; +} + +static inline void scr_re_out_put(ScrReOut *o, const char *p, size_t n) { + if (n > o->cap - o->len) *o = scr_re_out_grow(*o, n); + scr_copy_bytes(o->data + o->len, p, n); + o->len += n; +} + +/* The replace loop of scr_replace_impl for an ASCII subject and a + * build-time matcher. NULL when the template needs the general loop. */ +static ScrStr *scr_replace_matcher(ScrStr *s, ScrRegex *re, ScrStr *rep, + ScrRegexInput *input, int capture_count, + bool named, bool global, bool sticky) { + ScrReTemplateOp ops[SCR_RE_TEMPLATE_MAX]; + int nops = scr_re_template_ops(rep, capture_count, named, ops); + if (nops < 0) return NULL; + /* A template without captures is one literal run (or none). */ + bool plain = nops == 0 || (nops == 1 && ops[0].capture < 0); + const char *plain_text = nops ? rep->data + ops[0].off : NULL; + size_t plain_len = nops ? (size_t)ops[0].len : 0; + ScrRegexMatcherFn matcher = input->matcher; + const uint8_t *data = input->data; + const char *text = (const char *)data; + int len = input->len; + uint8_t **capture = input->capture; + if (global) re->last_index = 0; + int next = 0, pos = sticky ? scr_regex_start(re, len) : 0; + if (pos < 0) pos = len + 1; + if (sticky && !global) re->last_index = 0; + ScrReOut out = { NULL, NULL, 0, 0 }; + while (pos <= len) { + if (matcher(data, len, pos, capture) != 1) break; + int start = (int)(capture[0] - data); + int end = (int)(capture[1] - data); + if (sticky && !global) re->last_index = end; + if (!out.s) { + out.s = scr_str_alloc_raw(0, (size_t)len + 16); + out.data = out.s->data; + out.cap = out.s->cap; + } + scr_re_out_put(&out, text + next, (size_t)(start - next)); + if (plain) { + scr_re_out_put(&out, plain_text, plain_len); + } else { + for (int i = 0; i < nops; i++) { + const ScrReTemplateOp *op = &ops[i]; + if (op->capture < 0) { + scr_re_out_put(&out, rep->data + op->off, (size_t)op->len); + } else { + const uint8_t *cs = capture[2 * op->capture], *ce = capture[2 * op->capture + 1]; + if (cs && ce) scr_re_out_put(&out, (const char *)cs, (size_t)(ce - cs)); + } + } + } + next = end; + if (!global) break; + /* ASCII holds no surrogate pair: an empty match advances one unit. */ + pos = end == start ? end + 1 : end; + } + if (!out.s) return scr_str_retain(s); + scr_re_out_put(&out, text + next, (size_t)(len - next)); + out.s->len = out.len; + out.data[out.len] = '\0'; + if (scr_str_known_ascii(rep) || scr_bytes_all_ascii(rep->data, rep->len)) + scr_str_note_ascii(out.s); + return out.s; +} + /* The shared match loop (the shape of quickjs's JS_RegExpDelete): search * from `pos`, copy the gap, emit the substitution, continue from the match * end when global — advancing zero-length matches per AdvanceStringIndex. */ @@ -924,6 +1166,14 @@ static ScrStr *scr_replace_impl(ScrStr *s, ScrRegex *re, ScrStr *rep) { const char *groupnames = lre_get_groupnames(bc); ScrRegexInput input; scr_regex_input_init(&input, re, bc, s); + if (input.matcher) { + ScrStr *direct = scr_replace_matcher(s, re, rep, &input, capture_count, + groupnames != NULL, global, sticky); + if (direct) { + scr_regex_input_dispose(&input); + return direct; + } + } int len = input.len; uint8_t **capture = input.capture; ScrJsonBuf b; @@ -948,8 +1198,12 @@ static ScrStr *scr_replace_impl(ScrStr *s, ScrRegex *re, ScrStr *rep) { pos = end == start ? scr_advance(&input, end, unicode) : end; } if (matched) scr_regex_put_span(&b, &input, next, len); + bool ascii = !input.wide && (scr_str_known_ascii(rep) || scr_bytes_all_ascii(rep->data, rep->len)); scr_regex_input_dispose(&input); - return matched ? scr_jb_finish(&b) : scr_str_retain(s); + if (!matched) return scr_str_retain(s); + ScrStr *out = scr_jb_finish(&b); + if (ascii) scr_str_note_ascii(out); + return out; } ScrStr *scr_regex_replace(ScrStr *s, ScrRegex *re, ScrStr *rep) { diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 3ac5c3214..4e318f03d 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -1341,6 +1341,11 @@ struct ScrArr; * every other '$' spelling stays literal). An empty search value matches * at 0, or at every UTF-16 position under replaceAll. Borrows; +1. */ ScrStr *scr_str_replace_literal(ScrStr *s, ScrStr *search, ScrStr *rep, bool all); +/* Every occurrence of the ASCII byte chars[k] replaced by the k-th + * NUL-separated text of `texts`, in one pass (at most 16 distinct + * characters; the compiler fuses chains of global one-character + * replacements into it). Borrows all three; +1. */ +ScrStr *scr_str_replace_chars(ScrStr *s, ScrStr *chars, ScrStr *texts); /* The same over a nonempty well-formed UTF-8 needle (literal regex * patterns share it). Borrows; +1. */ ScrStr *scr_str_replace_bytes(ScrStr *s, const char *pat, size_t plen, ScrStr *rep, bool all); @@ -1404,6 +1409,8 @@ ScrStr *scr_regexp_escape(ScrStr *s); * longest digit prefix, exact value correctly rounded (overflow → * ±Infinity). The frontend completes an omitted radix to 0. Borrows s. */ double scr_parse_int(ScrStr *s, double radix); +/* parseInt over a borrowed UTF-8 byte span. */ +double scr_parse_int_bytes(const char *p, size_t n, double radix); double scr_parse_float(ScrStr *s); /* ToNumber(string) — ECMA-262 7.1.4.1 StringToNumber exactly: trim @@ -1415,6 +1422,8 @@ double scr_parse_float(ScrStr *s); * a sign on those is NaN, per grammar). Any trailing garbage → NaN. * Borrows s; never throws. */ double scr_string_to_number(ScrStr *s); +/* StringToNumber over a borrowed UTF-8 byte span. */ +double scr_bytes_to_number(const char *p, size_t len); /* encodeURIComponent — ECMA-262 Encode with the component unreserved set * (ALPHA/DIGIT/- _ . ! ~ * ' ( )): every other byte of the UTF-8 string @@ -1977,6 +1986,18 @@ ScrArr *scr_regex_exec(ScrStr *s, ScrRegex *re); * rest, and the index/input properties, are left out. Same lastIndex * transitions and NULL result as scr_regex_exec. Borrows both; +1 or NULL. */ ScrArr *scr_regex_exec_groups(ScrStr *s, ScrRegex *re, double mask); +/* scr_regex_exec_groups whose captures in `lazy` (a subset of `mask`) are + * read only through scr_regex_group_eq/_number/_parse_int: they may stay + * spans of the subject instead of strings. */ +ScrArr *scr_regex_exec_spans(ScrStr *s, ScrRegex *re, double mask, double lazy); +/* s.match(re) read only through constant indices and null tests: a global + * regex answers scr_regex_match; otherwise scr_regex_exec_spans. */ +ScrArr *scr_regex_match_groups(ScrStr *s, ScrRegex *re, double mask, double lazy); +/* Reads of a lazy capture k of such a result (borrowed): `capture === lit`, + * Number(capture) and parseInt(capture, radix). */ +bool scr_regex_group_eq(ScrArr *m, double k, ScrStr *lit); +double scr_regex_group_number(ScrArr *m, double k); +double scr_regex_group_parse_int(ScrArr *m, double k, double radix); /* s.search(re): the first match's UTF-16 index, or -1 — a fresh exec from * position 0 (Symbol.search never touches lastIndex, so no flag fence: * /g is irrelevant, /y anchors at 0). Borrows both; never throws. */ diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 1aafcfbcd..41beed0ae 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -1852,35 +1852,127 @@ ScrStr *scr_str_replace_bytes(ScrStr *s, const char *pat, size_t plen, ScrStr *r scr_jb_write(&b, from, (size_t)(end - from)); return scr_jb_finish(&b); } - /* A plain replacement: count, size the result once, then copy. */ - size_t count = 1; - if (all) { - for (const char *p = scr_byte_find(hit + plen, (size_t)(end - hit - plen), pat, plen); p; - p = scr_byte_find(p + plen, (size_t)(end - p - plen), pat, plen)) - count++; - } - size_t kept = s->len - count * plen; - if (rep->len && count > (SCR_STR_MAX_CAP - kept) / rep->len) scr_oom(); - size_t total = kept + count * rep->len; - ScrStr *out = scr_str_alloc_raw(total, total); - char *w = out->data; + /* A plain replacement: one pass into a block with room for a few + * growing hits; a longer result grows geometrically, and a result well + * below its block is trimmed. */ + size_t grow = rep->len > plen ? rep->len - plen : 0; + size_t cap = s->len + (grow < 64 ? 4 * grow : grow) + 16; + if (cap > SCR_STR_MAX_CAP) cap = SCR_STR_MAX_CAP; + ScrStr *out = scr_str_alloc_raw(0, cap); + cap = out->cap; + size_t used = 0; const char *from = s->data; - for (size_t k = 0; k < count; k++) { - size_t gap = (size_t)(hit - from); - memcpy(w, from, gap); - w += gap; - memcpy(w, rep->data, rep->len); - w += rep->len; + for (;;) { + size_t gap = hit ? (size_t)(hit - from) : (size_t)(end - from); + size_t need = gap + (hit ? rep->len : 0); + if (need > cap - used) { + if (need > SCR_STR_MAX_CAP - used) scr_oom(); + size_t want = cap * 2 < used + need + (size_t)(end - from) ? used + need + (size_t)(end - from) + : cap * 2; + if (want > SCR_STR_MAX_CAP) want = SCR_STR_MAX_CAP; + out->len = used; + out = scr_str_regrow(out, want); + cap = want; + } + scr_copy_bytes(out->data + used, from, gap); + used += gap; + if (!hit) break; + scr_copy_bytes(out->data + used, rep->data, rep->len); + used += rep->len; from = hit + plen; - if (k + 1 < count) hit = scr_byte_find(from, (size_t)(end - from), pat, plen); + hit = all ? scr_byte_find(from, (size_t)(end - from), pat, plen) : NULL; } - memcpy(w, from, (size_t)(end - from)); - out->data[total] = '\0'; + out->len = used; + if (cap - used > used / 4 + 64) out = scr_str_regrow(out, used); + out->data[used] = '\0'; scr_str_hash_forget(out); if (s->ascii && rep->ascii) scr_str_note_ascii(out); return out; } +/* The first byte at or after `i` that is one of the replaced characters + * (`which` maps them to k + 1), or len. Up to eight characters are tested + * eight bytes at a time. */ +static size_t scr_replace_chars_next(const char *p, size_t i, size_t len, const uint8_t *which, + const char *chars, size_t n) { + if (n <= 8) { + const uint64_t ones = UINT64_C(0x0101010101010101); + const uint64_t highs = UINT64_C(0x8080808080808080); + for (; len - i >= 8; i += 8) { + uint64_t w; + memcpy(&w, p + i, 8); + uint64_t hit = 0; + for (size_t k = 0; k < n; k++) { + uint64_t x = w ^ (ones * (unsigned char)chars[k]); + hit |= (x - ones) & ~x & highs; + } + if (hit) break; + } + } + for (; i < len; i++) { + unsigned char c = (unsigned char)p[i]; + if (c < 0x80 && which[c]) return i; + } + return len; +} + +ScrStr *scr_str_replace_chars(ScrStr *s, ScrStr *chars, ScrStr *texts) { + enum { MAX = 16 }; + size_t n = chars->len < MAX ? chars->len : MAX; + uint8_t which[128] = { 0 }; + const char *text[MAX]; + size_t text_len[MAX]; + size_t grow = 0; + bool ascii = s->ascii; + const char *t = texts->data, *t_end = texts->data + texts->len; + for (size_t k = 0; k < n; k++) { + const char *stop = memchr(t, '\0', (size_t)(t_end - t)); + if (!stop) stop = t_end; + text[k] = t; + text_len[k] = (size_t)(stop - t); + if (text_len[k] > grow) grow = text_len[k]; + if (ascii && !scr_bytes_all_ascii(t, text_len[k])) ascii = false; + which[(unsigned char)chars->data[k] & 0x7F] = (uint8_t)(k + 1); + t = stop < t_end ? stop + 1 : t_end; + } + const char *p = s->data; + size_t len = s->len; + size_t i = scr_replace_chars_next(p, 0, len, which, chars->data, n); + if (i == len) return scr_str_retain(s); + grow = grow > 1 ? grow - 1 : 0; + size_t cap = len + (grow < 64 ? 4 * grow : grow) + 16; + if (cap > SCR_STR_MAX_CAP) cap = SCR_STR_MAX_CAP; + ScrStr *out = scr_str_alloc_raw(0, cap); + cap = out->cap; + size_t used = 0, from = 0; + for (;;) { + size_t gap = i - from; + size_t k = i < len ? (size_t)which[(unsigned char)p[i]] - 1 : 0; + size_t need = gap + (i < len ? text_len[k] : 0); + if (need > cap - used) { + if (need > SCR_STR_MAX_CAP - used) scr_oom(); + size_t want = cap * 2 < used + need + (len - from) ? used + need + (len - from) : cap * 2; + if (want > SCR_STR_MAX_CAP) want = SCR_STR_MAX_CAP; + out->len = used; + out = scr_str_regrow(out, want); + cap = want; + } + scr_copy_bytes(out->data + used, p + from, gap); + used += gap; + if (i == len) break; + scr_copy_bytes(out->data + used, text[k], text_len[k]); + used += text_len[k]; + from = i + 1; + i = scr_replace_chars_next(p, from, len, which, chars->data, n); + } + out->len = used; + if (cap - used > used / 4 + 64) out = scr_str_regrow(out, used); + out->data[used] = '\0'; + scr_str_hash_forget(out); + if (ascii) scr_str_note_ascii(out); + return out; +} + ScrStr *scr_str_replace_literal(ScrStr *s, ScrStr *search, ScrStr *rep, bool all) { if (search->len) return scr_str_replace_bytes(s, search->data, search->len, rep, all); ScrJsonBuf b; @@ -2100,9 +2192,12 @@ static double scr_digits_to_double(const char *p, size_t n, int radix) { } double scr_parse_int(ScrStr *s, double radix) { + return scr_parse_int_bytes(s->data, s->len, radix); +} + +double scr_parse_int_bytes(const char *p, size_t n, double radix) { double r32 = scr_to_int32_d(radix); - const char *p = s->data; - size_t n = s->len, i = 0; + size_t i = 0; while (i < n) { unsigned char c = (unsigned char)p[i]; if (c > 0x20 && c < 0x80) break; /* ASCII non-whitespace */ @@ -2216,8 +2311,11 @@ double scr_parse_float(ScrStr *s) { * Node-exact for giant hex, Infinity overflow included: power-of-two * radixes take the exact path, never V8's approximate chunk loop). */ double scr_string_to_number(ScrStr *s) { - const char *p = s->data; - size_t b = 0, e = s->len; + return scr_bytes_to_number(s->data, s->len); +} + +double scr_bytes_to_number(const char *p, size_t len) { + size_t b = 0, e = len; while (b < e) { unsigned char c = (unsigned char)p[b]; if (c > 0x20 && c < 0x80) break; /* ASCII non-whitespace */ diff --git a/packages/runtime/test/test_regex.c b/packages/runtime/test/test_regex.c index a0b069619..ae98d627b 100644 --- a/packages/runtime/test/test_regex.c +++ b/packages/runtime/test/test_regex.c @@ -11,6 +11,7 @@ */ #include "../src/scr_runtime.h" +#include #include #include #include @@ -400,6 +401,97 @@ static void test_build_time_matcher(void) { scr_str_release(wide); } +RE(re_matched_g, "(b)", "g"); +RE(re_pair, "(a)(\\d+)?", ""); +RE(re_digits_g, "\\d+", "g"); + +static void test_replace_matcher(void) { + /* The direct loop over a build-time matcher: templates decode once. */ + re_matched_g.matcher = fake_b_matcher; + ScrStr *s = S("abcb"); + ScrStr *t1 = S("[$1|$&|$$|$2|$01]"); + ScrStr *t2 = S("<$`>"); + ScrStr *t3 = S(""); + ScrStr *out = scr_regex_replace(s, &re_matched_g, t1); + check(str_is(out, "a[b|b|$|$2|b]c[b|b|$|$2|b]") && scr_str_known_ascii(out), + "matcher replace: $n, $&, $$ and out-of-range references; ASCII result"); + scr_str_release(out); + expect_str(scr_regex_replace(s, &re_matched_g, t2), "ac", "matcher replace: $` takes the general loop"); + expect_str(scr_regex_replace(s, &re_matched_g, t3), "ac", "matcher replace: empty template"); + check(re_matched_g.last_index == 0, "matcher replace: a global regex ends at lastIndex 0"); + scr_str_release(s); + scr_str_release(t1); + scr_str_release(t2); + scr_str_release(t3); +} + +static void test_exec_spans(void) { + /* Lazy captures: compared and converted without strings on ASCII + * subjects, materialized on wide ones. */ + ScrStr *ascii = S("xa12"); + ScrArr *m = scr_regex_exec_spans(ascii, &re_pair, 6.0, 6.0); + check(m && m->len == 4 && m->present[1] == SCR_ARR_UNDEFINED, + "exec_spans: ASCII subjects keep lazy captures as spans"); + ScrStr *a = S("a"); + ScrStr *twelve = S("12"); + check(m && scr_regex_group_eq(m, 1, a) && !scr_regex_group_eq(m, 2, a) && scr_regex_group_eq(m, 2, twelve), + "group_eq compares the span"); + check(m && scr_regex_group_number(m, 2) == 12 && scr_regex_group_parse_int(m, 2, 16) == 18, + "group_number and group_parse_int convert the span"); + check(m && isnan(scr_regex_group_number(m, 5)) && !scr_regex_group_eq(m, 5, a), + "reads past the result are undefined"); + scr_arr_release(m); + ScrStr *short_subject = S("a"); + m = scr_regex_exec_spans(short_subject, &re_pair, 6.0, 6.0); + check(m && isnan(scr_regex_group_number(m, 2)) && scr_regex_group_parse_int(m, 2, 36) == 86464843759093.0, + "a non-participating capture reads as undefined"); + scr_arr_release(m); + ScrStr *wide = S("\xC3\xA9" "a7"); + m = scr_regex_exec_spans(wide, &re_pair, 6.0, 6.0); + check(m && m->len == 3 && m->present[2] == SCR_ARR_VALUE && scr_regex_group_number(m, 2) == 7, + "exec_spans: wide subjects materialize lazy captures"); + scr_arr_release(m); + m = scr_regex_exec_spans(ascii, &re_pair, 38.0, 2.0); + check(m && m->len == 3 && scr_regex_group_eq(m, 1, a), "exec_spans: a read past the captures keeps strings"); + scr_arr_release(m); + ScrStr *list = S("1 22 333"); + m = scr_regex_match_groups(list, &re_digits_g, 1.0, 1.0); + check(m && m->len == 3 && scr_regex_group_number(m, 2) == 333, "match_groups: a global regex answers every match"); + scr_arr_release(m); + m = scr_regex_match_groups(ascii, &re_pair, 6.0, 4.0); + check(m && scr_regex_group_number(m, 2) == 12, "match_groups: a non-global regex reads like exec_spans"); + scr_arr_release(m); + scr_str_release(ascii); + scr_str_release(short_subject); + scr_str_release(wide); + scr_str_release(list); + scr_str_release(a); + scr_str_release(twelve); +} + +static void test_replace_chars(void) { + ScrStr *chars = S("&<>"); + ScrStr *texts = scr_str_new("&\0<\0>", 15); + ScrStr *s = S("a&c"); + expect_str(scr_str_replace_chars(s, chars, texts), "a<b>&c", "replace_chars: every character"); + ScrStr *none = S("plain text without specials"); + ScrStr *same = scr_str_replace_chars(none, chars, texts); + check(same == none, "replace_chars: no hit retains the subject"); + scr_str_release(same); + ScrStr *longer = S("0123456789abcdefghij<0123456789abcdefghij>0123456789&"); + expect_str(scr_str_replace_chars(longer, chars, texts), + "0123456789abcdefghij<0123456789abcdefghij>0123456789&", + "replace_chars: word-at-a-time scan"); + ScrStr *wide = S("\xC3\xA9<\xC3\xA9"); + expect_str(scr_str_replace_chars(wide, chars, texts), "\xC3\xA9<\xC3\xA9", "replace_chars: UTF-8 subject"); + scr_str_release(chars); + scr_str_release(texts); + scr_str_release(s); + scr_str_release(none); + scr_str_release(longer); + scr_str_release(wide); +} + static void test_string_replace(void) { /* String search values: one pass, GetSubstitution without captures. */ ScrStr *s = S("a.b.c"); @@ -443,6 +535,9 @@ int main(int argc, char **argv) { test_exec_groups(); test_string_replace(); test_build_time_matcher(); + test_replace_matcher(); + test_exec_spans(); + test_replace_chars(); #ifdef SCR_RC_AUDIT check(scr_str_live_count() == 0, "no live heap strings at exit (RC audit)"); diff --git a/tests/corpus/5115-regex-replace-compiled.ts b/tests/corpus/5115-regex-replace-compiled.ts new file mode 100644 index 000000000..04cdea690 --- /dev/null +++ b/tests/corpus/5115-regex-replace-compiled.ts @@ -0,0 +1,42 @@ +// Global and single replacements over compiled regex matchers: literal +// templates, $$ $& $n $nn, position directives, named groups, empty +// matches, unicode and non-ASCII subjects, and lastIndex transitions. +const show = (label: string, v: unknown): void => console.log(label, JSON.stringify(v)); +const s = "a1b22c333d"; +show("g", s.replace(/\d+/g, "#")); +show("first", s.replace(/\d+/, "#")); +show("amp", s.replace(/\d+/g, "<$&>")); +show("dollar", s.replace(/\d+/g, "$$")); +show("cap", s.replace(/([a-z])(\d+)/g, "$2$1")); +show("cap12", "abcdefghijklmn".replace(/(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)/, "[$12|$13|$1$2|$0|$01|$10]")); +show("cap1x", "ab".replace(/(a)(b)/, "$3$11$22$02")); +show("unset", "ac".replace(/a(b)?c/, "[$1]")); +show("before", s.replace(/\d+/g, "[$`]")); +show("after", s.replace(/b/g, "[$']")); +show("named", "2026-10".replace(/(?\d+)-(?\d+)/, "$/$|$|$")); +show("empty", "abc".replace(/x*/g, "-")); +show("empty2", "abc".replace(/(?:)/g, "_")); +show("emptyu", "abc".replace(/x*/gu, "-")); +show("wide", "héllo wörld 12".replace(/\w+/g, "W")); +show("wide2", "😀a😀".replace(/(?:)/gu, "|")); +show("ci", "AbAB".replace(/ab/gi, "x")); +show("ml", "a\nb\nc".replace(/^/gm, "> ")); +show("end", "abc".replace(/$/g, "!")); +const sticky = /\d/y; +sticky.lastIndex = 1; +show("sticky", ["a1b".replace(sticky, "#"), sticky.lastIndex]); +show("sticky2", ["a1b".replace(sticky, "#"), sticky.lastIndex]); +const gy = /\d/gy; +show("gy", ["12a3".replace(gy, "#"), gy.lastIndex]); +const g = /o/g; +g.lastIndex = 3; +show("glast", ["foo".replace(g, "0"), g.lastIndex]); +show("all", "a.b.c".replaceAll(/\./g, "$&$&")); +show("nomatch", "abc".replace(/\d/g, "#")); +show("long", "x".repeat(100).replace(/x/g, "yy").length); +show("tpl17", "abc".replace(/b/g, "1$&2$&3$&4$&5$&6$&7$&8$&9$&")); +show("lookbehind", "a1b2".replace(/(?<=a)\d/g, "#")); +show("backref", "aa bb cd".replace(/(\w)\1/g, "<$1>")); +show("dollarend", "ab".replace(/b/, "$")); +show("dollarx", "ab".replace(/b/, "$x$")); diff --git a/tests/corpus/5116-regex-capture-reads.ts b/tests/corpus/5116-regex-capture-reads.ts new file mode 100644 index 000000000..cda96f6f6 --- /dev/null +++ b/tests/corpus/5116-regex-capture-reads.ts @@ -0,0 +1,78 @@ +// exec and match results read through constant indices: captures only +// compared with literals or converted by Number()/parseInt() next to +// captures read as strings, non-participating groups, wide subjects, +// global and sticky regexes, and reads past the capture count. +const line = /^(\w+) (\d+)? ?(\S*) (x)?(.*)$/; +const inputs = ["GET 200 /a", "POST /b x", "PUT 0x1F /c", "DEL ", "héllo 42 /ü", "GET 00012 /z rest"]; +for (const input of inputs) { + const m = line.exec(input); + if (m === null) { + console.log("no match", input); + continue; + } + console.log( + m[1] === "GET", + m[1] !== "POST", + m[4] === "x", + m[4] !== undefined, + Number(m[2]), + parseInt(m[2]), + parseInt(m[2], 16), + parseInt(m[4], 36), + Number(m[4]), + m[3] === "", + m[3], + ); +} +// A capture read as a string and compared stays a string. +const kv = /(\w+)=(\w*)/; +for (const s of ["a=1", "b=", "c=x"]) { + const m = kv.exec(s); + if (!m) continue; + console.log(m[2] === "1", m[2]!.length, Number(m[2]), m[1]); +} +// match() of a non-global regex reads like exec. +for (const s of ["v1.22.333", "v4", "x"]) { + const m = s.match(/^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/); + if (m === null) { + console.log("none"); + continue; + } + console.log(Number(m[1]), Number(m[2]), m[3] === "333", m[3] === undefined, parseInt(m[2])); +} +// match() of a global regex answers every match. +const all = "a1 b22 c333".match(/\d+/g); +if (all !== null) console.log(all[0], all[2], all[5], Number(all[1]), all[2] === "333"); +// Sticky match keeps lastIndex transitions. +const st = /\d+/y; +st.lastIndex = 1; +const sm = "a12b".match(st); +console.log(sm !== null ? sm[0] : null, st.lastIndex, sm !== null && Number(sm[0]) === 12); +const sm2 = "a12b".match(st); +console.log(sm2 === null, st.lastIndex); +// A read beyond the capture count is undefined. +const two = /(a)(b)/.exec("ab"); +if (two) console.log(two[2] === "b", two[3] === undefined, Number(two[5]), two[1] === "a"); +// Whitespace and signs follow StringToNumber. +const ws = /\[(.*?)\]/; +for (const s of ["[ 42 ]", "[-7]", "[]", "[1e3]", "[Infinity]", "[\t5\n]", "[+0x10]"]) { + const m = ws.exec(s); + if (m) console.log(Number(m[1]), parseInt(m[1])); +} +// A closure read keeps every capture a string. +const cm = /(\d+)-(\d+)/.exec("10-20"); +if (cm) { + const later = (): boolean => cm[1] === "10"; + console.log(later(), Number(cm[2])); +} +// Global exec loops keep their lastIndex state. +const re = /(\w)(\d)/g; +const text = "a1 b2 c3"; +for (let i = 0; i < 4; i++) { + const m = re.exec(text); + if (m === null) { + console.log("end", re.lastIndex); + break; + } + console.log(m[1] === "b", Number(m[2]), re.lastIndex); +} diff --git a/tests/corpus/5117-regex-log-report-spans.ts b/tests/corpus/5117-regex-log-report-spans.ts new file mode 100644 index 000000000..b7d073ac6 --- /dev/null +++ b/tests/corpus/5117-regex-log-report-spans.ts @@ -0,0 +1,73 @@ +// A log-parsing report: logfmt and access-log lines parsed with regex +// captures, levels compared with literals, numeric fields converted, and +// secrets redacted with global replacements. +let seed = 7; +function random(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +const levels = ["debug", "info", "warn", "error"]; +const services = ["api", "auth", "billing", "search"]; +function makeLines(count: number): string[] { + const lines: string[] = []; + for (let i = 0; i < count; i++) { + const level = levels[Math.floor(random() * levels.length)]!; + const service = services[Math.floor(random() * services.length)]!; + const ms = Math.floor(random() * 2000); + const user = 1000 + Math.floor(random() * 9000); + const token = Math.floor(random() * 1e9).toString(16); + if (i % 3 === 0) { + lines.push( + `192.168.${i % 256}.${(i * 7) % 256} - u${user} [14/Oct/2026:08:${String(i % 60).padStart(2, "0")}:00 +0000] "GET /v1/${service}/items/${i % 500}?token=${token} HTTP/2" ${random() < 0.95 ? 200 : 503} ${ms}`, + ); + } else { + lines.push( + `ts=2026-10-14T08:${String(i % 60).padStart(2, "0")}:00Z level=${level} service=${service} user=${user} duration_ms=${ms} msg="request done" token=${token}`, + ); + } + } + return lines; +} + +const logfmt = /^ts=(\S+) level=(\w+) service=(\w+) user=(\d+) duration_ms=(\d+) msg="([^"]*)"/; +const access = /^(\d+\.\d+\.\d+\.\d+) - u(\d+) \[([^\]]+)\] "(\w+) (\S+) HTTP\/[\d.]+" (\d{3}) (\d+)$/; +const secret = /token=([0-9a-f]+)/g; +const numbers = /\d+/g; + +const scale = Number(process.argv[2] ?? "0.02"); +const lines = makeLines(Math.floor(40000 * scale)); +const perService = new Map(); +let errors = 0; +let warnings = 0; +let slow = 0; +let totalMs = 0; +let unavailable = 0; +let redacted = 0; +let tagged = 0; +let shapes = 0; +let vip = 0; +for (let round = 0; round < 3; round++) { + for (const line of lines) { + const m = logfmt.exec(line); + if (m !== null) { + if (m[2] === "error") errors++; + else if (m[2] === "warn") warnings++; + const ms = Number(m[5]); + totalMs += ms; + if (ms > 1500) slow++; + perService.set(m[3]!, (perService.get(m[3]!) ?? 0) + 1); + if (parseInt(m[4]) % 97 === 0) vip++; + } else { + const a = line.match(access); + if (a === null) continue; + if (a[6] === "503") unavailable++; + totalMs += Number(a[7]); + } + redacted += line.replace(secret, "token=***").length; + tagged += line.replace(secret, "token=<$1:$&>").length; + shapes += line.replace(numbers, "0").length; + } +} +for (const key of [...perService.keys()].sort()) console.log(key, perService.get(key)); +console.log("errors", errors, "warnings", warnings, "slow", slow, "unavailable", unavailable, "vip", vip); +console.log("ms", totalMs, "redacted", redacted, "tagged", tagged, "shapes", shapes); diff --git a/tests/corpus/5118-regex-replace-char-chains.ts b/tests/corpus/5118-regex-replace-char-chains.ts new file mode 100644 index 000000000..b2c9881eb --- /dev/null +++ b/tests/corpus/5118-regex-replace-char-chains.ts @@ -0,0 +1,18 @@ +// Chains of global one-character replacements (the HTML-escape idiom) +// against step-by-step semantics: later steps that match earlier output, +// escaped syntax characters, $ templates and stored regex objects. +function escapeHtml(text: string): string { + return text.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} +for (const s of ["plain", "a & b < c > d \"e\"", "&&&", "", "<<>>", "héllo & wörld <ü>", "x".repeat(40) + "&" + "y".repeat(40), "&"]) { + console.log(JSON.stringify(escapeHtml(s)), escapeHtml(s).length); +} +// A later step that matches an earlier replacement keeps its order. +console.log("a&b".replace(/&/g, "&").replace(/a/g, "A")); +console.log("a.b-c".replace(/\./g, "-").replace(/-/g, "+")); +console.log("a.b-c".replace(/-/g, "+").replace(/\./g, "/")); +console.log("x$y".replace(/\$/g, "S").replace(/y/g, "z")); +console.log("tab\there".replace(/ /g, "_").replaceAll(/e/g, "E")); +console.log("$1 a".replace(/a/g, "$&$&").replace(/1/g, "one")); +const r = /a/g; +console.log("aaa".replace(r, "b").replace(/b/g, "c"), r.lastIndex);