diff --git a/docs/content/docs/how-it-works.mdx b/docs/content/docs/how-it-works.mdx index 3038608c4..3d1bb011b 100644 --- a/docs/content/docs/how-it-works.mdx +++ b/docs/content/docs/how-it-works.mdx @@ -70,7 +70,7 @@ Without `-o`, artifacts are written to `.scriptc/` next to the input. Different ### External linking -Program objects define `main` and leave selected `scr_*` runtime functions undefined. A strong reference to `scr_runtime_abi_v16` checks runtime compatibility at link time. +Program objects define `main` and leave selected `scr_*` runtime functions undefined. A strong reference to `scr_runtime_abi_v17` checks runtime compatibility at link time. `--print=native-link-info` emits the program object and a JSON recipe containing the runtime pack and link order for external builds. This object ABI is experimental and requires an exact runtime version. See [Native Program Objects](/docs/native-objects) for examples. diff --git a/examples/native-object/README.md b/examples/native-object/README.md index 444a2ca7c..1e7124f85 100644 --- a/examples/native-object/README.md +++ b/examples/native-object/README.md @@ -15,4 +15,4 @@ $ ./app-ld `cc` uses the C compiler as a linker driver. `ld` links the same precompiled runtime objects directly with the selected Apple SDK. The object defines `main`; it is a complete program object, not a library to load into another process. Use `scriptc build --lib --profile ...` for a host-callable static library. -The external object ABI is experimental. Always consume the runtime pack at the exact `runtime_pack.version` reported by the same scriptc installation. The object requires `scr_runtime_abi_v16`, so a mismatched runtime fails during the link instead of starting with an incompatible ABI. +The external object ABI is experimental. Always consume the runtime pack at the exact `runtime_pack.version` reported by the same scriptc installation. The object requires `scr_runtime_abi_v17`, so a mismatched runtime fails during the link instead of starting with an incompatible ABI. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 681ee309d..3d79e2806 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v16"); + expect(error?.stderr).toContain("scr_runtime_abi_v17"); }); }); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 99be735da..1b3745673 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,5 @@ import { emitLlvmLayouts } from "./layouts.js"; +import { compileRegexMatcher, type RegexMatcherCode } from "./regex-matchers.js"; import { FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js"; import { llvmBytes as llBytes } from "../literals.js"; import { InternalCompilerError } from "../../errors.js"; @@ -552,7 +553,10 @@ export class LlEmitter { * (pattern, flags) pair; the bytecode slot starts null and the runtime * compiles it lazily on first use. The source/flags strings intern at * REGISTRATION (bodies emit before the literal table flushes). */ - readonly regexInstances = new Map(); + readonly regexInstances = new Map< + string, + { sym: string; src: string; fl: string; pattern: string; flags: string } + >(); /** Interned tagged-template strings objects: per-site key → { symbol, * interned cooked-literal refs }. One immortal ScrArr of string slots * per template SITE (the spec's per-occurrence identity — the C @@ -1851,6 +1855,20 @@ export class LlEmitter { ]; if (this.constantNumericTables.size > 0) this.declare(`declare double @scr_arr_get_number(ptr, double)`); + // Native matchers of the regex literals inside the compiled subset. + const regexMatchers = new Map(); + for (const [key, re] of this.regexInstances) { + const code = compileRegexMatcher( + `${re.sym}_match`, + re.pattern, + re.flags, + this.sizeType, + !this.wasi, + ); + if (code === null) continue; + regexMatchers.set(key, code); + for (const decl of code.decls) this.declare(decl); + } const out: string[] = [ `; Generated by scriptc (LLVM backend) from ${this.mod.sourceFile}. Do not edit.`, @@ -1867,7 +1885,7 @@ export class LlEmitter { `%ScrUnion = type { ${this.sizeType}, i32, ptr, ptr, ptr, i64 }`, `%ScrClosure = type { ${this.sizeType}, ptr, ${this.sizeType}, ptr, i32, i32, ptr, ptr }`, `%ScrFfiTable = type { ptr, ${this.sizeType}, ${this.sizeType}, ptr, i8, ptr, ptr, ${this.sizeType}, ${this.sizeType}, ${this.sizeType}, ptr, ptr }`, - `%ScrRegex = type { ${this.sizeType}, ptr, ptr, ptr, double, ptr, ptr }`, + `%ScrRegex = type { ${this.sizeType}, ptr, ptr, ptr, double, ptr, ptr, ptr }`, // ScrArr mirrors scr_runtime.h field-for-field: { rc, len, cap, elem, // cyc, spare, filled, ops, data, present, ext } (field numbers in // SCR_ARR_FIELD), where filled bounds the hole-free prefix, ops points at the element type's constant RC descriptor @@ -2048,14 +2066,16 @@ export class LlEmitter { } for (const [key, re] of this.regexInstances) { // One immortal ScrRegex per (pattern, flags) literal, pointing at - // the interned source/flags strings. The bc and native-matcher slots - // start null (lazy compile, cached by the runtime) — a mutable - // global, not constant. + // the interned source/flags strings. The bc and translated-program + // slots start null (lazy compile, cached by the runtime) — a mutable + // global, not constant. The last slot is the build-time matcher. + const matcher = regexMatchers.has(key) ? `@${re.sym}_match` : "null"; out.push( - `@${re.sym} = internal ${tl}global %ScrRegex { ${this.sizeType} -1, ptr ${re.src}, ptr ${re.fl}, ptr null, double 0.0, ptr null, ptr null } ; ${key.replace(/\n/g, "\\n")}`, + `@${re.sym} = internal ${tl}global %ScrRegex { ${this.sizeType} -1, ptr ${re.src}, ptr ${re.fl}, ptr null, double 0.0, ptr null, ptr null, ptr ${matcher} } ; ${key.replace(/\n/g, "\\n")}`, ); } if (this.regexInstances.size > 0) out.push(``); + for (const code of regexMatchers.values()) out.push(...code.lines, ``); for (const [, inst] of this.templateStringsInstances) { // One immortal ScrArr per tagged-template site: a [N x ptr] data // global of interned cooked-string literals, and the ScrArr header diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index f0466a0b4..61f0cd07c 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -498,6 +498,8 @@ export function regexLiteralTemplate( sym: `sc_re_${host.regexInstances.size}`, src: host.internLiteral(e.pattern), fl: host.internLiteral(e.flags), + pattern: e.pattern, + flags: e.flags, }; host.regexInstances.set(key, re); } diff --git a/packages/compiler/src/backend/llvm/regex-matchers.test.ts b/packages/compiler/src/backend/llvm/regex-matchers.test.ts new file mode 100644 index 000000000..a3627cb7f --- /dev/null +++ b/packages/compiler/src/backend/llvm/regex-matchers.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "vitest"; +import { + compileRegexMatcher, + regexLiteralBailReason, + regexLiteralCompiles, +} from "./regex-matchers.js"; + +function body(pattern: string, flags = "", vector = true): string { + const code = compileRegexMatcher("m", pattern, flags, "i64", vector); + if (code === null) throw new Error(`/${pattern}/${flags} did not compile`); + return code.lines.join("\n"); +} + +describe("regex literal matcher subset", () => { + test.each([ + ['^(\\d+\\.\\d+\\.\\d+\\.\\d+) - - \\[([^\\]]+)\\] "(\\w+) ([^ ?"]+)(?:\\?([^ "]*))?$', ""], + ["curl|bot|spider", "i"], + ["\\d+", "g"], + ["(?\\d{4})-(?\\d\\d)", ""], + ["[\\b\\-\\]\\\\.]+", ""], + ["\\x41\\u0042\\0\\t\\/\\-", ""], + ["a{2,}?b{0,3}c*?", "y"], + ["^\\s*$", "m"], + ["\\bfoo\\B", "su"], + ["[^]", ""], + ])("/%s/%s compiles", (pattern, flags) => { + expect(regexLiteralBailReason(pattern, flags)).toBe(""); + }); + + test.each([ + ["(a)\\1", ""], + ["a(?=b)", ""], + ["a(?!b)", ""], + ["(?<=a)b", ""], + ["\\k(?a)", ""], + ["\\p{L}", "u"], + ["é", ""], + ["\\u00e9", ""], + ["[é]", "i"], + ["\\cA", ""], + ["\\01", ""], + ["a{", ""], + ["]", ""], + ["\\a", ""], + ["(?i:a)", ""], + ["a{1,200000}", ""], + ["^*", ""], + ])("/%s/%s keeps the interpreter", (pattern, flags) => { + expect(regexLiteralCompiles(pattern, flags)).toBe(false); + }); + + test("large patterns keep the interpreter", () => { + expect(regexLiteralCompiles("(?:ab|cd)".repeat(20), "")).toBe(true); + expect(regexLiteralCompiles("(?:ab|cd)".repeat(200), "")).toBe(false); + }); +}); + +describe("regex literal matcher code", () => { + test("reports its capture count for a negative length", () => { + expect(body("(a)(?:b)(?c)")).toContain("ret i32 3"); + }); + + test("a single possible first byte searches with memchr", () => { + const code = compileRegexMatcher("m", "x\\d+", "g", "i64", true); + expect(code?.decls).toContain("declare ptr @memchr(ptr, i32, i64)"); + expect(code?.lines.join("\n")).toContain("call ptr @memchr(ptr %from, i32 120, i64 %rest.w)"); + }); + + test("a sticky pattern makes one attempt", () => { + const text = body("\\d+", "y"); + expect(text).not.toContain("@memchr"); + expect(text).toMatch(/attempt_fail:\n {2}br label %nomatch/); + }); + + test("a non-multiline ^ attempts only position 0", () => { + expect(body("^ab")).toContain("br i1 %a0, label %attempt, label %nomatch"); + }); + + test("first-character sets and long runs use 16-byte scans", () => { + const text = body("\\d+"); + expect(text).toContain("load <16 x i8>"); + expect(text).toContain("@llvm.cttz.i16"); + expect(body("\\d+", "", false)).not.toContain("<16 x i8>"); + // A bounded count below 32 stays scalar. + expect(body("a\\d{1,3}")).not.toContain("<16 x i8>"); + }); + + test("loops no continuation can share push no backtrack entries", () => { + // \d+ then a dot; [^\]]+ then ]; the run at the end of the pattern. + expect(body("\\d+\\.\\d+")).not.toContain("@scr_regex_stack_grow"); + expect(body("\\[[^\\]]+\\]")).not.toContain("@scr_regex_stack_grow"); + expect(body("a.*")).not.toContain("@scr_regex_stack_grow"); + // .* followed by b can give characters back. + expect(body("a.*b")).toContain("@scr_regex_stack_grow"); + // $ is zero-width: \d+ may still give back before it. + expect(body("\\d+$")).toContain("@scr_regex_stack_grow"); + }); + + test("class tests use compares for few ranges and a table otherwise", () => { + expect(body("[a-c]x")).not.toContain("constant [128 x i8]"); + expect(body("[aceg]x")).toContain("private unnamed_addr constant [128 x i8]"); + }); + + test("case folding closes ASCII letters only", () => { + // [^a]/i excludes both cases: compares 0-64, 66-96, 98-127 as ranges. + const text = body("x[^a]", "i"); + expect(text).toContain("icmp ule i32"); + expect(text).not.toContain("constant [128 x i8]"); + }); +}); diff --git a/packages/compiler/src/backend/llvm/regex-matchers.ts b/packages/compiler/src/backend/llvm/regex-matchers.ts new file mode 100644 index 000000000..ad5c47c86 --- /dev/null +++ b/packages/compiler/src/backend/llvm/regex-matchers.ts @@ -0,0 +1,1806 @@ +/** Regex literals compiled to native matchers at build time. + * + * A regex literal whose pattern stays inside a common subset (literals, + * classes and their escapes, `.`, anchors, word boundaries, greedy and lazy + * quantifiers, alternation, capturing and non-capturing groups) gets a + * specialized LLVM function next to its immortal template. The runtime calls + * it in place of the bytecode engines for one-byte (ASCII) subjects; every + * other subject, and every pattern outside the subset, keeps the interpreter. + * + * The generated code is a direct backtracking matcher over the pattern's + * ECMAScript semantics: ordered alternation, RepeatMatcher's capture reset + * at each iteration and its empty-iteration check, greedy-first and + * lazy-first quantifiers. Choice points and the undo records of register + * writes share one explicit stack; a failure pops the newest entry and + * resumes there. Single-character loops count instead of pushing one entry + * per character, and they push nothing when no continuation can start with + * a character of their own class. + * + * Function contract (scr_runtime.h `ScrRegexMatcher`): + * `i32 (ptr s, i32 len, i32 start, ptr capture)` searches the ASCII bytes + * `s[0..len)` from `start` (one anchored attempt for sticky patterns) and, + * on a match, stores `2 * ncap` pointers into `s` (NULL for unset captures) + * and returns 1; it returns 0 when nothing matches. Called with a negative + * `len`, it returns the capture count (whole match included) so the runtime + * can check it against the compiled bytecode. */ + +const ASCII = 128; +const INF = -1; +/** Larger patterns keep the interpreter (binary size). */ +const MAX_NODES = 400; +const MAX_COUNT = 100000; + +type CharSet = boolean[]; + +type AssertKind = "bol" | "bolm" | "eol" | "eolm" | "wordb" | "nwordb"; + +interface SetNode { + kind: "set"; + set: CharSet; +} +interface SeqNode { + kind: "seq"; + items: RxNode[]; +} +interface AltNode { + kind: "alt"; + alts: RxNode[]; +} +interface CapNode { + kind: "cap"; + index: number; + body: RxNode; +} +interface RepNode { + kind: "rep"; + body: RxNode; + min: number; + max: number; + greedy: boolean; + /** Capture groups inside the body: [capFirst, capEnd). */ + capFirst: number; + capEnd: number; +} +interface AssertNode { + kind: "assert"; + what: AssertKind; +} +type RxNode = SetNode | SeqNode | AltNode | CapNode | RepNode | AssertNode; + +class RegexBail { + constructor(readonly reason: string) {} +} + +function bail(reason: string): never { + throw new RegexBail(reason); +} + +function emptySet(): CharSet { + const s: CharSet = []; + for (let c = 0; c < ASCII; c++) s.push(false); + return s; +} + +function addRange(s: CharSet, lo: number, hi: number): void { + for (let c = lo < 0 ? 0 : lo; c <= hi && c < ASCII; c++) s[c] = true; +} + +function unionInto(into: CharSet, from: CharSet): void { + for (let c = 0; c < ASCII; c++) if (from[c]) into[c] = true; +} + +function inverted(s: CharSet): CharSet { + const out: CharSet = []; + for (let c = 0; c < ASCII; c++) out.push(!s[c]); + return out; +} + +/** Canonicalize closure for ASCII members: with or without `u`, ASCII + * letters fold only onto their other ASCII case. */ +function caseClosed(s: CharSet): CharSet { + const out = s.slice(); + for (let c = 65; c <= 90; c++) { + if (s[c] || s[c + 32]) { + out[c] = true; + out[c + 32] = true; + } + } + return out; +} + +function sameSet(a: CharSet, b: CharSet): boolean { + for (let c = 0; c < ASCII; c++) if (a[c] !== b[c]) return false; + return true; +} + +function setCount(s: CharSet): number { + let n = 0; + for (let c = 0; c < ASCII; c++) if (s[c]) n++; + return n; +} + +function disjoint(a: CharSet, b: CharSet): boolean { + for (let c = 0; c < ASCII; c++) if (a[c] && b[c]) return false; + return true; +} + +function digitSet(): CharSet { + const s = emptySet(); + addRange(s, 48, 57); + return s; +} + +function wordSet(): CharSet { + const s = emptySet(); + addRange(s, 48, 57); + addRange(s, 65, 90); + addRange(s, 97, 122); + s[95] = true; + return s; +} + +/** The ASCII members of WhiteSpace and LineTerminator. */ +function spaceSet(): CharSet { + const s = emptySet(); + addRange(s, 9, 13); + s[32] = true; + return s; +} + +function isDigitCode(c: number): boolean { + return c >= 48 && c <= 57; +} + +function hexValue(c: number): number { + if (c >= 48 && c <= 57) return c - 48; + if (c >= 65 && c <= 70) return c - 55; + if (c >= 97 && c <= 102) return c - 87; + return -1; +} + +const SYNTAX_CHARS = "^$\\.*+?()[]{}|/"; + +/** Parser for the supported subset. Anything else (lookaround, back + * references, `\c`, `\p`, octal escapes, non-ASCII pattern text, Annex B + * literal braces) bails, which keeps the interpreter for that literal. The + * pattern already passed the TypeScript parser, and the runtime compiles + * its bytecode (and reports its syntax errors) before any match runs. */ +class RxParser { + private i = 0; + caps = 1; + nodes = 0; + + constructor( + private readonly src: string, + private readonly unicode: boolean, + private readonly fold: boolean, + private readonly dotAll: boolean, + private readonly multiline: boolean, + ) {} + + parse(): RxNode { + const node = this.disjunction(); + if (this.i !== this.src.length) bail("unbalanced"); + return node; + } + + private code(): number { + return this.i < this.src.length ? this.src.charCodeAt(this.i) : -1; + } + + private ch(): string { + return this.i < this.src.length ? this.src.charAt(this.i) : ""; + } + + private count(): void { + if (++this.nodes > MAX_NODES) bail("too large"); + } + + private disjunction(): RxNode { + const alts: RxNode[] = [this.alternative()]; + while (this.ch() === "|") { + this.i++; + alts.push(this.alternative()); + } + this.count(); + return alts.length === 1 ? alts[0]! : { kind: "alt", alts }; + } + + private alternative(): RxNode { + const items: RxNode[] = []; + for (;;) { + const c = this.ch(); + if (c === "" || c === "|" || c === ")") break; + const assertion = this.assertion(); + if (assertion !== null) { + if (this.quantifierAhead()) bail("quantified assertion"); + items.push(assertion); + continue; + } + const capsBefore = this.caps; + const atom = this.atom(); + const q = this.quantifier(); + if (q === null) { + items.push(atom); + continue; + } + this.count(); + items.push({ + kind: "rep", + body: atom, + min: q.min, + max: q.max, + greedy: q.greedy, + capFirst: capsBefore, + capEnd: this.caps, + }); + } + this.count(); + return items.length === 1 ? items[0]! : { kind: "seq", items }; + } + + private assertion(): AssertNode | null { + const c = this.ch(); + let what: AssertKind | null = null; + if (c === "^") { + what = this.multiline ? "bolm" : "bol"; + this.i++; + } else if (c === "$") { + what = this.multiline ? "eolm" : "eol"; + this.i++; + } else if (c === "\\" && this.i + 1 < this.src.length) { + const n = this.src.charAt(this.i + 1); + if (n === "b" || n === "B") { + what = n === "b" ? "wordb" : "nwordb"; + this.i += 2; + } + } + if (what === null) return null; + this.count(); + return { kind: "assert", what }; + } + + private quantifierAhead(): boolean { + const c = this.ch(); + return c === "*" || c === "+" || c === "?" || c === "{"; + } + + private number(): number { + const start = this.i; + let value = 0; + while (isDigitCode(this.code())) { + value = value * 10 + (this.code() - 48); + if (value > MAX_COUNT) bail("large count"); + this.i++; + } + return this.i === start ? -1 : value; + } + + private quantifier(): { min: number; max: number; greedy: boolean } | null { + const c = this.ch(); + let min = 0; + let max = INF; + if (c === "*") { + this.i++; + } else if (c === "+") { + min = 1; + this.i++; + } else if (c === "?") { + max = 1; + this.i++; + } else if (c === "{") { + const save = this.i; + this.i++; + const lo = this.number(); + if (lo < 0) { + this.i = save; + return null; + } + min = lo; + max = lo; + if (this.ch() === ",") { + this.i++; + if (this.ch() === "}") { + max = INF; + } else { + const hi = this.number(); + if (hi < 0) { + this.i = save; + return null; + } + max = hi; + } + } + if (this.ch() !== "}") { + this.i = save; + return null; + } + this.i++; + if (max !== INF && min > max) bail("bad range"); + } else { + return null; + } + let greedy = true; + if (this.ch() === "?") { + greedy = false; + this.i++; + } + return { min, max, greedy }; + } + + private literal(code: number): SetNode { + const set = emptySet(); + set[code] = true; + return { kind: "set", set: this.fold ? caseClosed(set) : set }; + } + + private atom(): RxNode { + this.count(); + const c = this.ch(); + const code = this.code(); + if (c === ".") { + this.i++; + const set = emptySet(); + addRange(set, 0, ASCII - 1); + if (!this.dotAll) { + set[10] = false; + set[13] = false; + } + return { kind: "set", set }; + } + if (c === "(") { + this.i++; + let index = -1; + if (this.ch() === "?") { + const n = this.i + 1 < this.src.length ? this.src.charAt(this.i + 1) : ""; + if (n === ":") { + this.i += 2; + } else if (n === "<") { + const m = this.i + 2 < this.src.length ? this.src.charAt(this.i + 2) : ""; + if (m === "=" || m === "!") bail("lookbehind"); + this.i += 2; + const nameStart = this.i; + while (/[A-Za-z0-9_$]/.test(this.ch())) this.i++; + if (this.i === nameStart || this.ch() !== ">") bail("group name"); + this.i++; + index = this.caps++; + } else { + bail("group modifier or lookahead"); + } + } else { + index = this.caps++; + } + const body = this.disjunction(); + if (this.ch() !== ")") bail("unclosed group"); + this.i++; + return index < 0 ? body : { kind: "cap", index, body }; + } + if (c === "[") return this.charClass(); + if (c === "\\") return this.atomEscape(); + if (code < 0 || code >= ASCII) bail("non-ASCII pattern text"); + if ("*+?{".includes(c)) bail("nothing to repeat or literal brace"); + if (c === "}" || c === "]") bail("lone bracket"); + this.i++; + return this.literal(code); + } + + private classEscapeSet(c: string): CharSet | null { + switch (c) { + case "d": + return digitSet(); + case "D": + return inverted(digitSet()); + case "w": + return wordSet(); + case "W": + return inverted(wordSet()); + case "s": + return spaceSet(); + case "S": + return inverted(spaceSet()); + default: + return null; + } + } + + /** The code unit of a character escape after `\` (the `\` consumed), or + * bails. `inClass` admits `\b` (backspace) and `\-`. */ + private characterEscape(inClass: boolean): number { + const c = this.ch(); + const code = this.code(); + this.i++; + switch (c) { + case "t": + return 9; + case "n": + return 10; + case "v": + return 11; + case "f": + return 12; + case "r": + return 13; + case "0": + if (isDigitCode(this.code())) bail("octal escape"); + return 0; + case "x": { + const a = hexValue(this.code()); + const b = this.i + 1 < this.src.length ? hexValue(this.src.charCodeAt(this.i + 1)) : -1; + if (a < 0 || b < 0) bail("Annex B \\x"); + this.i += 2; + return a * 16 + b; + } + case "u": { + let value = 0; + for (let k = 0; k < 4; k++) { + const h = this.i + k < this.src.length ? hexValue(this.src.charCodeAt(this.i + k)) : -1; + if (h < 0) bail("\\u form"); + value = value * 16 + h; + } + this.i += 4; + return value; + } + case "b": + if (inClass) return 8; + bail("unreachable \\b"); + case "-": + if (inClass || !this.unicode) return 45; + bail("\\- outside a class"); + default: + break; + } + if (code < 0 || code >= ASCII) bail("non-ASCII escape"); + if (/[A-Za-z0-9_]/.test(c)) bail("escape letter or digit"); + if (this.unicode && !SYNTAX_CHARS.includes(c)) bail("u-mode identity escape"); + return code; + } + + private atomEscape(): RxNode { + this.i++; + const set = this.classEscapeSet(this.ch()); + if (set !== null) { + this.i++; + return { kind: "set", set }; + } + const code = this.characterEscape(false); + if (code >= ASCII) bail("non-ASCII escape"); + return this.literal(code); + } + + private charClass(): SetNode { + this.i++; + let negate = false; + if (this.ch() === "^") { + negate = true; + this.i++; + } + const set = emptySet(); + for (;;) { + const c = this.ch(); + if (c === "") bail("unclosed class"); + if (c === "]") { + this.i++; + break; + } + const first = this.classAtom(); + if ( + this.ch() === "-" && + this.i + 1 < this.src.length && + this.src.charAt(this.i + 1) !== "]" + ) { + this.i++; + const second = this.classAtom(); + if (typeof first !== "number" || typeof second !== "number") bail("class escape range"); + if (first > second) bail("bad class range"); + if (this.fold && second >= ASCII) bail("non-ASCII folded range"); + addRange(set, first, second); + continue; + } + if (typeof first === "number") { + if (first < ASCII) set[first] = true; + else if (this.fold) bail("non-ASCII folded member"); + } else { + unionInto(set, first); + } + } + const closed = this.fold ? caseClosed(set) : set; + return { kind: "set", set: negate ? inverted(closed) : closed }; + } + + /** A class member: a code unit, or the set of a class escape. Non-ASCII + * code units can only come from `\x`/`\u` escapes here. */ + private classAtom(): number | CharSet { + const c = this.ch(); + const code = this.code(); + if (c !== "\\") { + if (code < 0 || code >= ASCII) bail("non-ASCII class member"); + this.i++; + return code; + } + this.i++; + const set = this.classEscapeSet(this.ch()); + if (set !== null) { + this.i++; + return set; + } + if (this.ch() === "c") bail("\\c"); + return this.characterEscape(true); + } +} + +/** What can start a match from a point of the pattern. */ +interface FirstInfo { + set: CharSet; + /** Some path reaches the end without consuming a character. */ + nullable: boolean; + /** Some path passes a non-multiline `^` (matches only at 0). */ + bol: boolean; + /** Some path reaches the end without consuming and without any + * assertion or empty-iteration check that could fail. */ + uncond: boolean; +} + +function firstOf(node: RxNode): FirstInfo { + switch (node.kind) { + case "set": + return { set: node.set, nullable: false, bol: false, uncond: false }; + case "seq": { + const acc: FirstInfo = { set: emptySet(), nullable: true, bol: false, uncond: true }; + for (const item of node.items) { + const f = firstOf(item); + unionInto(acc.set, f.set); + if (f.bol) acc.bol = true; + if (!f.nullable) { + acc.nullable = false; + acc.uncond = false; + break; + } + if (!f.uncond) acc.uncond = false; + } + return acc; + } + case "alt": { + const acc: FirstInfo = { set: emptySet(), nullable: false, bol: false, uncond: false }; + for (const alt of node.alts) { + const f = firstOf(alt); + unionInto(acc.set, f.set); + if (f.nullable) acc.nullable = true; + if (f.bol) acc.bol = true; + if (f.uncond) acc.uncond = true; + } + return acc; + } + case "cap": + return firstOf(node.body); + case "rep": { + if (node.max === 0) return { set: emptySet(), nullable: true, bol: false, uncond: true }; + const f = firstOf(node.body); + return { + set: f.set, + nullable: node.min === 0 || f.nullable, + bol: f.bol, + uncond: node.min === 0 || f.uncond, + }; + } + case "assert": + return node.what === "bol" + ? { set: emptySet(), nullable: false, bol: true, uncond: false } + : { set: emptySet(), nullable: true, bol: false, uncond: false }; + } +} + +/** Whether a node can match without consuming (assertions included). */ +function canMatchEmpty(node: RxNode): boolean { + switch (node.kind) { + case "set": + return false; + case "seq": + return node.items.every(canMatchEmpty); + case "alt": + return node.alts.some(canMatchEmpty); + case "cap": + return canMatchEmpty(node.body); + case "rep": + return node.min === 0 || node.max === 0 || canMatchEmpty(node.body); + case "assert": + return true; + } +} + +/** `first` followed by `follow`. */ +function thenFollow(first: FirstInfo, follow: FirstInfo): FirstInfo { + if (!first.nullable) return { set: first.set, nullable: false, bol: first.bol, uncond: false }; + const set = first.set.slice(); + unionInto(set, follow.set); + return { + set, + nullable: follow.nullable, + bol: first.bol || follow.bol, + uncond: first.uncond && follow.uncond, + }; +} + +/** Inclusive code ranges of a set. */ +function intervals(s: CharSet): number[][] { + const out: number[][] = []; + for (let c = 0; c < ASCII; c++) { + if (!s[c]) continue; + let e = c; + while (e + 1 < ASCII && s[e + 1]) e++; + out.push([c, e]); + c = e; + } + return out; +} + +/** Vector tests use at most this many range compares. */ +const VECTOR_RANGES = 6; + +function splat16(n: number): string { + const parts: string[] = []; + for (let k = 0; k < 16; k++) parts.push(`i8 ${n}`); + return `<${parts.join(", ")}>`; +} + +class RxGen { + readonly body: string[] = []; + private nTmp = 0; + private nLabel = 0; + /** Resume label of each stack entry kind, by id. */ + private readonly resumes: string[] = []; + private readonly undoIds = new Map(); + /** Register count: 2 * ncap capture slots, then loop registers. */ + regs: number; + private readonly tables: CharSet[] = []; + readonly globals: string[] = []; + + constructor( + readonly sym: string, + readonly ncap: number, + /** 16-byte vector scans for long character runs. */ + readonly vector: boolean, + ) { + this.regs = 2 * ncap; + } + + /** The compares of a vector membership test: the set's ranges, or its + * complement's then negated (bytes >= 128 never occur), or null. */ + vectorPlan(set: CharSet): { ranges: number[][]; negate: boolean } | null { + if (!this.vector) return null; + const n = setCount(set); + if (n === 0 || n === ASCII) return null; + const direct = intervals(set), + complement = intervals(inverted(set)); + if (direct.length <= complement.length && direct.length <= VECTOR_RANGES) + return { ranges: direct, negate: false }; + if (complement.length <= VECTOR_RANGES) return { ranges: complement, negate: true }; + return null; + } + + /** Membership of 16 code units at s[index..index+16): an i16 mask, bit + * k set when s[index + k] is in the set. */ + testVector(plan: { ranges: number[][]; negate: boolean }, index: string): string { + const wide = this.tmp(), + at = this.tmp(), + v = this.tmp(); + this.line(`${wide} = sext i32 ${index} to i64`); + this.line(`${at} = getelementptr inbounds i8, ptr %s, i64 ${wide}`); + this.line(`${v} = load <16 x i8>, ptr ${at}, align 1`); + let acc = ""; + for (const [lo, hi] of plan.ranges) { + const hit = this.tmp(); + if (lo === hi) { + this.line(`${hit} = icmp eq <16 x i8> ${v}, ${splat16(lo!)}`); + } else { + const d = this.tmp(); + this.line(`${d} = sub <16 x i8> ${v}, ${splat16(lo!)}`); + this.line(`${hit} = icmp ule <16 x i8> ${d}, ${splat16(hi! - lo!)}`); + } + if (acc === "") { + acc = hit; + } else { + const both = this.tmp(); + this.line(`${both} = or <16 x i1> ${acc}, ${hit}`); + acc = both; + } + } + const bits = this.tmp(); + this.line(`${bits} = bitcast <16 x i1> ${acc} to i16`); + if (!plan.negate) return bits; + const flipped = this.tmp(); + this.line(`${flipped} = xor i16 ${bits}, -1`); + return flipped; + } + + /** Index of the lowest set bit of a nonzero i16 mask, as i32. */ + lowestBit(mask: string): string { + const tz = this.tmp(), + wide = this.tmp(); + this.line(`${tz} = call i16 @llvm.cttz.i16(i16 ${mask}, i1 true)`); + this.line(`${wide} = zext i16 ${tz} to i32`); + return wide; + } + + /** The first-character search: from %at, store the next position whose + * character is in `set` and branch to %attempt, or branch to %nomatch. */ + emitSeek(set: CharSet): void { + this.block("seek"); + const a = this.tmp(); + this.line(`${a} = load i32, ptr %at`); + const plan = this.vectorPlan(set); + let from = "seek", + start = a; + if (plan !== null) { + const head = this.label("vsh"), + load = this.label("vsl"), + next = this.label("vsn"), + hit = this.label("vsx"), + tail = this.label("vst"); + this.line(`br label %${head}`); + this.block(head); + const i = this.tmp(), + i16 = this.tmp(), + fits = this.tmp(); + this.line(`${i} = phi i32 [ ${a}, %seek ], [ ${i16}, %${next} ]`); + this.line(`${i16} = add i32 ${i}, 16`); + this.line(`${fits} = icmp sle i32 ${i16}, %len`); + this.line(`br i1 ${fits}, label %${load}, label %${tail}`); + this.block(load); + const mask = this.testVector(plan, i), + any = this.tmp(); + this.line(`${any} = icmp ne i16 ${mask}, 0`); + this.line(`br i1 ${any}, label %${hit}, label %${next}`); + this.block(next); + this.line(`br label %${head}`); + this.block(hit); + const k = this.lowestBit(mask), + found = this.tmp(); + this.line(`${found} = add i32 ${i}, ${k}`); + this.line(`store i32 ${found}, ptr %at`); + this.line(`br label %attempt`); + this.block(tail); + from = tail; + start = i; + } + const head = this.label("ssh"), + probe = this.label("sst"), + next = this.label("ssn"), + found = this.label("ssf"); + this.line(`br label %${head}`); + this.block(head); + const j = this.tmp(), + j1 = this.tmp(), + inb = this.tmp(); + this.line(`${j} = phi i32 [ ${start}, %${from} ], [ ${j1}, %${next} ]`); + this.line(`${inb} = icmp slt i32 ${j}, %len`); + this.line(`br i1 ${inb}, label %${probe}, label %nomatch`); + this.block(probe); + const hit = this.test(set, this.loadChar(j)); + this.line(`br i1 ${hit}, label %${found}, label %${next}`); + this.block(next); + this.line(`${j1} = add i32 ${j}, 1`); + this.line(`br label %${head}`); + this.block(found); + this.line(`store i32 ${j}, ptr %at`); + this.line(`br label %attempt`); + } + + tmp(): string { + return `%t${this.nTmp++}`; + } + + label(prefix: string): string { + return `${prefix}${this.nLabel++}`; + } + + block(label: string): void { + this.body.push(`${label}:`); + } + + line(text: string): void { + this.body.push(` ${text}`); + } + + newReg(): number { + return this.regs++; + } + + newResume(label: string): number { + this.resumes.push(label); + return this.resumes.length - 1; + } + + loadPos(): string { + const p = this.tmp(); + this.line(`${p} = load i32, ptr %pos`); + return p; + } + + storePos(v: string): void { + this.line(`store i32 ${v}, ptr %pos`); + } + + loadReg(r: number): string { + const v = this.tmp(); + this.line(`${v} = load i32, ptr %r${r}`); + return v; + } + + loadChar(index: string): string { + const wide = this.tmp(), + at = this.tmp(), + c = this.tmp(); + this.line(`${wide} = sext i32 ${index} to i64`); + this.line(`${at} = getelementptr inbounds i8, ptr %s, i64 ${wide}`); + this.line(`${c} = load i8, ptr ${at}`); + return c; + } + + /** Straight-line membership test of an i8 ASCII code unit. */ + test(set: CharSet, c: string): string { + const n = setCount(set); + if (n === ASCII) return "true"; + if (n === 0) return "false"; + const ranges = intervals(set); + if (ranges.length <= 3) { + const c32 = this.tmp(); + this.line(`${c32} = zext i8 ${c} to i32`); + let acc = ""; + for (const [lo, hi] of ranges) { + const hit = this.tmp(); + if (lo === hi) { + this.line(`${hit} = icmp eq i32 ${c32}, ${lo!}`); + } else { + const d = this.tmp(); + this.line(`${d} = sub i32 ${c32}, ${lo!}`); + this.line(`${hit} = icmp ule i32 ${d}, ${hi! - lo!}`); + } + if (acc === "") { + acc = hit; + } else { + const both = this.tmp(); + this.line(`${both} = or i1 ${acc}, ${hit}`); + acc = both; + } + } + return acc; + } + const table = this.table(set), + masked = this.tmp(), + wide = this.tmp(), + slot = this.tmp(), + v = this.tmp(), + hit = this.tmp(); + this.line(`${masked} = and i8 ${c}, 127`); + this.line(`${wide} = zext i8 ${masked} to i64`); + this.line(`${slot} = getelementptr inbounds [128 x i8], ptr ${table}, i64 0, i64 ${wide}`); + this.line(`${v} = load i8, ptr ${slot}`); + this.line(`${hit} = icmp ne i8 ${v}, 0`); + return hit; + } + + private table(set: CharSet): string { + for (let k = 0; k < this.tables.length; k++) + if (sameSet(this.tables[k]!, set)) return `@${this.sym}_t${k}`; + const k = this.tables.length; + this.tables.push(set); + let bytes = ""; + for (let c = 0; c < ASCII; c++) bytes += set[c] ? "\\01" : "\\00"; + this.globals.push(`@${this.sym}_t${k} = private unnamed_addr constant [128 x i8] c"${bytes}"`); + return `@${this.sym}_t${k}`; + } + + /** Push data words and an entry id; may split the current block. */ + push(values: string[], id: number): void { + const n = values.length + 1; + const sp = this.tmp(), + need = this.tmp(), + cap = this.tmp(), + full = this.tmp(); + const grow = this.label("grow"), + put = this.label("put"); + this.line(`${sp} = load i32, ptr %sp`); + this.line(`${need} = add i32 ${sp}, ${n}`); + this.line(`${cap} = load i32, ptr %scap`); + this.line(`${full} = icmp sgt i32 ${need}, ${cap}`); + this.line(`br i1 ${full}, label %${grow}, label %${put}`); + this.block(grow); + this.line(`call void @scr_regex_stack_grow(ptr %stk, ptr %scap, ptr %sbuf, i32 ${need})`); + this.line(`br label %${put}`); + this.block(put); + const base = this.tmp(); + this.line(`${base} = load ptr, ptr %stk`); + const words = values.concat([String(id)]); + for (let k = 0; k < words.length; k++) { + const idx = this.tmp(), + wide = this.tmp(), + slot = this.tmp(); + this.line(`${idx} = add i32 ${sp}, ${k}`); + this.line(`${wide} = sext i32 ${idx} to i64`); + this.line(`${slot} = getelementptr inbounds i32, ptr ${base}, i64 ${wide}`); + this.line(`store i32 ${words[k]!}, ptr ${slot}`); + } + this.line(`store i32 ${need}, ptr %sp`); + } + + /** Pop `n` data words in push order (the dispatcher popped the id). */ + pop(n: number): string[] { + const sp = this.tmp(), + base = this.tmp(), + rest = this.tmp(); + this.line(`${sp} = load i32, ptr %sp`); + this.line(`${base} = load ptr, ptr %stk`); + const out: string[] = []; + for (let k = 0; k < n; k++) { + const idx = this.tmp(), + wide = this.tmp(), + slot = this.tmp(), + v = this.tmp(); + this.line(`${idx} = sub i32 ${sp}, ${n - k}`); + this.line(`${wide} = sext i32 ${idx} to i64`); + this.line(`${slot} = getelementptr inbounds i32, ptr ${base}, i64 ${wide}`); + this.line(`${v} = load i32, ptr ${slot}`); + out.push(v); + } + this.line(`${rest} = sub i32 ${sp}, ${n}`); + this.line(`store i32 ${rest}, ptr %sp`); + return out; + } + + /** Register write that a backtrack restores. With an empty stack no + * resume can observe the old value (the attempt fails outright and the + * next attempt reinitializes), so no undo record is pushed. */ + setReg(r: number, value: string): void { + let id = this.undoIds.get(r); + if (id === undefined) { + id = this.newResume(`undo${r}`); + this.undoIds.set(r, id); + } + const sp = this.tmp(), + live = this.tmp(); + const save = this.label("save"), + set = this.label("set"); + this.line(`${sp} = load i32, ptr %sp`); + this.line(`${live} = icmp ne i32 ${sp}, 0`); + this.line(`br i1 ${live}, label %${save}, label %${set}`); + this.block(save); + const old = this.loadReg(r); + this.push([old], id); + this.line(`br label %${set}`); + this.block(set); + this.line(`store i32 ${value}, ptr %r${r}`); + } + + resetCaps(first: number, end: number): void { + for (let g = first; g < end; g++) { + this.setReg(2 * g, "-1"); + this.setReg(2 * g + 1, "-1"); + } + } + + emit(node: RxNode, succ: string, follow: FirstInfo): string { + switch (node.kind) { + case "set": + return this.emitRun([node.set], succ); + case "seq": + return this.emitSeq(node.items, succ, follow); + case "alt": + return this.emitAlt(node.alts, succ, follow); + case "cap": + return this.emitCap(node, succ, follow); + case "rep": + return this.emitRep(node, succ, follow); + case "assert": + return this.emitAssert(node.what, succ); + } + } + + /** Consecutive single characters: one bounds check, then each test. */ + private emitRun(sets: CharSet[], succ: string): string { + const entry = this.label("run"), + check = this.label("chk"); + this.block(entry); + const p = this.loadPos(), + end = this.tmp(), + inb = this.tmp(); + this.line(`${end} = add i32 ${p}, ${sets.length}`); + this.line(`${inb} = icmp sle i32 ${end}, %len`); + this.line(`br i1 ${inb}, label %${check}, label %fail`); + this.block(check); + let acc = ""; + for (let k = 0; k < sets.length; k++) { + let at = p; + if (k > 0) { + at = this.tmp(); + this.line(`${at} = add i32 ${p}, ${k}`); + } + const hit = this.test(sets[k]!, this.loadChar(at)); + if (acc === "") { + acc = hit; + } else { + const both = this.tmp(); + this.line(`${both} = and i1 ${acc}, ${hit}`); + acc = both; + } + } + // Storing before the branch is safe: every resume sets the position. + this.storePos(end); + this.line(`br i1 ${acc}, label %${succ}, label %fail`); + return entry; + } + + private emitSeq(items: RxNode[], succ: string, follow: FirstInfo): string { + // Group consecutive single characters into runs. + const parts: (RxNode | CharSet[])[] = []; + for (const item of items) { + const last = parts.length > 0 ? parts[parts.length - 1]! : null; + if (item.kind === "set" && Array.isArray(last) && last.length < 16) { + last.push(item.set); + } else if (item.kind === "set") { + parts.push([item.set]); + } else { + parts.push(item); + } + } + let next = succ; + let after = follow; + for (let k = parts.length - 1; k >= 0; k--) { + const part = parts[k]!; + if (Array.isArray(part)) { + next = this.emitRun(part, next); + after = { set: part[0]!, nullable: false, bol: false, uncond: false }; + } else { + next = this.emit(part, next, after); + after = thenFollow(firstOf(part), after); + } + } + if (parts.length === 0) { + const entry = this.label("empty"); + this.block(entry); + this.line(`br label %${succ}`); + return entry; + } + return next; + } + + private emitAlt(alts: RxNode[], succ: string, follow: FirstInfo): string { + const entries: string[] = []; + for (const alt of alts) entries.push(this.emit(alt, succ, follow)); + const resumeLabels: string[] = []; + const ids: number[] = []; + for (let k = 1; k < alts.length; k++) { + const l = this.label("alt"); + resumeLabels.push(l); + ids.push(this.newResume(l)); + } + const entry = this.label("alts"); + this.block(entry); + const p = this.loadPos(); + this.push([p], ids[0]!); + this.line(`br label %${entries[0]!}`); + for (let k = 1; k < alts.length; k++) { + this.block(resumeLabels[k - 1]!); + const [q] = this.pop(1); + this.storePos(q!); + if (k + 1 < alts.length) this.push([q!], ids[k]!); + this.line(`br label %${entries[k]!}`); + } + return entry; + } + + private emitCap(node: CapNode, succ: string, follow: FirstInfo): string { + const close = this.label("close"); + this.block(close); + const end = this.loadPos(); + this.setReg(2 * node.index + 1, end); + this.line(`br label %${succ}`); + const body = this.emit(node.body, close, follow); + const open = this.label("open"); + this.block(open); + const start = this.loadPos(); + this.setReg(2 * node.index, start); + this.line(`br label %${body}`); + return open; + } + + private emitAssert(what: AssertKind, succ: string): string { + const entry = this.label("as"); + this.block(entry); + const p = this.loadPos(); + if (what === "bol" || what === "eol") { + const hit = this.tmp(); + this.line(`${hit} = icmp eq i32 ${p}, ${what === "bol" ? "0" : "%len"}`); + this.line(`br i1 ${hit}, label %${succ}, label %fail`); + return entry; + } + if (what === "bolm" || what === "eolm") { + const edge = this.tmp(), + look = this.label("nl"); + this.line(`${edge} = icmp eq i32 ${p}, ${what === "bolm" ? "0" : "%len"}`); + this.line(`br i1 ${edge}, label %${succ}, label %${look}`); + this.block(look); + let at = p; + if (what === "bolm") { + at = this.tmp(); + this.line(`${at} = sub i32 ${p}, 1`); + } + const nl = emptySet(); + nl[10] = true; + nl[13] = true; + const hit = this.test(nl, this.loadChar(at)); + this.line(`br i1 ${hit}, label %${succ}, label %fail`); + return entry; + } + const word = wordSet(); + const prev = this.label("wp"), + noPrev = this.label("wq"), + mid = this.label("wm"), + next = this.label("wn"), + noNext = this.label("wo"), + done = this.label("wd"); + const hasPrev = this.tmp(); + this.line(`${hasPrev} = icmp sgt i32 ${p}, 0`); + this.line(`br i1 ${hasPrev}, label %${prev}, label %${noPrev}`); + this.block(prev); + const pm = this.tmp(); + this.line(`${pm} = sub i32 ${p}, 1`); + const w1 = this.test(word, this.loadChar(pm)); + this.line(`br label %${mid}`); + this.block(noPrev); + this.line(`br label %${mid}`); + this.block(mid); + const a = this.tmp(), + hasNext = this.tmp(); + this.line(`${a} = phi i1 [ ${w1}, %${prev} ], [ false, %${noPrev} ]`); + this.line(`${hasNext} = icmp slt i32 ${p}, %len`); + this.line(`br i1 ${hasNext}, label %${next}, label %${noNext}`); + this.block(next); + const w2 = this.test(word, this.loadChar(p)); + this.line(`br label %${done}`); + this.block(noNext); + this.line(`br label %${done}`); + this.block(done); + const b = this.tmp(), + edge = this.tmp(); + this.line(`${b} = phi i1 [ ${w2}, %${next} ], [ false, %${noNext} ]`); + this.line(`${edge} = xor i1 ${a}, ${b}`); + const yes = what === "wordb" ? succ : "fail", + no = what === "wordb" ? "fail" : succ; + this.line(`br i1 ${edge}, label %${yes}, label %${no}`); + return entry; + } + + private emitRep(node: RepNode, succ: string, follow: FirstInfo): string { + if (node.max === 0) { + const entry = this.label("none"); + this.block(entry); + this.line(`br label %${succ}`); + return entry; + } + if (node.body.kind === "set") { + return node.greedy + ? this.emitGreedyChars(node.body.set, node.min, node.max, succ, follow) + : this.emitLazyChars(node.body.set, node.min, node.max, succ); + } + if (node.min === 1 && node.max === 1) return this.emit(node.body, succ, follow); + return this.emitLoop(node, succ, follow); + } + + /** `x*`, `x+`, `x{n,m}` over one character: count the run, then give + * characters back one at a time on backtrack (skipping positions where + * the continuation cannot start). Possessive when giving back can never + * succeed. */ + private emitGreedyChars( + set: CharSet, + min: number, + max: number, + succ: string, + follow: FirstInfo, + ): string { + const entry = this.label("gl"), + head = this.label("glh"), + probe = this.label("glt"), + step = this.label("glb"), + done = this.label("gld"), + ok = this.label("glo"); + this.block(entry); + const p = this.loadPos(), + rem = this.tmp(); + this.line(`${rem} = sub i32 %len, ${p}`); + let lim = rem; + if (max !== INF) { + const small = this.tmp(); + lim = this.tmp(); + this.line(`${small} = icmp slt i32 ${rem}, ${max}`); + this.line(`${lim} = select i1 ${small}, i32 ${rem}, i32 ${max}`); + } + // Long runs: 16 characters per step while a whole chunk fits. + const plan = max === INF || max >= 32 ? this.vectorPlan(set) : null; + let from = entry, + start = "0", + vectorExit = ""; + if (plan !== null) { + const vhead = this.label("gvh"), + vload = this.label("gvl"), + vnext = this.label("gvn"), + vexit = this.label("gvx"), + tail = this.label("gvt"); + this.line(`br label %${vhead}`); + this.block(vhead); + const kv = this.tmp(), + kv16 = this.tmp(), + fits = this.tmp(); + this.line(`${kv} = phi i32 [ 0, %${entry} ], [ ${kv16}, %${vnext} ]`); + this.line(`${kv16} = add i32 ${kv}, 16`); + this.line(`${fits} = icmp sle i32 ${kv16}, ${lim}`); + this.line(`br i1 ${fits}, label %${vload}, label %${tail}`); + this.block(vload); + const vat = this.tmp(); + this.line(`${vat} = add i32 ${p}, ${kv}`); + const mask = this.testVector(plan, vat), + all = this.tmp(); + this.line(`${all} = icmp eq i16 ${mask}, -1`); + this.line(`br i1 ${all}, label %${vnext}, label %${vexit}`); + this.block(vnext); + this.line(`br label %${vhead}`); + this.block(vexit); + const missing = this.tmp(); + this.line(`${missing} = xor i16 ${mask}, -1`); + const off = this.lowestBit(missing), + kx = this.tmp(); + this.line(`${kx} = add i32 ${kv}, ${off}`); + this.line(`br label %${done}`); + vectorExit = `, [ ${kx}, %${vexit} ]`; + this.block(tail); + from = tail; + start = kv; + } + this.line(`br label %${head}`); + this.block(head); + const k = this.tmp(), + k1 = this.tmp(), + more = this.tmp(); + this.line(`${k} = phi i32 [ ${start}, %${from} ], [ ${k1}, %${step} ]`); + this.line(`${more} = icmp slt i32 ${k}, ${lim}`); + this.line(`br i1 ${more}, label %${probe}, label %${done}`); + this.block(probe); + const at = this.tmp(); + this.line(`${at} = add i32 ${p}, ${k}`); + const hit = this.test(set, this.loadChar(at)); + this.line(`br i1 ${hit}, label %${step}, label %${done}`); + this.block(step); + this.line(`${k1} = add i32 ${k}, 1`); + this.line(`br label %${head}`); + this.block(done); + const kf = this.tmp(), + low = this.tmp(); + this.line(`${kf} = phi i32 [ ${k}, %${head} ], [ ${k}, %${probe} ]${vectorExit}`); + this.line(`${low} = icmp slt i32 ${kf}, ${min}`); + this.line(`br i1 ${low}, label %fail, label %${ok}`); + this.block(ok); + const np = this.tmp(); + this.line(`${np} = add i32 ${p}, ${kf}`); + this.storePos(np); + const possessive = + follow.uncond || (!follow.nullable && !follow.bol && disjoint(set, follow.set)); + if (possessive || min === max) { + this.line(`br label %${succ}`); + return entry; + } + const give = this.label("gv"); + const id = this.newResume(give); + const keep = this.label("glp"); + const more2 = this.tmp(); + this.line(`${more2} = icmp sgt i32 ${kf}, ${min}`); + this.line(`br i1 ${more2}, label %${keep}, label %${succ}`); + this.block(keep); + this.push([p, kf], id); + this.line(`br label %${succ}`); + + this.block(give); + const [base, count] = this.pop(2); + const back = this.tmp(); + this.line(`${back} = sub i32 ${count!}, 1`); + let kk = back; + if (!follow.nullable && !follow.bol) { + // The continuation must consume a character of follow.set first. + const scan = this.label("gs"), + look = this.label("gst"), + down = this.label("gsd"), + found = this.label("gsf"); + const from = this.label("gse"); + this.line(`br label %${from}`); + this.block(from); + this.line(`br label %${scan}`); + this.block(scan); + const cur = this.tmp(), + dec = this.tmp(), + under = this.tmp(); + this.line(`${cur} = phi i32 [ ${back}, %${from} ], [ ${dec}, %${down} ]`); + this.line(`${under} = icmp slt i32 ${cur}, ${min}`); + this.line(`br i1 ${under}, label %fail, label %${look}`); + this.block(look); + const at2 = this.tmp(); + this.line(`${at2} = add i32 ${base!}, ${cur}`); + const can = this.test(follow.set, this.loadChar(at2)); + this.line(`br i1 ${can}, label %${found}, label %${down}`); + this.block(down); + this.line(`${dec} = sub i32 ${cur}, 1`); + this.line(`br label %${scan}`); + this.block(found); + kk = cur; + } + const again = this.label("gva"), + resume = this.label("gvr"); + const repush = this.tmp(); + this.line(`${repush} = icmp sgt i32 ${kk}, ${min}`); + this.line(`br i1 ${repush}, label %${again}, label %${resume}`); + this.block(again); + this.push([base!, kk], id); + this.line(`br label %${resume}`); + this.block(resume); + const rp = this.tmp(); + this.line(`${rp} = add i32 ${base!}, ${kk}`); + this.storePos(rp); + this.line(`br label %${succ}`); + return entry; + } + + /** `x*?`, `x+?`, `x{n,m}?` over one character: take the minimum, then + * one more character per backtrack. */ + private emitLazyChars(set: CharSet, min: number, max: number, succ: string): string { + const entry = this.label("ll"), + head = this.label("llh"), + probe = this.label("llt"), + step = this.label("llb"), + ok = this.label("llo"), + start = this.label("lls"); + this.block(entry); + const p = this.loadPos(), + rem = this.tmp(), + enough = this.tmp(); + this.line(`${rem} = sub i32 %len, ${p}`); + this.line(`${enough} = icmp sge i32 ${rem}, ${min}`); + this.line(`br i1 ${enough}, label %${start}, label %fail`); + this.block(start); + this.line(`br label %${head}`); + this.block(head); + const k = this.tmp(), + k1 = this.tmp(), + reached = this.tmp(); + this.line(`${k} = phi i32 [ 0, %${start} ], [ ${k1}, %${step} ]`); + this.line(`${reached} = icmp sge i32 ${k}, ${min}`); + this.line(`br i1 ${reached}, label %${ok}, label %${probe}`); + this.block(probe); + const at = this.tmp(); + this.line(`${at} = add i32 ${p}, ${k}`); + const hit = this.test(set, this.loadChar(at)); + this.line(`br i1 ${hit}, label %${step}, label %fail`); + this.block(step); + this.line(`${k1} = add i32 ${k}, 1`); + this.line(`br label %${head}`); + this.block(ok); + const np = this.tmp(); + this.line(`${np} = add i32 ${p}, ${min}`); + this.storePos(np); + if (max !== INF && max <= min) { + this.line(`br label %${succ}`); + return entry; + } + const extend = this.label("lx"); + const id = this.newResume(extend); + this.push([p, String(min)], id); + this.line(`br label %${succ}`); + + this.block(extend); + const [base, count] = this.pop(2); + const pos = this.tmp(), + inb = this.tmp(), + look = this.label("lxt"), + take = this.label("lxy"); + this.line(`${pos} = add i32 ${base!}, ${count!}`); + this.line(`${inb} = icmp slt i32 ${pos}, %len`); + this.line(`br i1 ${inb}, label %${look}, label %fail`); + this.block(look); + const can = this.test(set, this.loadChar(pos)); + this.line(`br i1 ${can}, label %${take}, label %fail`); + this.block(take); + const c1 = this.tmp(), + np2 = this.tmp(); + this.line(`${c1} = add i32 ${count!}, 1`); + this.line(`${np2} = add i32 ${pos}, 1`); + this.storePos(np2); + const again = this.label("lxa"); + if (max !== INF) { + const below = this.tmp(); + this.line(`${below} = icmp slt i32 ${c1}, ${max}`); + this.line(`br i1 ${below}, label %${again}, label %${succ}`); + } else { + this.line(`br label %${again}`); + } + this.block(again); + this.push([base!, c1], id); + this.line(`br label %${succ}`); + return entry; + } + + /** RepeatMatcher over a general atom: captures inside reset at each + * iteration; an optional iteration that matched empty fails. */ + private emitLoop(node: RepNode, succ: string, follow: FirstInfo): string { + const bodyFirst = firstOf(node.body); + const emptyCheck = canMatchEmpty(node.body); + const startReg = emptyCheck ? this.newReg() : -1; + // After the body: another iteration or the continuation. An empty + // check may fail in between, so nothing after the body is + // unconditional. + const bodyFollow: FirstInfo = { + set: follow.set.slice(), + nullable: follow.nullable, + bol: follow.bol || (node.max !== 1 && bodyFirst.bol), + uncond: false, + }; + if (node.max !== 1) { + unionInto(bodyFollow.set, bodyFirst.set); + if (bodyFirst.nullable) bodyFollow.nullable = true; + } + + if (node.min === 0 && node.max === 1) { + const iter = this.label("oi"), + after = this.label("oa"); + this.block(after); + if (emptyCheck) { + const p = this.loadPos(), + r = this.loadReg(startReg), + same = this.tmp(); + this.line(`${same} = icmp eq i32 ${p}, ${r}`); + this.line(`br i1 ${same}, label %fail, label %${succ}`); + } else { + this.line(`br label %${succ}`); + } + const bodyEntry = this.emit(node.body, after, bodyFollow); + this.block(iter); + const p = this.loadPos(); + if (emptyCheck) this.setReg(startReg, p); + this.resetCaps(node.capFirst, node.capEnd); + this.line(`br label %${bodyEntry}`); + const entry = this.label("opt"); + if (node.greedy) { + const skip = this.label("os"); + const id = this.newResume(skip); + this.block(entry); + const q = this.loadPos(); + this.push([q], id); + this.line(`br label %${iter}`); + this.block(skip); + const [r] = this.pop(1); + this.storePos(r!); + this.line(`br label %${succ}`); + } else { + const take = this.label("ot"); + const id = this.newResume(take); + this.block(entry); + const q = this.loadPos(); + this.push([q], id); + this.line(`br label %${succ}`); + this.block(take); + const [r] = this.pop(1); + this.storePos(r!); + this.line(`br label %${iter}`); + } + return entry; + } + + const counted = !(node.min === 0 && node.max === INF); + const cnt = counted ? this.newReg() : -1; + const entry = this.label("lp"), + head = this.label("lph"), + iter = this.label("lpi"), + after = this.label("lpa"); + // after: the empty check, then count the iteration. + this.block(after); + if (emptyCheck) { + const p = this.loadPos(), + r = this.loadReg(startReg); + let same = this.tmp(); + this.line(`${same} = icmp eq i32 ${p}, ${r}`); + if (counted && node.min > 0) { + const c = this.loadReg(cnt), + optional = this.tmp(), + both = this.tmp(); + this.line(`${optional} = icmp sge i32 ${c}, ${node.min}`); + this.line(`${both} = and i1 ${same}, ${optional}`); + same = both; + } + const count = this.label("lpc"); + this.line(`br i1 ${same}, label %fail, label %${count}`); + this.block(count); + } + if (counted) { + const c = this.loadReg(cnt), + c1 = this.tmp(); + this.line(`${c1} = add i32 ${c}, 1`); + this.setReg(cnt, c1); + } + this.line(`br label %${head}`); + const bodyEntry = this.emit(node.body, after, bodyFollow); + + this.block(entry); + if (counted) this.setReg(cnt, "0"); + this.line(`br label %${head}`); + + this.block(head); + if (counted) { + const c = this.loadReg(cnt); + if (node.min > 0) { + const req = this.tmp(), + next = this.label("lpm"); + this.line(`${req} = icmp slt i32 ${c}, ${node.min}`); + this.line(`br i1 ${req}, label %${iter}, label %${next}`); + this.block(next); + } + if (node.max !== INF) { + const full = this.tmp(), + next = this.label("lpx"); + this.line(`${full} = icmp eq i32 ${c}, ${node.max}`); + this.line(`br i1 ${full}, label %${succ}, label %${next}`); + this.block(next); + } + } + const q = this.loadPos(); + if (node.greedy) { + const exit = this.label("lpe"); + const id = this.newResume(exit); + this.push([q], id); + this.line(`br label %${iter}`); + this.block(exit); + const [r] = this.pop(1); + this.storePos(r!); + this.line(`br label %${succ}`); + } else { + const more = this.label("lpn"); + const id = this.newResume(more); + this.push([q], id); + this.line(`br label %${succ}`); + this.block(more); + const [r] = this.pop(1); + this.storePos(r!); + this.line(`br label %${iter}`); + } + + this.block(iter); + const p = this.loadPos(); + if (emptyCheck) this.setReg(startReg, p); + this.resetCaps(node.capFirst, node.capEnd); + this.line(`br label %${bodyEntry}`); + return entry; + } + + /** The failure dispatcher and the undo resumes. */ + emitTail(): void { + for (const [r, id] of this.undoIds) { + this.block(this.resumes[id]!); + const [old] = this.pop(1); + this.line(`store i32 ${old!}, ptr %r${r}`); + this.line(`br label %fail`); + } + this.block("fail"); + const sp = this.tmp(), + empty = this.tmp(); + this.line(`${sp} = load i32, ptr %sp`); + this.line(`${empty} = icmp eq i32 ${sp}, 0`); + this.line(`br i1 ${empty}, label %attempt_fail, label %pop`); + this.block("pop"); + const top = this.tmp(), + base = this.tmp(), + wide = this.tmp(), + slot = this.tmp(), + id = this.tmp(); + this.line(`${top} = sub i32 ${sp}, 1`); + this.line(`${base} = load ptr, ptr %stk`); + this.line(`${wide} = sext i32 ${top} to i64`); + this.line(`${slot} = getelementptr inbounds i32, ptr ${base}, i64 ${wide}`); + this.line(`${id} = load i32, ptr ${slot}`); + this.line(`store i32 ${top}, ptr %sp`); + const cases = this.resumes.map((l, k) => `i32 ${k}, label %${l}`).join(" "); + this.line(`switch i32 ${id}, label %bad [ ${cases} ]`); + this.block("bad"); + this.line(`unreachable`); + } +} + +export interface RegexMatcherCode { + /** Module-level lines: tables and the function definition. */ + lines: string[]; + /** External declarations the function uses. */ + decls: string[]; +} + +export const REGEX_MATCHER_DECLS = [ + `declare void @scr_regex_stack_grow(ptr, ptr, ptr, i32)`, + `declare void @scr_regex_stack_free(ptr)`, +]; + +/** Parse a literal's pattern into the compiled subset, or null. */ +function parseSubset(pattern: string, flags: string): { root: RxNode; ncap: number } | null { + for (const f of flags) if (!"gimsuy".includes(f)) return null; + if (pattern.length > 2000) return null; + const parser = new RxParser( + pattern, + flags.includes("u"), + flags.includes("i"), + flags.includes("s"), + flags.includes("m"), + ); + try { + return { root: parser.parse(), ncap: parser.caps }; + } catch (e) { + if (e instanceof RegexBail) return null; + throw e; + } +} + +/** Whether a literal compiles to a native matcher (tests and size caps). */ +export function regexLiteralCompiles(pattern: string, flags: string): boolean { + return parseSubset(pattern, flags) !== null; +} + +/** Why a literal keeps the interpreter ("" when it compiles). */ +export function regexLiteralBailReason(pattern: string, flags: string): string { + try { + new RxParser( + pattern, + flags.includes("u"), + flags.includes("i"), + flags.includes("s"), + flags.includes("m"), + ).parse(); + return ""; + } catch (e) { + if (e instanceof RegexBail) return e.reason; + throw e; + } +} + +/** The native matcher `@sym` of a regex literal, or null when the pattern + * is outside the compiled subset. */ +export function compileRegexMatcher( + sym: string, + pattern: string, + flags: string, + sizeType: string, + vector: boolean, +): RegexMatcherCode | null { + const parsed = parseSubset(pattern, flags); + if (parsed === null) return null; + const sticky = flags.includes("y"); + const { root, ncap } = parsed; + const gen = new RxGen(sym, ncap, vector); + const end: FirstInfo = { set: emptySet(), nullable: true, bol: false, uncond: true }; + const start = gen.emit(root, "match", end); + const first = thenFollow(firstOf(root), end); + + const fn: string[] = []; + fn.push(`define internal i32 @${sym}(ptr %s, i32 %len, i32 %start, ptr %out) #0 {`); + fn.push(`entry:`); + fn.push(` %pos = alloca i32`); + fn.push(` %at = alloca i32`); + fn.push(` %sp = alloca i32`); + fn.push(` %scap = alloca i32`); + fn.push(` %stk = alloca ptr`); + fn.push(` %sbuf = alloca [64 x i32]`); + for (let r = 0; r < gen.regs; r++) fn.push(` %r${r} = alloca i32`); + fn.push(` store ptr %sbuf, ptr %stk`); + fn.push(` store i32 64, ptr %scap`); + fn.push(` store i32 %start, ptr %at`); + fn.push(` %query = icmp slt i32 %len, 0`); + fn.push(` br i1 %query, label %count, label %range`); + fn.push(`count:`); + fn.push(` ret i32 ${ncap}`); + fn.push(`range:`); + fn.push(` %past = icmp sgt i32 %start, %len`); + fn.push(` br i1 %past, label %nomatch, label %scan`); + + // scan: the next position at or after %at where a match can begin. + fn.push(`scan:`); + fn.push(` %a = load i32, ptr %at`); + const firstCount = setCount(first.set); + if (sticky || first.nullable) { + fn.push(` br label %attempt`); + } else if (first.bol && firstCount === 0) { + fn.push(` %a0 = icmp eq i32 %a, 0`); + fn.push(` br i1 %a0, label %attempt, label %nomatch`); + } else { + if (first.bol) { + fn.push(` %a0 = icmp eq i32 %a, 0`); + fn.push(` br i1 %a0, label %attempt, label %seek`); + } else { + fn.push(` br label %seek`); + } + if (firstCount === 1) { + fn.push(`seek:`); + const byte = intervals(first.set)[0]![0]!; + fn.push(` %rest = sub i32 %len, %a`); + if (sizeType === "i32") fn.push(` %rest.w = add i32 %rest, 0`); + else fn.push(` %rest.w = zext i32 %rest to ${sizeType}`); + fn.push(` %a.w = sext i32 %a to i64`); + fn.push(` %from = getelementptr inbounds i8, ptr %s, i64 %a.w`); + fn.push(` %hit = call ptr @memchr(ptr %from, i32 ${byte}, ${sizeType} %rest.w)`); + fn.push(` %none = icmp eq ptr %hit, null`); + fn.push(` br i1 %none, label %nomatch, label %found`); + fn.push(`found:`); + fn.push(` %hit.i = ptrtoint ptr %hit to i64`); + fn.push(` %s.i = ptrtoint ptr %s to i64`); + fn.push(` %off = sub i64 %hit.i, %s.i`); + fn.push(` %off.n = trunc i64 %off to i32`); + fn.push(` store i32 %off.n, ptr %at`); + fn.push(` br label %attempt`); + } else { + gen.emitSeek(first.set); + } + } + + fn.push(`attempt:`); + fn.push(` %b = load i32, ptr %at`); + fn.push(` store i32 %b, ptr %pos`); + fn.push(` store i32 0, ptr %sp`); + for (let r = 2; r < 2 * ncap; r++) fn.push(` store i32 -1, ptr %r${r}`); + fn.push(` br label %${start}`); + + fn.push(`attempt_fail:`); + if (sticky) { + fn.push(` br label %nomatch`); + } else { + fn.push(` %c = load i32, ptr %at`); + fn.push(` %c1 = add i32 %c, 1`); + fn.push(` store i32 %c1, ptr %at`); + fn.push(` %over = icmp sgt i32 %c1, %len`); + fn.push(` br i1 %over, label %nomatch, label %scan`); + } + + fn.push(`match:`); + fn.push(` %m.end = load i32, ptr %pos`); + fn.push(` %m.start = load i32, ptr %at`); + for (let slot = 0; slot < 2 * ncap; slot++) { + const v = slot === 0 ? "%m.start" : slot === 1 ? "%m.end" : `%m.v${slot}`; + if (slot >= 2) fn.push(` ${v} = load i32, ptr %r${slot}`); + fn.push(` %m.w${slot} = sext i32 ${v} to i64`); + fn.push(` %m.p${slot} = getelementptr inbounds i8, ptr %s, i64 %m.w${slot}`); + let value = `%m.p${slot}`; + if (slot >= 2) { + fn.push(` %m.u${slot} = icmp slt i32 ${v}, 0`); + fn.push(` %m.q${slot} = select i1 %m.u${slot}, ptr null, ptr %m.p${slot}`); + value = `%m.q${slot}`; + } + fn.push(` %m.o${slot} = getelementptr inbounds ptr, ptr %out, i64 ${slot}`); + fn.push(` store ptr ${value}, ptr %m.o${slot}`); + } + fn.push(` %m.stk = load ptr, ptr %stk`); + fn.push(` %m.heap = icmp ne ptr %m.stk, %sbuf`); + fn.push(` br i1 %m.heap, label %m.free, label %m.ret`); + fn.push(`m.free:`); + fn.push(` call void @scr_regex_stack_free(ptr %m.stk)`); + fn.push(` br label %m.ret`); + fn.push(`m.ret:`); + fn.push(` ret i32 1`); + + fn.push(`nomatch:`); + fn.push(` %n.stk = load ptr, ptr %stk`); + fn.push(` %n.heap = icmp ne ptr %n.stk, %sbuf`); + fn.push(` br i1 %n.heap, label %n.free, label %n.ret`); + fn.push(`n.free:`); + fn.push(` call void @scr_regex_stack_free(ptr %n.stk)`); + fn.push(` br label %n.ret`); + fn.push(`n.ret:`); + fn.push(` ret i32 0`); + + gen.emitTail(); + for (const l of gen.body) fn.push(l); + fn.push(`}`); + + const decls = REGEX_MATCHER_DECLS.slice(); + if (gen.body.some((l) => l.includes("@llvm.cttz.i16"))) + decls.push(`declare i16 @llvm.cttz.i16(i16, i1 immarg)`); + if (!sticky && !first.nullable && firstCount === 1) + decls.push(`declare ptr @memchr(ptr, i32, ${sizeType})`); + return { lines: [...gen.globals, ...fn], decls }; +} diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 451e83951..131f1ae20 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 16 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v16" as const; +export const RUNTIME_ABI_VERSION = 17 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v17" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 6384dd12b..f35905e76 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 952f4d303..f75d46c48 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -116,7 +116,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 33bc50b80..56fd2c8a4 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -312,7 +312,7 @@ ${[0, 1, 2, 3, 4, 5].map((index) => ` call void @add${index}(i64 ${index + 2})` expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v16"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v17"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -646,7 +646,7 @@ ${[0, 1, 2, 3, 4, 5].map((index) => ` call void @add${index}(i64 ${index + 2})` .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v16"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v17"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4950-regex-compiled-captures.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4950-regex-compiled-captures.ts.json new file mode 100644 index 000000000..413617244 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4950-regex-compiled-captures.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4950-regex-compiled-captures.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4951-regex-compiled-last-index.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4951-regex-compiled-last-index.ts.json new file mode 100644 index 000000000..d11819cd2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4951-regex-compiled-last-index.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4951-regex-compiled-last-index.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4952-regex-compiled-anchors-classes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4952-regex-compiled-anchors-classes.ts.json new file mode 100644 index 000000000..faeec80da --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4952-regex-compiled-anchors-classes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4952-regex-compiled-anchors-classes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4953-regex-compiled-backtracking.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4953-regex-compiled-backtracking.ts.json new file mode 100644 index 000000000..5fc4ebf20 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4953-regex-compiled-backtracking.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4953-regex-compiled-backtracking.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4954-regex-log-report.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4954-regex-log-report.ts.json new file mode 100644 index 000000000..b4a5fa745 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4954-regex-log-report.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4954-regex-log-report.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4955-regex-compiled-differential.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4955-regex-compiled-differential.ts.json new file mode 100644 index 000000000..dff438d0a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4955-regex-compiled-differential.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4955-regex-compiled-differential.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index e857063a6..43cc9438a 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -306,7 +306,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 0c50b6b10..5f5364aff 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -402,7 +402,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 07a64cdd1..035c42318 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -126,7 +126,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v16(void) {} +void scr_runtime_abi_v17(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index a4b4138bd..25bd99ed3 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -221,10 +221,37 @@ static uint8_t *scr_regex_bc(ScrRegex *re) { } re->bc = bc; re->native = scr_regex_native_build(re, bc); + /* A build-time matcher must agree with the bytecode's capture count. */ + if (re->matcher && re->matcher(NULL, -1, 0, NULL) != lre_get_capture_count(bc)) + re->matcher = NULL; scr_note_compiled(re); return bc; } +void scr_regex_stack_grow(int32_t **stack, int32_t *cap, int32_t *inline_buf, int32_t need) { + size_t ncap = (size_t)*cap * 2; + if (ncap < (size_t)need) ncap = (size_t)need; + if (ncap > (size_t)INT32_MAX) scr_regex_oom(); + int32_t *grown; + if (*stack == inline_buf) { + grown = malloc(ncap * sizeof *grown); + if (grown) memcpy(grown, *stack, (size_t)*cap * sizeof *grown); + } else { + grown = realloc(*stack, ncap * sizeof *grown); + } + if (!grown) scr_regex_oom(); + *stack = grown; + *cap = (int32_t)ncap; +} + +void scr_regex_stack_free(int32_t *stack) { free(stack); } + +/* The build-time matcher of a regex (literal instances use their + * template's). */ +static inline ScrRegexMatcherFn scr_regex_matcher(const ScrRegex *re) { + return re->literal ? re->literal->matcher : re->matcher; +} + /* ── RC entry points ────────────────────────────────────────────────── * Literal instances borrow bytecode from their immortal templates; * constructed regexes own their compiled bytecode. */ @@ -418,6 +445,7 @@ typedef struct ScrRegexInput { const uint8_t *data; uint16_t *utf16; const ScrReNative *native; /* one-byte subjects of translated patterns */ + ScrRegexMatcherFn matcher; /* one-byte subjects of build-time matchers */ int len; int wide; bool borrowed_utf16; /* utf16 belongs to the subject cache below */ @@ -451,6 +479,7 @@ static void scr_regex_input_init(ScrRegexInput *input, const ScrRegex *re, if (s->len > (size_t)INT32_MAX - 1) scr_regex_oom(); input->wide = !scr_text_ascii(s); input->native = input->wide ? NULL : (const ScrReNative *)re->native; + input->matcher = input->wide ? NULL : scr_regex_matcher(re); input->utf16 = input->wide ? scr_to_utf16(s, &input->len) : NULL; input->borrowed_utf16 = false; input->data = input->wide ? (const uint8_t *)input->utf16 : (const uint8_t *)s->data; @@ -507,6 +536,7 @@ static void scr_regex_input_init_cached(ScrRegexInput *input, const ScrRegex *re } input->wide = scr_subject_wide; input->native = input->wide ? NULL : (const ScrReNative *)re->native; + input->matcher = input->wide ? NULL : scr_regex_matcher(re); input->utf16 = scr_subject_utf16; input->borrowed_utf16 = true; input->data = input->wide ? (const uint8_t *)input->utf16 : (const uint8_t *)s->data; @@ -519,7 +549,10 @@ static void scr_regex_input_dispose(ScrRegexInput *input) { if (!input->borrowed_utf16) free(input->utf16); } -static int scr_exec(ScrRegexInput *input, const uint8_t *bc, int index) { +/* The bytecode engines; kept out of line so a build-time matcher call does + * not pay their frame. */ +__attribute__((noinline)) static int scr_exec_engine(ScrRegexInput *input, const uint8_t *bc, + int index) { int rc = input->native ? scr_re_native_exec(input->native, input->capture, input->data, index, input->len) : lre_exec(input->capture, bc, input->data, index, input->len, input->wide, lre_opaque()); @@ -530,6 +563,12 @@ static int scr_exec(ScrRegexInput *input, const uint8_t *bc, int index) { return rc; } +/* Build-time matchers trap on allocation failure themselves. */ +static inline int scr_exec(ScrRegexInput *input, const uint8_t *bc, int index) { + if (input->matcher) return input->matcher(input->data, input->len, index, input->capture); + return scr_exec_engine(input, bc, index); +} + static int scr_capture_index(const ScrRegexInput *input, const uint8_t *capture) { return (int)((capture - input->data) >> input->wide); } diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 828e9519e..1ba4c234f 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -74,7 +74,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v16(void); +void scr_runtime_abi_v17(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── @@ -1903,6 +1903,9 @@ ScrStr *scr_arr_join(ScrArr *a, ScrStr *sep); * missing ones skip. Borrows both; +1 result; never throws. */ ScrStr *scr_str_raw(ScrArr *raw, ScrArr *subs); +typedef int32_t (*ScrRegexMatcherFn)(const uint8_t *s, int32_t len, int32_t start, + uint8_t **capture); + typedef struct ScrRegex { size_t rc; /* SIZE_MAX = immortal literal template */ ScrStr *source; /* pattern text between the slashes */ @@ -1911,8 +1914,18 @@ typedef struct ScrRegex { double last_index; struct ScrRegex *literal; /* immutable bytecode owner for literal instances */ void *native; /* one-byte-subject matcher built with bc (NULL: lre_exec only) */ + /* Build-time matcher of a literal template (NULL: none). Searches the + * ASCII bytes s[0..len) from start (one anchored attempt when sticky); + * on a match stores 2 * ncap pointers into s (NULL when unset) and + * returns 1, otherwise 0. A negative len returns ncap. */ + ScrRegexMatcherFn matcher; } ScrRegex; +/* Backtrack stack of build-time regex matchers: grow *stack (initially the + * matcher's inline buffer) to at least `need` words; traps on OOM. */ +void scr_regex_stack_grow(int32_t **stack, int32_t *cap, int32_t *inline_buf, int32_t need); +void scr_regex_stack_free(int32_t *stack); + static inline ScrRegex *scr_regex_retain(ScrRegex *re) { if (re->rc != SIZE_MAX) re->rc++; return re; diff --git a/packages/runtime/test/test_regex.c b/packages/runtime/test/test_regex.c index 49078c723..a0b069619 100644 --- a/packages/runtime/test/test_regex.c +++ b/packages/runtime/test/test_regex.c @@ -346,6 +346,60 @@ static void test_exec_groups(void) { scr_str_release(s); } +/* A build-time matcher stand-in for /(b)/: the first 'b'. Counts its calls, + * and grows the backtrack stack once to exercise the runtime helper. */ +static int fake_calls = 0; +static int32_t fake_b_matcher(const uint8_t *s, int32_t len, int32_t start, uint8_t **capture) { + if (len < 0) return 2; + fake_calls++; + int32_t inline_buf[4]; + int32_t *stack = inline_buf, cap = 4; + for (int32_t i = 0; i < 4; i++) inline_buf[i] = i; + scr_regex_stack_grow(&stack, &cap, inline_buf, 9); + bool kept = stack != inline_buf && cap >= 9 && stack[3] == 3; + scr_regex_stack_free(stack); + if (!kept) return 0; + for (int32_t i = start; i < len; i++) { + if (s[i] == 'b') { + capture[0] = capture[2] = (uint8_t *)s + i; + capture[1] = capture[3] = (uint8_t *)s + i + 1; + return 1; + } + } + return 0; +} +static int32_t fake_wrong_count(const uint8_t *s, int32_t len, int32_t start, uint8_t **capture) { + (void)s; + (void)start; + (void)capture; + if (len < 0) return 7; + fake_calls++; + return 0; +} +RE(re_matched, "(b)", ""); +RE(re_mismatched, "(b)", ""); + +static void test_build_time_matcher(void) { + re_matched.matcher = fake_b_matcher; + re_mismatched.matcher = fake_wrong_count; + ScrStr *ascii = S("abc"); + ScrStr *wide = S("\xC3\xA9" "b"); + ScrArr *m = scr_regex_exec(ascii, &re_matched); + ScrStr *g = m ? scr_arr_get_ref(m, 1) : NULL; + check(fake_calls == 1 && g && str_is(g, "b"), "matcher: ASCII subjects use the build-time matcher"); + scr_str_release(g); + scr_arr_release(m); + check(scr_regex_test(&re_matched, wide) && fake_calls == 1, + "matcher: non-ASCII subjects keep the bytecode engine"); + ScrRegex *instance = scr_regex_literal(&re_matched); + check(scr_regex_test(instance, ascii) && fake_calls == 2, "matcher: literal instances use the template's"); + scr_regex_release(instance); + check(scr_regex_test(&re_mismatched, ascii) && fake_calls == 2 && re_mismatched.matcher == NULL, + "matcher: a capture-count mismatch falls back to the bytecode engine"); + scr_str_release(ascii); + scr_str_release(wide); +} + static void test_string_replace(void) { /* String search values: one pass, GetSubstitution without captures. */ ScrStr *s = S("a.b.c"); @@ -388,6 +442,7 @@ int main(int argc, char **argv) { test_capture_capacity(); test_exec_groups(); test_string_replace(); + test_build_time_matcher(); #ifdef SCR_RC_AUDIT check(scr_str_live_count() == 0, "no live heap strings at exit (RC audit)"); diff --git a/tests/corpus/4950-regex-compiled-captures.ts b/tests/corpus/4950-regex-compiled-captures.ts new file mode 100644 index 000000000..4f7e73acb --- /dev/null +++ b/tests/corpus/4950-regex-compiled-captures.ts @@ -0,0 +1,76 @@ +// Regex literals that compile to native matchers: capture groups that do +// not participate stay undefined, quantified groups reset their captures on +// every iteration, alternation is ordered, and optional iterations that +// match nothing stop the loop. +export {}; + +function show(m: RegExpExecArray | null): string { + if (m === null) return "null"; + const parts: string[] = []; + for (let i = 0; i < m.length; i++) parts.push(m[i] === undefined ? "undefined" : JSON.stringify(m[i])); + return `${m.index}: ${parts.join(", ")}`; +} + +// Nonparticipating captures. +console.log(show(/a(x)?c/.exec("ac"))); +console.log(show(/a(x)?c/.exec("axc"))); +console.log(show(/(a)|(b)/.exec("b"))); +console.log(show(/(?:(a)|b)(c)?/.exec("bd"))); +console.log(show(/^(\w+)(?:=(\w*))?$/.exec("key"))); +console.log(show(/^(\w+)(?:=(\w*))?$/.exec("key="))); +console.log(show(/^(\w+)(?:=(\w*))?$/.exec("key=value"))); + +// Capture reset on each iteration (RepeatMatcher step 4). +console.log(show(/(z)((a+)?(b+)?(c))*/.exec("zaacbbbcac"))); +console.log(show(/(?:(a)|b)+/.exec("ab"))); +console.log(show(/(?:(a)|(b))+/.exec("aba"))); +console.log(show(/(?:(\d)|([a-z]))*!/.exec("1a2b!"))); +console.log(show(/((a)|(b))*c/.exec("abac"))); +console.log(show(/(?:x(\d)?)+/.exec("x1xx"))); + +// Ordered alternation and backtracking into groups. +console.log(show(/(a|ab)(c|bcd)(d*)/.exec("abcd"))); +console.log(show(/(?:ab|a)(bc|c)/.exec("abc"))); +console.log(show(/^(?:a|ab)+$/.exec("abab"))); +console.log(show(/(foo|foobar)bar/.exec("foobarbar"))); + +// Greedy and lazy quantifiers, counted forms. +console.log(show(/<(.+)>/.exec(""))); +console.log(show(/<(.+?)>/.exec(""))); +console.log(show(/(a{2,3})(a*)/.exec("aaaaa"))); +console.log(show(/(a{2,3}?)(a*)/.exec("aaaaa"))); +console.log(show(/(\d{3})-(\d{2,})-?(\d{0,2})/.exec("123-4567-89"))); +console.log(show(/(?:ab){2}(c)?/.exec("ababc"))); +console.log(show(/(?:ab){2,}?(ab)*/.exec("abababab"))); +console.log(show(/(x|y){1,2}?z/.exec("xyz"))); +console.log(show(/(\w+?)(\d*)$/.exec("abc123"))); +console.log(show(/a.*?b.*c/.exec("aXbYbZc"))); +console.log(show(/(.*)(\d+)/.exec("abc12345"))); +console.log(show(/(.*?)(\d+)/.exec("abc12345"))); + +// Empty iterations. +console.log(show(/(a*)*b/.exec("aab"))); +console.log(show(/(a*)+b/.exec("b"))); +console.log(show(/(a*?)*b/.exec("aab"))); +console.log(show(/(?:a?)*?x/.exec("aax"))); +console.log(show(/(a|)+b/.exec("aab"))); +console.log(show(/(?:()|a)+$/.exec("aa"))); +console.log(show(/(a?){2,}c/.exec("ac"))); +console.log(show(/(?:(^)|a)+/.exec("aa"))); +console.log(show(/(\b)+a/.exec("a"))); + +// Named groups compile too; their values come from the same captures. +const date = /(?\d{4})-(?\d\d)(?:-(?\d\d))?/; +for (const s of ["2026-10-13", "on 2026-10", "none"]) { + const m = date.exec(s); + if (m === null) console.log("null"); + else console.log(show(m), m.groups!.y, m.groups!.d); +} + +// Captures read through match and replace templates. +console.log(JSON.stringify("k1=v1;k2=;k3".match(/(\w+)=(\w*)/))); +console.log("k1=v1;k2=;k3".replace(/(\w+)(?:=(\w*))?/g, "[$1|$2]")); +console.log("aaa bbb".replace(/(a)|(b)/g, "<$1$2>")); +console.log("2026-10-13".replace(/(\d+)-(\d+)-(\d+)/, "$3/$2/$1 ($&) $$ $`$'")); +for (const m of "x=1, y=, z=33".matchAll(/(\w)=(\d+)?/g)) console.log(m.index, m[0], m[1], m[2]); +console.log(JSON.stringify("a1b22c333d".split(/(\d)(\d)?/))); diff --git a/tests/corpus/4951-regex-compiled-last-index.ts b/tests/corpus/4951-regex-compiled-last-index.ts new file mode 100644 index 000000000..a6b53c6ea --- /dev/null +++ b/tests/corpus/4951-regex-compiled-last-index.ts @@ -0,0 +1,72 @@ +// Native regex matchers under the global and sticky lastIndex protocol: +// exec/test loops, empty matches that advance by one code unit (or one code +// point under the u flag), sticky anchoring, and subjects that switch +// between ASCII and non-ASCII text on the same regex. +export {}; + +const word = /[a-z]+/g; +const text = "alpha beta gamma"; +for (let i = 0; i < 5; i++) { + const m = word.exec(text); + console.log(m === null ? "null" : `${m[0]}@${m.index}`, word.lastIndex); +} + +const digits = /\d+/g; +let count = 0; +while (digits.test("a1b22c333")) count++; +console.log("test loop", count, digits.lastIndex); +digits.lastIndex = 4; +console.log(digits.exec("a1b22c333")?.[0], digits.lastIndex); +digits.lastIndex = 99; +console.log(digits.exec("a1b22c333"), digits.lastIndex); + +// Sticky: one anchored attempt at lastIndex. +const sticky = /\d+|[a-z]/y; +const parts: string[] = []; +for (let i = 0; i < 8; i++) { + const m = sticky.exec("12ab3-4"); + parts.push(m === null ? `null@${sticky.lastIndex}` : `${m[0]}@${m.index}`); +} +console.log(parts.join(" ")); +const stickyG = /a|b/gy; +console.log(JSON.stringify("aabxab".match(stickyG)), "aabxab".replace(stickyG, "_")); +console.log(/b/y.test("ab"), /^b/my.test("a\nb"), JSON.stringify("ab".split(/b/y))); + +// Empty matches advance past the current position. +console.log(JSON.stringify("abc".match(/x*/g))); +console.log("abc".replace(/x*/g, "-")); +console.log("abc".replace(/(?:)/g, ".")); +console.log(JSON.stringify("a,b,,c".split(/,*/))); +console.log(JSON.stringify("ab12cd".match(/\d*/g))); +const empties: string[] = []; +for (const m of "aXbX".matchAll(/X?/g)) empties.push(`${m[0]}@${m.index}`); +console.log(JSON.stringify(empties)); + +// Non-ASCII subjects run on the interpreter; ASCII ones on the native +// matcher. lastIndex is a UTF-16 index either way. +const mixed = /[a-z]+/g; +for (const s of ["hé llo", "hello", "😀ab😀cd"]) { + mixed.lastIndex = 0; + const seen: string[] = []; + for (let m = mixed.exec(s); m !== null; m = mixed.exec(s)) seen.push(`${m[0]}@${m.index}/${mixed.lastIndex}`); + console.log(JSON.stringify(s), seen.join(" ")); +} +// Empty-match advancement: code units without u, code points with u. +console.log(JSON.stringify("😀a".match(/(?:)/g)), JSON.stringify("😀a".match(/(?:)/gu))); +console.log("😀a".replace(/x*/g, "-") === "-\ud83d-\ude00-a-", "😀a".replace(/x*/gu, "-")); +console.log(JSON.stringify("a😀b".split(/(?:)/u)), "a😀b".split(/(?:)/).length); +const eu = /\b|$/gu; +const idx: number[] = []; +for (let m = eu.exec("ab 😀 c"); m !== null && idx.length < 10; m = eu.exec("ab 😀 c")) { + idx.push(m.index); + if (m[0] === "") eu.lastIndex = eu.lastIndex + 1; +} +console.log(idx.join(",")); + +// search ignores lastIndex and leaves it alone. +const g = /c/g; +g.lastIndex = 3; +console.log("abcabc".search(g), g.lastIndex, "abc".search(/x/), "abc".search(/^/)); + +// Literal sites in a loop reset their own state each evaluation. +for (let i = 0; i < 3; i++) console.log(/o/g.exec("foo boo")?.[0], /o/g.test("xo"), "foo".search(/o/g)); diff --git a/tests/corpus/4952-regex-compiled-anchors-classes.ts b/tests/corpus/4952-regex-compiled-anchors-classes.ts new file mode 100644 index 000000000..fd8249daa --- /dev/null +++ b/tests/corpus/4952-regex-compiled-anchors-classes.ts @@ -0,0 +1,41 @@ +// Native regex matchers for anchors (with and without m), word boundaries, +// character classes and their escapes, the dot with and without s, and +// ASCII case folding under i (with and without u). +export {}; + +function all(re: RegExp, s: string): string { + return JSON.stringify(s.match(re)); +} + +// $ only matches at the very end without m (also before a final newline: +// no), and at each line end with m. ^ likewise. +console.log(/a$/.test("a\n"), /a$/m.test("a\n"), /a$/m.test("a\r\nb"), /^b/m.test("a\rb")); +console.log(all(/^\w+$/gm, "one\ntwo\r\nthree\rfour\n"), all(/\w+$/g, "one\ntwo")); +console.log(all(/^/gm, "a\n\nb"), all(/$/gm, "a\n\nb"), "a\nb\n".replace(/$/gm, ";")); +console.log(all(/^\d|\d$/g, "12 34"), /^$/.test(""), /^$/m.test("a\n"), /x^/.test("x")); + +// Word boundaries. +console.log(all(/\b\w/g, "hi there_you-2b"), all(/\B\w/g, "ab cd"), "a b".replace(/\b/g, "|")); +console.log(/\bfoo\b/.test("a foo."), /\bfoo\b/.test("afoo"), all(/\B/g, "ab"), all(/\b/g, "")); + +// Classes, negation, ranges and escapes. +console.log(all(/[a-c]+/g, "abcdcba"), all(/[^a-c]+/g, "abcdcba"), all(/[^"]*"/g, 'x"y""')); +console.log(all(/[\d.]+/g, "v1.25 and 3."), all(/[\w-]+/g, "a-b c_d"), all(/[-a]/g, "-ba-")); +console.log(all(/[a\-z]/g, "a-mz"), all(/[\]\\]/g, "a]b\\c"), all(/[.]/g, "a.b"), all(/[^]/g, "ab")); +console.log(all(/[\b]/g, "a\bb"), all(/\x41B\t/g, "AB\tAB"), all(/\0/g, "a\0b"), /[]/.test("a")); +console.log(all(/\s+/g, "a \t\n\v\f\rb"), all(/\S+/g, "a b"), all(/\D+/g, "a1b"), all(/\W/g, "a-b c")); +console.log(all(/\/\*.*?\*\//g, "a /* x */ b /**/ c"), all(/\$\{(\w+)\}/g, "${a}-${bc}")); + +// The dot. +console.log(all(/a.b/g, "a\nb a\rb axb"), all(/a.b/gs, "a\nb a\rb axb"), all(/.+/g, "x\ny")); + +// ASCII case folding. +console.log(all(/abc/gi, "ABC aBc abc"), all(/[a-c]+/gi, "xABcx"), all(/[^a]/gi, "aAb")); +console.log(all(/[^\W_]+/gi, "Ab_9 x"), all(/\w+/gi, "sK ſK"), all(/[k-l]/giu, "KkLl")); +console.log(/\W/i.test("s"), /\W/iu.test("s"), /\W/iu.test("k"), /[\W]/i.test("K")); +console.log(/s/i.test("ſ"), /s/iu.test("ſ"), /k/iu.test("K"), /[a-z]/i.test("K")); +console.log(all(/GET|post/gi, "get POST Get pOsT"), "Hello World".replace(/o/gi, "0")); + +// Patterns outside the compiled subset still run (on the interpreter). +console.log(all(/(\w)\1/g, "aabbcd"), all(/a(?=b)/g, "abac"), all(/(?<=\$)\d+/g, "$5 $66")); +console.log(all(/\p{L}+/gu, "héllo wörld"), all(/[à-ÿ]+/g, "àé ok")); diff --git a/tests/corpus/4953-regex-compiled-backtracking.ts b/tests/corpus/4953-regex-compiled-backtracking.ts new file mode 100644 index 000000000..96e3d5bdf --- /dev/null +++ b/tests/corpus/4953-regex-compiled-backtracking.ts @@ -0,0 +1,41 @@ +// Native regex matchers under heavy backtracking: nested quantifiers on +// failing inputs, long give-back runs, and backtrack stacks that grow far +// past their inline size. Results match the interpreter and Node. +export {}; + +function show(m: RegExpExecArray | null): string { + if (m === null) return "null"; + const parts: string[] = []; + for (let i = 0; i < m.length; i++) parts.push(m[i] === undefined ? "undefined" : `${m[i]!.length}`); + return `${m.index}: ${parts.join(", ")}`; +} + +// Exponential in the input length; small enough to finish quickly. +for (const n of [4, 10, 16, 18]) { + const s = "a".repeat(n); + console.log(n, /(a+)+b/.test(s), /(a|aa)+c/.test(s), show(/(a*)*$/.exec(s)), /(?:a?){12}a{12}/.test(s)); +} +console.log(show(/^(\w+\s?)*$/.exec("word word word word word!")), show(/^(\w+\s?)*$/.exec("word word word"))); +console.log(show(/(x+x+)+y/.exec("xxxxxxxxxxxxxx")), show(/(x+x+)+y/.exec("xxxxxy"))); + +// Long give-back and lazy extension. +const long = "a".repeat(5000) + "b" + "a".repeat(5000); +console.log(show(/(.*)b(.*)/.exec(long)), show(/(.*?)b/.exec(long)), show(/a*?b/.exec(long))); +console.log(show(/(a*)a{4000}b/.exec(long)), show(/^(?:a|b)*$/.exec(long)), /^(?:a|b)*c/.test(long)); +console.log(show(/((?:ab)*?)c/.exec("ab".repeat(3000) + "c")), show(/(?:(a)|b)*$/.exec("ab".repeat(4000)))); +console.log(show(/(\w+)\s(\w+)$/.exec("x ".repeat(2000) + "end here"))); + +// Many alternatives and deep nesting. +const kw = /\b(?:if|else|while|for|return|function|const|let|var|class|new|this)\b/g; +console.log(JSON.stringify("let x = new Foo(); if (x) return this; else while (1) {}".match(kw))); +console.log(show(/((((((((((a))))))))))+/.exec("aaa")), show(/(?:(?:(?:(?:a|b)c)?d)*e)+/.exec("acdbcdedebcde"))); + +// The same regex reused across many subjects. +const ip = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/; +let valid = 0; +for (let i = 0; i < 2000; i++) { + const s = `${i % 300}.${(i * 7) % 260}.${i % 10}.${(i * 13) % 999}`; + const m = ip.exec(s); + if (m !== null && m.slice(1).every((x) => Number(x) < 256)) valid++; +} +console.log("valid", valid); diff --git a/tests/corpus/4954-regex-log-report.ts b/tests/corpus/4954-regex-log-report.ts new file mode 100644 index 000000000..bafe15919 --- /dev/null +++ b/tests/corpus/4954-regex-log-report.ts @@ -0,0 +1,88 @@ +// A log report built with regex literals: access-log lines with optional +// query strings, key=value application logs, quoted CSV fields, ISO +// timestamps, and template placeholders. +export {}; + +let seed = 7; +function next(): number { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed / 4294967296; +} +function pick(items: T[]): T { + return items[Math.floor(next() * items.length)]!; +} + +const access = /^(\S+) \S+ (\S+) \[([^\]]+)\] "([A-Z]+) ([^ "?]+)(?:\?([^ "]*))? HTTP\/(1\.[01]|2)" (\d{3}) (\d+|-)(?: "([^"]*)" "([^"]*)")?$/; +const app = /^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)(?:\.(\d{1,3}))?Z\s+(DEBUG|INFO|WARN|ERROR)\s+\[([\w.-]+)\]\s+(.*)$/; +const pair = /(\w+)=("(?:[^"\\]|\\.)*"|\S*)/g; +const csvField = /(?:^|,)(?:"((?:[^"]|"")*)"|([^,]*))/g; +const placeholder = /\{\{\s*(\w+)(?:\s*\|\s*(\w+))?\s*\}\}/g; +const userAgentBot = /bot|crawler|spider|curl/i; + +const lines: string[] = []; +for (let i = 0; i < 400; i++) { + const ip = `192.168.${Math.floor(next() * 4)}.${Math.floor(next() * 255)}`; + const user = next() < 0.8 ? "-" : pick(["ana", "bo", "cy"]); + const method = pick(["GET", "POST", "PUT", "DELETE", "PATCH"]); + const path = pick(["/", "/api/v1/items", "/login", "/static/main.css", "/search"]); + const query = next() < 0.4 ? `?q=${Math.floor(next() * 100)}&page=${Math.floor(next() * 5)}` : ""; + const status = pick([200, 200, 200, 201, 301, 404, 500]); + const size = next() < 0.1 ? "-" : String(Math.floor(next() * 50000)); + const tail = next() < 0.7 ? ` "https://ref.example/${i}" "${pick(["Mozilla/5.0", "Googlebot/2.1", "curl/8.0", "Spider-X"])}"` : ""; + lines.push(`${ip} - ${user} [13/Oct/2026:10:${String(i % 60).padStart(2, "0")}:00 +0000] "${method} ${path}${query} HTTP/${pick(["1.0", "1.1", "2"])}" ${status} ${size}${tail}`); +} +lines.push("garbage line"); +lines.push('1.2.3.4 - - [x] "GET /bad path HTTP/1.1" 200 1'); + +const statusCounts = new Map(); +let bytes = 0; +let bots = 0; +let queries = 0; +const users = new Set(); +for (const line of lines) { + const m = access.exec(line); + if (m === null) { + console.log("unparsed:", line); + continue; + } + statusCounts.set(m[8]!, (statusCounts.get(m[8]!) ?? 0) + 1); + if (m[9] !== "-") bytes += Number(m[9]); + if (m[6] !== undefined) queries += m[6].split("&").length; + if (m[11] !== undefined && userAgentBot.test(m[11])) bots++; + if (m[2] !== "-") users.add(m[2]!); +} +const statusKeys = [...statusCounts.keys()].sort(); +console.log(statusKeys.map((k) => `${k}:${statusCounts.get(k)}`).join(" "), bytes, bots, queries, [...users].sort().join(",")); + +const appLines = [ + '2026-10-13T08:00:01.123Z INFO [api.server] started port=8080 mode="production" workers=4', + "2026-10-13T08:00:02Z WARN [db-pool] slow query ms=812 table=orders", + '2026-10-13T08:00:03.5Z ERROR [api.handler] failed path=/x err="timeout \\"upstream\\"" retry=', + "2026-10-13 08:00:04 INFO [bad] no T separator", + "2026-10-13T08:00:05.999Z DEBUG [cache] hit key=user:42", +]; +for (const line of appLines) { + const m = app.exec(line); + if (m === null) { + console.log("no match:", line); + continue; + } + const fields: string[] = []; + for (const p of m[10]!.matchAll(pair)) fields.push(`${p[1]}=>${p[2]}`); + console.log(m[8], m[9], m[7] ?? "000", m[4], fields.join("; ")); +} + +const csv = ['a,b,c', '"quoted, comma",plain,"with ""quotes"""', ",,", '"",x,', "single"]; +for (const row of csv) { + const cells: string[] = []; + for (const f of row.matchAll(csvField)) cells.push(f[1] !== undefined ? f[1].replace(/""/g, '"') : f[2]!); + console.log(cells.length, JSON.stringify(cells)); +} + +const template = "Hello {{ name }}, you have {{count|number}} new {{ things | plural }}. {{missing}} {{ bad"; +const values = new Map([["name", "Ana"], ["count", "3"], ["things", "message"]]); +console.log(template.replace(placeholder, (_all, key: string, filter: string | undefined) => { + const v = values.get(key) ?? `<${key}>`; + return filter === "plural" ? v + "s" : filter === "number" ? `#${v}` : v; +})); +console.log(lines.slice(0, 50).join("\n").replace(/\d+/g, "#").length, lines[3]!.replace(/\b\d{1,3}(\.\d{1,3}){3}\b/, "IP")); diff --git a/tests/corpus/4955-regex-compiled-differential.ts b/tests/corpus/4955-regex-compiled-differential.ts new file mode 100644 index 000000000..f5332e134 --- /dev/null +++ b/tests/corpus/4955-regex-compiled-differential.ts @@ -0,0 +1,123 @@ +// Differential cases for regex literals compiled to native matchers: +// generated patterns over a small alphabet (classes, escapes, anchors, +// greedy and lazy quantifiers, alternation, capturing, non-capturing and +// named groups, all flag combinations) run through exec, test, replace, +// search, split and match on ASCII and non-ASCII subjects. +export {}; + +const inputs: string[] = ["bbbbbbbbbbbbbbbbbbbbb", "\naaaaaaaaaaaaaaaaaaaaaa11111111111111112SSSSSSSSSSSSSSSSSS_ .", " ", "\r\r\r\r\r\r", "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", "A-", " A ", "Bb\rKKKKKaaaaa_\"\n1", "A\nx\rsKxxxxxx\"\nSb K11111111111KKKKKKKKKKKKKKKcs\r-sc", " . xc-----------\nss\"AbaaaaaaaaaaaaaaaaaaaaaaBkBk__Kaaaaaaaaaaaaaa", "ssssssssssssssssssssssss", "xa\n\r2k\n", "-s--------------------\n\n\na-SAb.KSSSSSSSSSS2\n.B1kcccccccxSKB\"\"x", "\r\r\r\r\r\r\rb_\r-KssssssssssssssB"]; +const patterns: RegExp[] = [ + /\B\-/gm, + /([a-ca-z0-9]?([^___]+[^\W\]](_^)$)\-)([\n])?? /gy, + /(-?(?A*?\db*))??/g, + /[a-c]x+?"?/, + /((?[a] |(?(ba)x))*?-+?\-(?\sA+(\.|\D1*){0,1}){0,1}|\Ba(?:((?b)\b^[^0-9.\s])(?("??\.)\-*_{0}a?)){1,2}x)(?:(?:\/\s))??(-c{2})((?:\B(?:\B(b*a*? |.b??\D{0,1})-(aa1{2,3}|ab|.{2,2}a)){0,1} +){0,1})/my, + /\b(([b\]](?:"|(a*\W|a[\s1\d]\d*b+){2}\n(?:\s|b+){1,2}|(?:\d{1}\/b+){0,1}\S?|(\Sbb|\w)(?:a{0}?[bx-z\s]\d*?)?(?:b\d{0}a )??)a){1,1}){0,2}?[^ "]^/g, + /(?\S\/*_{1,1}?\b)[\n\sb]\/$/, + /\w/gi, + /(?:^|(?:-*?){0,1}\.*-{1,3}(-??((?:b*\-a\d){0,1}(?:\db\-){0,1}|\s*1\B)?\s{2,2}))_{2,}_[^a-c]|-*?/gsy, + /(\n$)/g, + /\B[\].]+?_b/g, + /[ba-za-z]|\.[\-aa]+(x*[_"b](b{0}[^.b\n]){0,2}?\W+|(?:\.*xb*?|a?\/)-(\/+b(?:(?a{1,1}_\d*)|(ba)))\n+?)/gyu, + /\/\//y, + /^b{2}?((?:^[\sa]??(?:\/""{1,1}))$){1,2}[^A-Zx-z]/gm, + /\-\w+(([A-Z.]{1}(?-([^ 0-90-9]{1,}?b+?)(\Db){2}[.]+)){2}\-{0,1}|\-\D*(\/\W*a(([_]ab{0}x)??){0,1})(_*1{2,3}))/m, + /c[\]]/gi, + /(?:A{1})??/g, + /[ ]*((?:\.b(Ac[^\-\w\w]|(\da\S{1,}_|baa[^ ]|A\w *){0,1}[A-Z""]{0,0}(?\d[A-Z]*\n){2}(b{2,2}?\d"|a)??|.a))\D(?:((?: {0,}?|a){0,2}?(?:b\s["]*\d)){1,2}\/+){0,1}| (.|(?:A\d\/*^)(?:[^.0-9\W]{0,}?(\d\d*)( *b{0,0}|c*?[0-9]+?\D)??(\nx)??)??|^){0,2}?x+){0,1}/my, + /(\s(\B|(?:"*[^\na\s])\/b{1,1}(A?[\n](a+?aa|A_*|b\d*?b|[^a\]]*?||a+){0,2}?))*){0,2}?((?:[x-z](?(\D?[a\]]*|b?[a])\n\s+))){2}[a"a-c]{2,3}(?:(([a-c]\s(b)?(?:c*?\d+_?\n))+?([\s\- ](?:ab{1,2}[A-Z])|\-[1a]+?(\nca{1,}a)|^(x*?1)(?:b+[^A-Z\d])(?:\d\s"+1))?){0,2}?(\n)?(a(?:[^0-9A-Z]{2,2}?(\d\n+a\d{0,2})c)(?:a[ax-z\w]{2,}?a*?|)?\w*).+)/, + /\-{2,3}\D*?A/i, + /\b\.{0,}($)(?(?[\d]*"??1?)?^)|x*|\-?^|(b\W*){2}/, + /(\b((?:\W){0,2}?\B((x\dbb+)$[\nA-Z]\w{2,2}|[^.\s0-9]*$(?a{1,2})|(?\s )+\b\d?){0,2}?\n{2}){0,1}bA)|a{1,1}|(?:([a-c1 ])._)/, + /(\/??)_{0,0}(?:[ba]\d(\.(\s+\b[\]]|(?:\d?b{2}b|["a]b\s*?)|[0-9b\-]{1,}(b\W\S)){2}|[ A-Z\s]??(?:_*?[ ]+)([^"a-z\n][^0-9][ 1a]*| ?-{1,3}?(?:\db*?[^a"]a?))(\s+|[\-a-z"]*))\w*?)/gy, + /("?_(A[\-\d]{1,}){1,2})+[._]1??/g, + /[\-]([\d"]\/*?[^\]\w]+(?:\n*?$(\.)(?:\b(?:"?\.{2,}A{0,})))|($$|[^\]a]{0,}?^(?:(\-|\.A*|b)\nx[^a_"]|(?:a){1,2}(?:[\W\nb]\/\d))){2})(?:\-*[x-zax-z]?|((?:(?A{2,}?[^\s]b{1,3}x)A*\D|A)){0,1}(?:x(([x-za-z0-9]a||_a{0,}a\-))))?\n/gy, + /\W{0,}[.b\w][^.]/mu, + /\b[^ \s](?(?:([\d\-\]]?){0}?-{0}(c+[\-_0-9]))((?(\d*?\.*-+?)(\d[^b\w]{2}?\W?b+|||bxb{0,}\S+|b){0,1}x{1,1}?[11a-c]{0,}|(?:b){2})(?: +(?:\.{0,}?a{0,}\d*)|\s[^_x-z][.]){0,2}?){0,2}?)?[\nb\-]+?/iy, + /("\d?(?([\W](?:ab{2}[^A-ZA-Z]+)\D(?:\d+?)??|)[\W\w]([^A-Z_]c(?[a-c\n\-]|\-\-+b{2,})){0,1}$)??)(?:((1*A)(?:(\.+?"\S?)|(\d? [a]\d??|aa\d+?[x-z ]){2}(b|b{2}\.?\d){0,2}?(\da{1}\n))[^A-Z\s]((?:\sa*?)\.*?(?1{0}ba{1,}c{2,3}?|1A\d)?A){0}|\W(?b+?(a)( +|b[^\wa-c]{1,}x)(b?a*){2}){1,2}(?:(?:b\d\d)\b(["\-]+b?-{2,})??| ?([x-z]*[.a-za]aa)1*)[^a-c\W]|"{2}(?:(\d+a?ba))\B)(?(a *)((a\d){0,2}?(\-\d\d*)*?(b_*)?)\D{1})((\/{0,1}){1,2})?)(?(\s{0,0}(?.{0,}x+([^_a-ca]?[.\]]?_+?)( *){0,1}){0,1}-?\b){0,2}?-?(?:[\]x-z"]+|(?:(\d??a)){1,2})){0,1}/im, + /\-/gi, + /A{0}?\.(([.\s_])a?(?: )??)\w|(a*[ .])\S1|c{2,2}/s, + /[^\n]|.\W{2,3}?\b$/g, + /\d?? +?/g, + /(((?:(a{0})??\/+?\b1)\.|[\]"])b\n*?)-*b[\]\wA-Z]{0,}/g, + /"+?(a(\Sc*\B){0}?\W{1,}[1\n]*?){2,3}[\nb]??(\/*?)?/, + /["A-Z](?\-? +)?/g, + /x-+A(?:\b((["0-9\-])(([x-zA-Z]??ab*)\s\D[\]"\n])^b)??\B|\d*?$(?:A\s)?|.\n(b($){0,1}((?a\S*?\d*\.){2}1*(a ?.-{1,1}|\S*\Wab))+(?(?:.?\s+|[ \s]\d{2,})|){0,1})){1,2}/gyu, + /([\d"\]]{2}\b[\s_]([ba\n]{2}\nc-)|\/\B((?c\B.\-){1,2}(1*([\sa-c\n].c{1,} {1,}?|a+a+)(?:b?\-+?|\-*?[1\sa]a{0}))(?:\d^1*\s*?)A{2,}){0,1})[aA-Z"]b/iy, + /(c(^\W$ |(\B1(b*a\w){0,2}?)){0,2}?((\.??( )|b(?[aa]"1*))\/){1,2})(?:[^\-\w][^\]\s]\-[.]??)/gsy, + /\w{1,}(\B\d+?)(?:a\b){0,1}(x-"|\B)/gs, + /a$ +|([^a0-9]\B^(A{1,3}?|-11)?|(?:a?(^){0,2}?- ){0,1}a{2}\B\n?)\/*(?:$[\]]+?b*(?\/*\-{1}?))/, + /(?: ?(?:x\.-){1,2}).{2,}(?:[\n"](?:\s*. +?1|\b)??(?:$[\wb]*){2}| +\.^\/+?)-|_{1,}?[^a-z]+/su, + / /gsyu, + /1_+/gsu, + /_a([^" \W]+(?:[_x-z]([x-za\w][\s]{1,})(?(\S{0,}?)??c\D{0,2}?){1,2}a?){1,2})/m, + /(([\W]($( \/+\.x{1,})??[ 1]{0,0}_+|\.??\/(?:\d*b*\d)$){0,2}?c\b)((?\S*?1*[b])\.{1}(1[^.bx-z]+c+|([A-Z\]_]{1,3})(c\d{1}b)??([\w\n\d]{2,}?\s*){0,2}?|(bb|b*?[1a-z])?? c)|\w{2,}))[^\sa-c_]\S/gms, + /1/gy, + /\/{2}x|(?:([\]\n][\W])\/?|\-+?|$(?:\b)??[\n])(\.){0,1}/g, + /\wb{0,}c{0}(x{2,3}){0,1}/g, + /(?:\/)["a-c0-9]*^[^0-9\w1]|(aa+\b){0,2}?|\S??/g, + /\w*\w/my, + /1"/, + /\b/u, + /(?:\-*?([.]{2}?)?)??A/gis, + /\b/i, + /-*^/gyu, + / (?:[\W\n]{0}?c *?)|_?[^a-z\s]/gsu, + /A{0,}(\.??c\.[^ \-])??b*?$/gi, + /(?:"\/(?\n((a??)[.]{2}?(b?|x*)*|\w)?(b{0,1}?\B)A{2}){0,1}){1,2}(\/*?(?:\W(?-{2,3}|1?$^(?b*a+b{2}){1,2}){0,1}\s{1,})?){0,2}?(?\s|\B(\b-(?\-[ 1]|(b){2}(?-\/*){0,2}?[^b.]?\B)|\W{2,}((\.*a[^x-z\]]{0,0}\d+)(\d.{0,})\-\n|(?:[^0-9ba])(?a?|){0,1}){2}\/??\b||_(?:-{1,1}?1\b)1*(\-{1,}(?:"){0,1}1+?(\d))??|(?_a{1,1}){0,2}?((b)(\/*a|[\-1]*[a-c]{1,}a*[\-\]])\b)|\S(?a)\/[\d_\d]{0,1}|(?\W?)\.{2,3}?[\n][^a]?)\S??[^0-9\wa-z])\.{1,}|(\b[.\dx-z]){2}[x-z]??$(?:(A{2}\/*|(?(?:\.a*|[A-Z]a){2}c\W[a-ca\-]){1,2}(x{2}\S{0,}?[a\W]{2})??_*\w|(\S|$\w)(?(a{0,}x){0,2}?\/A{1,3}){2}){0,2}?"| {1,}-_){0,2}?/, + /[^_ ]b/g, + /c?\bA{1,}/gyu, + /b(?:[\s]*?[a-z\d\n]??(?:"[a-c]*\/))?(?:(.+"+)?1{0})\//gmsu, + /"?([a-c]\W{0}\D??(?:\b))?\/{2,}|\D{2,4}?xx1+/g, + /\B|((?[^"\W]{1}?$"+\b)(?:[a-z]){0,1}|[^ ]{1,} )|-\S(\.*((?\n\bA{0,}?(-?\-+?[_a-z\n]){2}){0,1}\/)[^"a-c.]??)/m, + /(\s|\n|(a[A-Z\d]{2,}_{2}\.{0,1}){2}\.*["])([b0-9])?/gms, + /(A{2}?$^([^_\s]+?|(b_"*?(?.\d\.?))?-(A(?:A??a?|)([\saA-Z]b{0,}?\d??\s{2,}){0,1}[A-Z\d"]){2}))a[^1]?/g, + /A|(\b)??\b.*?(A(\d)(?\B(\d+[\n]{0,1}[^.\]\s])))/gi, + /(\d|\b[.]){1,2}([^\W1]b(?"{1}(^[a-c"\]]+)\S*)?)(\-(^ \b){0,2}?)/g, + /-*c .*?|_\b [^\n]/giu, + /(?[\d]){1,2}$/iy, + /[".]{2,3}?\-/is, + /(?:((?:([\W\W]?1\S*||\w??){0,1}_)??(?:(?a.+?_-{1,}|c)?x^)??(?\w*){1,2}\B|))_|a\naa+?/gis, + /\W\n(?:.$|(-?\.{0,0}\-([b]{1}){1,2})|\-+([^a-z0-9]*(?:\/){0,1}))??|[\]1]+c{2}?\-(?:[\]])/gy, + /\b(?:[\]_](_|A{1,}?(([^A-Z]_x){0}x[\]]$)\b){0,1}(\BA([\n\W]??^[a-c_]\D|){0,2}?){2}( )|\b(A[\d\-a-z][a0-9_]{1,3}\.)((([\n]a|\d*\/{0}?b){2}\sb+?)(\W+(\d?\d{1,}?[^\n]\.{1})")(?:(?:a+?b*?ab|\.b+?1[1]*){1,2}[_"\]]){0,1})(A){0,1})??|\/ /i, + /(?:_+-(((?:[^.]\d??a|\S|a{2}bb)(\d\d\d)){0,1}\D\bA{1,}?){1,2})[^x-z]/, + /x\-+/gy, + /\.*"-/, + /(?[x-za-c_]b)\.{0,}_*/, + /\.|(? ){0,2}?[b]*c+?[\-]/m, + /[\n\nA-Z]+?(?: +?([x-z]*\n?[0-9b\W]^)(?:[A-ZA-Za](\n+(?:a{0,}"??.b|\s*?c\d\/)a+))?|x-(A_?)(?\s)|c??)(?:b+[A-Za-za-c]([\d\]\]]{1,1}A){1,2}|$b)??/u, + /\n{2}?/g, + /(?:((A+$))\d\/[ \na-c])*(?:(?:(?(?:a?\/a{2,4})\/{2,3}(b_))+[.b]+?(?:[^a\s\]][ ]{0,}?([\s]??")|_+$){0,2}?){1,} )|"{2,}(\.$(a((?a-{2,3}|[a-c\W.]\-{2}\d{2})\b.*)(?:(?: \-{1,}))?)([\d0-9]{2,2}[0-9ba-z]*))"*?a*/, + /[1]{0,}((\d[a][^.a-c]|[^a\n]\Wb |c+\w( *\/^[\d]?)+?")){1,2}\s[a-c]??|(?:^){2}(\S-*|a(\b\D{1}|(("??b[A-Zx-z]\-*){2}A|(bbx[^a-z_])(?:[^\na-c]{0,2}[\]a-c1]b?-+)?(?:a[a-za-z]|)|$_)[a])b+){0,1}/g, + /$/is, + /[^x-z]/y, + /\n(\/\W{0,2}|^__*[\-\]\n]){0,1}/y, + /(\BA)??/gimy, + /[b](?:[\-_]{0,0}x[\d]+|(?:\D{1,}){0}($(?:( *-??|\d{2}A{2,3}?b+|[a-c._]\dx*){0,2}?[a-c]))(? {1} *(?:\-\n))[a\s]*?)\b/isy, + /[^0-9x-z].+c[\-b]+|_\d/gms, + /1c([bA-Z]?){2}|/s, + /$^/gs, +]; + +function show(m: RegExpExecArray | null): string { + if (m === null) return "-"; + const parts: string[] = []; + for (let i = 0; i < m.length; i++) parts.push(m[i] === undefined ? "U" : JSON.stringify(m[i])); + return `${m.index}:${parts.join(",")}`; +} + +for (const re of patterns) { + const out: string[] = []; + for (const s of inputs) { + re.lastIndex = 0; + const a = show(re.exec(s)); + const b = re.lastIndex; + const c = show(re.exec(s)); + re.lastIndex = 0; + out.push(`${a} ${b} ${c} ${re.lastIndex} ${re.test(s)} ${s.replace(re, "<$&|$1>")} ${s.search(re)}`); + re.lastIndex = 0; + if (!re.flags.includes("y")) out.push(JSON.stringify(s.split(re))); + if (re.flags.includes("g")) out.push(JSON.stringify(s.match(re))); + } + console.log(re.source, out.join(" ")); +} diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 17832b880..2906ac1e2 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -96,7 +96,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 16, marker: "scr_runtime_abi_v16" }, + runtime_abi: { version: 17, marker: "scr_runtime_abi_v17" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: {