From 0b483e8650dc6a87b565a7ed9fbcbb7578ef4f55 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 21:04:29 -0500 Subject: [PATCH] Map UTF-16 indexes of long non-ASCII strings in constant time and quick-check normalize - Index walks (slice, substring, substr, startsWith/endsWith with a position, lastIndexOf results) on a non-ASCII receiver of 64 KiB or more: once they would have paid for one pass over it, the receiver's index entry keeps every unit's byte offset (a 32-bit base per 64 units and one byte per unit), and indexes map with two loads. Before that, a walk within 64 units of the hot cursor starts from it, without the index search. The index entry stays 128 bytes on 64-bit targets. - lastIndexOf on non-ASCII receivers tests the needle's first byte before comparing, as the ASCII path does. - normalize: a quick check (UAX #15) returns the receiver when every code point is inert for the form: unchanged when normalized alone, combining class 0, and, in the ranges checked, never the second element of a composition. Other text normalizes segment by segment, each starting at a starter, so runs of inert code points copy as they are and a lone precomposed letter's result is kept per form. The per-code-point facts come from unicode_normalize, 256 code points at a time. --- .../corpus/8008-string-index-paths.ts.json | 6 + packages/runtime/src/scr_regex.c | 197 +++++++++++++++++- packages/runtime/src/scr_string.c | 189 ++++++++++++----- tests/corpus/8008-string-index-paths.ts | 102 +++++++++ .../8008-string-index-paths.ts.json | 1 + 5 files changed, 435 insertions(+), 60 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json create mode 100644 tests/corpus/8008-string-index-paths.ts create mode 100644 tests/harness/corpus-costs/8008-string-index-paths.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json new file mode 100644 index 000000000..5e2082ab9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8008-string-index-paths.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 37f94bd65..4296ce791 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -2469,6 +2469,146 @@ static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len, return (int)count; } +/* Normalization quick check (UAX #15): a string whose every code point is + * inert for the form is already in that form. A code point is inert when + * normalizing it alone leaves it unchanged, its canonical combining class + * is 0, and (for the composing forms) no composition can take it as its + * second element. The last condition holds in the ranges below: none holds + * a code point with combining class 0 that composes with a preceding one + * (those are Hangul jamo vowels and trailing consonants, U+3099/U+309A, + * and vowel signs of Indic and Southeast Asian scripts above U+0900), so a + * code point outside them takes the full algorithm. Inert flags are + * computed through unicode_normalize, 256 code points at a time, when a + * block is first seen. */ +#define SCR_NQC_BLOCKS 0x200 +typedef struct { + uint8_t known[4][SCR_NQC_BLOCKS]; + uint8_t inert[4][SCR_NQC_BLOCKS][32]; + uint8_t starter[4][SCR_NQC_BLOCKS][32]; /* class 0, never a second element */ + struct { + uint32_t cp; + uint8_t len; + char bytes[15]; + } single[4][256]; /* a lone code point's normalized UTF-8 */ +} ScrNormalizeQuick; +static SCR_TL ScrNormalizeQuick *scr_nqc; + +static void scr_nqc_free(void) { + free(scr_nqc); + scr_nqc = NULL; +} + +static bool scr_nqc_in_ranges(uint32_t c) { + return c < 0x900 || (c >= 0x2000 && c < 0x3099) || (c >= 0x309B && c < 0xA000) || + (c >= 0xAC00 && c < 0xD7A4) || (c >= 0xF900 && c < 0x10000) || + (c >= 0x1F000 && c < 0x1FB00); +} + +/* Bit 0: inert. Bit 1: a starter, with class 0 and never the second element + * of a composition, so nothing before it reorders or composes with it. */ +/* Whether c has class 0 and lies in the ranges above. U+0345 is the one + * code point of class 240, which the reordering test cannot tell from + * class 0; a mark of class 1..239 after U+0345 moves before it. */ +static bool scr_nqc_class0(uint32_t c) { + if (c == 0x345 || !scr_nqc_in_ranges(c)) return false; + uint32_t pair[2] = {0x345, c}; + uint32_t *out = NULL; + int n = unicode_normalize(&out, pair, 2, UNICODE_NFD, NULL, scr_normalize_realloc); + bool starter = n >= 2 && out[0] == 0x345; + free(out); + return starter; +} + +static unsigned scr_nqc_compute(uint32_t c, UnicodeNormalizationEnum mode) { + if (!scr_nqc_class0(c)) return 0; + uint32_t *out = NULL; + int n = unicode_normalize(&out, &c, 1, mode, NULL, scr_normalize_realloc); + bool same = n == 1 && out[0] == c; + free(out); + if (same) return 3u; + /* A starter's decomposition must begin with one too (U+FF9E's + * compatibility decomposition is U+3099, which composes with the code + * point before it). */ + UnicodeNormalizationEnum decompose = + mode == UNICODE_NFKC || mode == UNICODE_NFKD ? UNICODE_NFKD : UNICODE_NFD; + out = NULL; + n = unicode_normalize(&out, &c, 1, decompose, NULL, scr_normalize_realloc); + bool leads = n >= 1 && scr_nqc_class0(out[0]); + free(out); + return leads ? 2u : 0u; +} + +/* scr_nqc_compute's bits for c, from its block's flags. */ +static unsigned scr_nqc_class(uint32_t c, UnicodeNormalizationEnum mode) { + if (c >= (uint32_t)SCR_NQC_BLOCKS << 8) return 0; + if (!scr_nqc) { + scr_nqc = calloc(1, sizeof *scr_nqc); + if (!scr_nqc) return 0; + scr_atexit(scr_nqc_free); + } + uint32_t block = c >> 8; + uint8_t *inert = scr_nqc->inert[mode][block]; + uint8_t *starter = scr_nqc->starter[mode][block]; + if (!scr_nqc->known[mode][block]) { + for (uint32_t i = 0; i < 256; i++) { + unsigned bits = scr_nqc_compute((block << 8) | i, mode); + if (bits & 1) inert[i >> 3] |= (uint8_t)(1u << (i & 7)); + if (bits & 2) starter[i >> 3] |= (uint8_t)(1u << (i & 7)); + } + scr_nqc->known[mode][block] = 1; + } + unsigned bit = c & 0xff; + return ((inert[bit >> 3] >> (bit & 7)) & 1) | (((starter[bit >> 3] >> (bit & 7)) & 1) << 1); +} + +/* Normalize the code points of data[from, to) and append them. */ +static void scr_nqc_segment(ScrJsonBuf *b, const char *data, size_t from, size_t to, + UnicodeNormalizationEnum mode) { + size_t first_step; + uint32_t first = scr_case_decode(data + from, &first_step); + if (first_step == to - from && scr_nqc) { + /* One code point (a precomposed letter between inert ones): its + * normalized bytes repeat, so they are kept per form. */ + __typeof__(scr_nqc->single[0][0]) *slot = &scr_nqc->single[mode][first & 0xff]; + if (slot->cp != first || slot->len == 0) { + uint32_t *output = NULL; + int count = unicode_normalize(&output, &first, 1, mode, NULL, scr_normalize_realloc); + if (count < 0) scr_regex_oom(); + ScrJsonBuf one; + scr_jb_init(&one); + for (int i = 0; i < count; i++) scr_case_put_cp(&one, output[i]); + free(output); + ScrStr *bytes = scr_jb_finish(&one); + if (bytes->len == 0 || bytes->len > sizeof slot->bytes) { + scr_jb_write(b, bytes->data, bytes->len); + scr_str_release(bytes); + return; + } + slot->cp = first; + slot->len = (uint8_t)bytes->len; + memcpy(slot->bytes, bytes->data, bytes->len); + scr_str_release(bytes); + } + scr_jb_write(b, slot->bytes, slot->len); + return; + } + uint32_t local[32]; + uint32_t *input = to - from <= 32 ? local : malloc((to - from) * sizeof *input); + if (!input) scr_regex_oom(); + int length = 0; + for (size_t offset = from; offset < to;) { + size_t step; + input[length++] = scr_case_decode(data + offset, &step); + offset += step; + } + uint32_t *output = NULL; + int count = scr_unicode_normalize(&output, input, length, mode); + if (input != local) free(input); + if (count < 0) scr_regex_oom(); + for (int i = 0; i < count; i++) scr_case_put_cp(b, output[i]); + free(output); +} + ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { UnicodeNormalizationEnum mode; if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC; @@ -2483,23 +2623,58 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { /* Every ASCII string is already normalized in all four forms. Validate * the form first: even an empty subject must reject an invalid form. */ if (scr_text_ascii(s)) return scr_str_retain((ScrStr *)s); - if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); - uint32_t *input = malloc((s->len + 1) * sizeof(uint32_t)); - if (!input) scr_regex_oom(); - int length = 0; + bool inert = true; for (size_t offset = 0; offset < s->len;) { + if ((unsigned char)s->data[offset] < 0x80) { + offset++; + continue; + } size_t step; - input[length++] = scr_case_decode(s->data + offset, &step); + uint32_t c = scr_case_decode(s->data + offset, &step); offset += step; + if (!(scr_nqc_class(c, mode) & 1)) { + inert = false; + break; + } } - uint32_t *output = NULL; - int count = scr_unicode_normalize(&output, input, length, mode); - free(input); - if (count < 0) scr_regex_oom(); + if (inert) return scr_str_retain((ScrStr *)s); + if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); + /* An inert code point bounds reordering and composition, and so does any + * other starter (class 0, never a composition's second element), so the + * text normalizes segment by segment. A segment starts at a starter: runs + * of inert code points copy as they are, and a segment with any other + * code point goes through scr_unicode_normalize. */ ScrJsonBuf buffer; scr_jb_init(&buffer); - for (int i = 0; i < count; i++) scr_case_put_cp(&buffer, output[i]); - free(output); + size_t seg = 0, last_inert = 0; + bool plain = true; + for (size_t offset = 0; offset < s->len;) { + unsigned char lead = (unsigned char)s->data[offset]; + size_t step = 1; + unsigned kind = 3; + if (lead >= 0x80) kind = scr_nqc_class(scr_case_decode(s->data + offset, &step), mode); + if (kind & 1) { + if (!plain) { + scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = true; + } + last_inert = offset; + } else if (kind & 2) { + if (plain) scr_jb_write(&buffer, s->data + seg, offset - seg); + else scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = false; + } else if (plain) { + size_t start = last_inert > seg ? last_inert : seg; + scr_jb_write(&buffer, s->data + seg, start - seg); + seg = start; + plain = false; + } + offset += step; + } + if (plain) scr_jb_write(&buffer, s->data + seg, s->len - seg); + else scr_nqc_segment(&buffer, s->data, seg, s->len, mode); return scr_jb_finish(&buffer); } diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 64a4772e5..71d629941 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -83,6 +83,12 @@ typedef struct { size_t cu; /* UTF-16 code-unit offset, always a code-point boundary */ size_t cb; /* matching UTF-8 byte offset, never a continuation byte */ } ScrSidxPoint; +/* One allocation: the 32-bit base of every 64th unit follows the deltas. */ +typedef struct { + size_t n; /* the UTF-16 length the offsets describe */ + uint32_t *base; /* byte offset of every 64th unit */ + uint8_t delta[]; /* each unit's byte offset past its block's base */ +} ScrSidxOffsets; typedef struct ScrSidx { const ScrStr *s; /* NULL = empty slot */ size_t u16len; /* SCR_U16_UNKNOWN until the whole current string */ @@ -94,10 +100,12 @@ typedef struct ScrSidx { size_t indexed_cb; bool no_more_points; /* metadata allocation failed/overflowed: fail open */ bool points_complete; /* points cover every stride of indexed prefix */ + uint32_t walks; /* index walks before the offsets exist */ uint16_t *units; /* the UTF-16 code units, once reads warrant them */ - size_t nunits; - size_t reads; /* indexed reads that missed the unit array */ -} ScrSidx; + uint32_t nunits; /* units and offsets need at most 1 << 24 bytes */ + uint32_t reads; /* indexed reads that missed the unit array */ + ScrSidxOffsets *offsets; /* every unit's byte offset, once walks warrant */ +} ScrSidx; /* 128 bytes on 64-bit targets: every lookup clears or copies one */ /* Keep short-string cursor traffic out of the sparse cache. A short-lived * one-byte receiver can be far more common than a large indexed one; sharing * the round-robin slots would otherwise rebuild a warm index every few calls. @@ -136,6 +144,7 @@ static void scr_sidx_clear(ScrSidx *e, unsigned *live) { scr_str_scan.n = 0; } free(e->units); + if (e->offsets) free(e->offsets); /* rare: skip the call on the hot clear */ memset(e, 0, sizeof(*e)); } @@ -337,6 +346,9 @@ static void scr_sidx_concat_append(const ScrStr *s, size_t oldlen) { e->units = NULL; e->nunits = 0; e->reads = 0; + free(e->offsets); /* and so do the offsets */ + e->offsets = NULL; + e->walks = 0; } } } @@ -1576,6 +1588,8 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) { /* How far past its end a forward access may grow the cached ASCII run, and * how far beyond the access the growth looks ahead. */ #define SCR_SIDX_ASCII_REACH ((size_t)4096) +/* How far from the hot cursor an index walks from it. */ +#define SCR_SIDX_NEAR_UNITS ((size_t)64) #define SCR_SIDX_ASCII_AHEAD ((size_t)256) /* Grow the cached ASCII run forward until it covers byte `want` (plus a @@ -1624,8 +1638,118 @@ static inline size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, return scr_u16_to_byte_walk(s, e, u16, mid); } +/* Index walks on one long non-ASCII receiver (slices, substrings and + * indexed reads taken across it) each step through bytes from the nearest + * known position. Once those walks would have paid for one pass over the + * receiver (a walk per 32 bytes, at least 64 walks), a receiver from + * SCR_SIDX_MIN_BYTES to SCR_SIDX_OFFSETS_MAX bytes keeps every unit's byte + * offset: a 32-bit base per 64 units and one byte per unit past it (64 units + * span at most 192 bytes). The low half of an astral character has its high + * half's offset. The offsets are freed with the entry, and an append drops + * them (scr_sidx_concat_append). */ +#define SCR_SIDX_OFFSET_BLOCK ((size_t)64) +#define SCR_SIDX_OFFSETS_MAX ((size_t)1 << 24) +static bool scr_sidx_offsets(const ScrStr *s, ScrSidx *e) { + if (e->offsets) return true; + if (s->len < SCR_SIDX_MIN_BYTES || s->len > SCR_SIDX_OFFSETS_MAX) return false; + if (++e->walks < 64 || (size_t)e->walks * 32 < s->len) return false; + size_t n = scr_sidx_len(s, e); + if (s->ascii || e->u16len == s->len) return false; + size_t ndelta = (n + 1 + 3) & ~(size_t)3; + ScrSidxOffsets *o = + malloc(sizeof *o + ndelta + (n / SCR_SIDX_OFFSET_BLOCK + 1) * sizeof(uint32_t)); + if (!o) { + e->walks = 0; + return false; + } + o->n = n; + o->base = (uint32_t *)(o->delta + ndelta); + uint32_t *base = o->base; + uint8_t *delta = o->delta; + size_t cu = 0, cb = 0; + while (cb < s->len && cu < n) { + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + for (size_t w = seq == 4 ? 2 : 1; w > 0 && cu < n; w--, cu++) { + if (cu % SCR_SIDX_OFFSET_BLOCK == 0) base[cu / SCR_SIDX_OFFSET_BLOCK] = (uint32_t)cb; + delta[cu] = (uint8_t)(cb - base[cu / SCR_SIDX_OFFSET_BLOCK]); + } + cb += seq; + } + e->offsets = o; + return true; +} + +static inline size_t scr_sidx_offset_at(const ScrSidx *e, size_t u16) { + return e->offsets->base[u16 / SCR_SIDX_OFFSET_BLOCK] + e->offsets->delta[u16]; +} + +/* Walk from a known code-point boundary (cu, cb) to UTF-16 index u16 and + * leave the hot cursor there. Same contract as scr_u16_to_byte_walk. */ +static size_t scr_sidx_walk_from(const ScrStr *s, ScrSidx *e, size_t cu, size_t cb, size_t u16, + bool *mid, bool refresh_run) { + while (cu > u16) { + /* Eight bytes starting inside a sequence can reach back at most three + * more bytes to its lead. Such a span contains at most nine UTF-16 + * units (seven ASCII bytes plus an astral pair), so it cannot overshoot + * a target at least nine units away. */ + if (cu - u16 >= 9 && cb >= 8) { + size_t start = cb - 8; + while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) + start--; + cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); + cb = start; + } else { + scr_sidx_back(s, &cu, &cb); + } + } + bool m = false; + while (cu < u16 && cb < s->len) { + if (u16 - cu >= 9 && s->len - cb >= 8) { + size_t end = cb + 8; + while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) + end++; + cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); + cb = end; + continue; + } + SCR_SIDX_STEP(); + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + size_t w = seq == 4 ? 2 : 1; + if (cu + w > u16) { /* u16 lands between the halves of an astral char */ + m = true; + break; + } + cu += w; + cb += seq; + } + e->cu = cu; + e->cb = cb; + if (refresh_run && cb < s->len && (unsigned char)s->data[cb] < 0x80) + scr_sidx_ascii_run(s, e); + *mid = m; + return cb; +} + static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, ScrSidx *e, size_t u16, bool *mid) { + if (scr_sidx_offsets(s, e)) { + *mid = false; + if (u16 >= e->offsets->n) return s->len; + size_t cb = scr_sidx_offset_at(e, u16); + size_t cu = u16; + if (u16 > 0 && scr_sidx_offset_at(e, u16 - 1) == cb) { + *mid = true; /* the low half of an astral character */ + cu = u16 - 1; + } + e->cu = cu; + e->cb = cb; + return cb; + } + /* Within a short distance of the hot cursor (a scanner + * taking slices near its position), walk from that cursor: no index + * search, extension or ASCII-run growth is cheaper. */ + if (e->cb <= s->len && scr_sidx_abs_diff(e->cu, u16) <= SCR_SIDX_NEAR_UNITS) + return scr_sidx_walk_from(s, e, e->cu, e->cb, u16, mid, false); /* A short step forward from the hot cursor (a sequential scan of * non-ASCII text) walks from the cursor itself. */ if (u16 >= e->cu && u16 - e->cu <= 8) { @@ -1665,48 +1789,7 @@ static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, Sc } scr_sidx_materialize_identity_prefix(s, e); ScrSidxPoint near = scr_sidx_near_u16(s, e, u16); - size_t cu = near.cu, cb = near.cb; - while (cu > u16) { - /* Eight bytes starting inside a sequence can reach back at most three - * more bytes to its lead. Such a span contains at most nine UTF-16 - * units (seven ASCII bytes plus an astral pair), so it cannot overshoot - * a target at least nine units away. */ - if (cu - u16 >= 9 && cb >= 8) { - size_t start = cb - 8; - while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) - start--; - cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); - cb = start; - } else { - scr_sidx_back(s, &cu, &cb); - } - } - bool m = false; - while (cu < u16 && cb < s->len) { - if (u16 - cu >= 9 && s->len - cb >= 8) { - size_t end = cb + 8; - while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) - end++; - cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); - cb = end; - continue; - } - SCR_SIDX_STEP(); - size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); - size_t w = seq == 4 ? 2 : 1; - if (cu + w > u16) { /* u16 lands between the halves of an astral char */ - m = true; - break; - } - cu += w; - cb += seq; - } - e->cu = cu; - e->cb = cb; - if (cb < s->len && (unsigned char)s->data[cb] < 0x80) - scr_sidx_ascii_run(s, e); - *mid = m; - return cb; + return scr_sidx_walk_from(s, e, near.cu, near.cb, u16, mid, true); } static size_t scr_byte_to_u16_walk(const ScrStr *s, ScrSidx *e, size_t byte_off); @@ -1875,10 +1958,17 @@ static uint16_t *scr_str_units(const ScrStr *s, size_t *n) { * megabytes, which scanners read in full. The array is freed with the * entry, and an append drops it (scr_sidx_concat_append). */ #define SCR_SIDX_UNITS_MAX ((size_t)1 << 24) +static void scr_sidx_take_units(const ScrStr *s, ScrSidx *e) { + size_t n = 0; + e->units = scr_str_units(s, &n); + e->nunits = (uint32_t)n; +} + static bool scr_sidx_units(const ScrStr *s, ScrSidx *e) { if (!e->units) { - if (++e->reads < 32 || e->reads * 16 < s->len || s->len > SCR_SIDX_UNITS_MAX) return false; - e->units = scr_str_units(s, &e->nunits); + if (s->len > SCR_SIDX_UNITS_MAX || ++e->reads < 32 || (size_t)e->reads * 16 < s->len) + return false; + scr_sidx_take_units(s, e); } /* Publish the scan's receiver for emitted reads (scr_runtime.h). */ if (scr_str_scan.s != s) { @@ -1905,7 +1995,7 @@ void scr_str_scan_units_for(ScrStr *s, double from, double to) { (void)scr_sidx_units(s, e); return; } - e->units = scr_str_units(s, &e->nunits); + scr_sidx_take_units(s, e); } if (scr_str_scan.s != s) { scr_str_scan.s = s; @@ -1980,8 +2070,9 @@ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { size_t start_byte = scr_u16_to_byte_c(s, e, start16, &mid); size_t max_start = s->len - needle->len; if (start_byte < max_start) max_start = start_byte; + char first = needle->data[0]; for (size_t i = max_start + 1; i-- > 0;) { - if (memcmp(s->data + i, needle->data, needle->len) == 0) { + if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) { return (double)scr_byte_to_u16_c(s, e, i); } } diff --git a/tests/corpus/8008-string-index-paths.ts b/tests/corpus/8008-string-index-paths.ts new file mode 100644 index 000000000..a9a748ad8 --- /dev/null +++ b/tests/corpus/8008-string-index-paths.ts @@ -0,0 +1,102 @@ +// UTF-16 index operations on long non-ASCII strings: slices and substrings +// walking forward and backward, interleaved on two strings, at astral +// characters and their halves, with negative and out-of-range indexes; +// lastIndexOf, startsWith and endsWith with positions; and normalize on +// text already in each form, composed and decomposed text, Hangul, +// compatibility characters and combining marks. + +const unit = "Ünïcödé tëxt with ß and 日本語 and émoji 😀 mixed in, "; +const big = unit.repeat(3000); +const other = "ascii words then 中文 and 🎉🎉 and ü, ".repeat(4000); + +function digest(text: string): number { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0; + return h; +} + +let forward = 0; +for (let i = 0; i + 30 < big.length; i += 7) { + forward = (forward * 33 + digest(big.slice(i, i + 20)) + digest(big.substring(i + 3, i + 9))) >>> 0; +} +let backward = 0; +for (let i = big.length - 1; i > 30; i -= 11) { + backward = (backward * 33 + digest(big.slice(i - 25, i)) + big.substring(i, i - 5).length) >>> 0; +} +let interleaved = 0; +for (let i = 0; i + 40 < other.length && i + 40 < big.length; i += 13) { + interleaved = (interleaved * 33 + digest(big.slice(i, i + 9)) + digest(other.slice(i + 1, i + 40))) >>> 0; +} +console.log(big.length, forward, backward, interleaved); + +// Indexes at astral characters and their halves. +const emojiAt = big.indexOf("😀", 100000); +console.log( + emojiAt, + JSON.stringify(big.slice(emojiAt, emojiAt + 1)), + JSON.stringify(big.slice(emojiAt + 1, emojiAt + 3)), + JSON.stringify(big.substring(emojiAt - 1, emojiAt + 1)), + big.charCodeAt(emojiAt + 1), + JSON.stringify(big.at(emojiAt + 1)), + JSON.stringify(big.charAt(emojiAt)), +); +console.log( + JSON.stringify(big.slice(-5)), + JSON.stringify(big.slice(-30, -20)), + big.slice(big.length - 3, big.length + 10).length, + JSON.stringify(big.substring(-10, 4)), + big.slice(5, 2).length, + JSON.stringify(big.slice(NaN, 3)), +); +for (const at of [0, 1, 52, 99999, 150000, big.length - 1, big.length, -1, -2, -big.length]) { + console.log(at, JSON.stringify(big.at(at)), JSON.stringify(big.charAt(at)), big.charCodeAt(at)); +} + +// lastIndexOf, startsWith and endsWith with positions. +console.log( + big.lastIndexOf("日本"), + big.lastIndexOf("日本", 100000), + big.lastIndexOf("😀", 50), + big.lastIndexOf("absent"), + big.lastIndexOf("é", 0), + big.lastIndexOf("Ü"), + big.lastIndexOf(""), + big.lastIndexOf("", 10), +); +let starts = 0; +let ends = 0; +for (let i = 0; i < big.length; i += 17) { + if (big.startsWith("tëxt", i)) starts++; + if (big.endsWith("😀", i)) ends++; +} +console.log(starts, ends, big.startsWith("Ü"), big.endsWith("in, "), big.startsWith("ünï"), big.endsWith("in, ", big.length - 1)); + +// normalize. +const samples = [ + "plain ascii", + "précomposé café naïve", + "décomposé café", + "日本語のテキスト 한국어 문장", + "각 각", + "ｆｕｌｌｗｉｄｔｈ ｶﾀｶﾅ fi ① ℌ ²", + "ậ ậ ͅ", + "Ω Å Å Ω K", + "が が か゚", + "emoji 😀 and \ud800 lone", + unit, +]; +for (const sample of samples) { + const forms = ["NFC", "NFD", "NFKC", "NFKD"].map((form) => sample.normalize(form)); + console.log( + forms.map((text) => `${text.length}:${digest(text)}`).join(" "), + forms.map((text) => text === sample).join(","), + ); +} +const longNfc = big.normalize(); +const longNfd = big.normalize("NFD"); +console.log(longNfc === big, longNfc.length, longNfd.length, digest(longNfd.slice(0, 100)), longNfd.normalize("NFC") === big); +try { + "x".normalize("NFX"); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/harness/corpus-costs/8008-string-index-paths.ts.json b/tests/harness/corpus-costs/8008-string-index-paths.ts.json new file mode 100644 index 000000000..3a1b75cbc --- /dev/null +++ b/tests/harness/corpus-costs/8008-string-index-paths.ts.json @@ -0,0 +1 @@ +{ "plain": 4784, "san": 2792 }