Map UTF-16 indexes of long non-ASCII strings in constant time and quick-check normalize

- Index walks (slice, substring, substr, startsWith/endsWith with a position, lastIndexOf results) on a non-ASCII receiver of 64 KiB or more: once they would have paid for one pass over it, the receiver's index entry keeps every unit's byte offset (a 32-bit base per 64 units and one byte per unit), and indexes map with two loads. Before that, a walk within 64 units of the hot cursor starts from it, without the index search. The index entry stays 128 bytes on 64-bit targets. - lastIndexOf on non-ASCII receivers tests the needle's first byte before comparing, as the ASCII path does. - normalize: a quick check (UAX #15) returns the receiver when every code point is inert for the form: unchanged when normalized alone, combining class 0, and, in the ranges checked, never the second element of a composition. Other text normalizes segment by segment, each starting at a starter, so runs of inert code points copy as they are and a lone precomposed letter's result is kept per form. The per-code-point facts come from unicode_normalize, 256 code points at a time.
2026-11-01 07:33:12 +08:00 · 2026-10-25 21:04:29 -05:00
parent 794a8cf8a8
commit 0b483e8650
5 changed files with 435 additions and 60 deletions
@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8008-string-index-paths.ts"
+ ],
+ "diags": []
+}
@@ -2469,6 +2469,146 @@ static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len,
  return (int)count;
 }

+/* Normalization quick check (UAX #15): a string whose every code point is
+ * inert for the form is already in that form. A code point is inert when
+ * normalizing it alone leaves it unchanged, its canonical combining class
+ * is 0, and (for the composing forms) no composition can take it as its
+ * second element. The last condition holds in the ranges below: none holds
+ * a code point with combining class 0 that composes with a preceding one
+ * (those are Hangul jamo vowels and trailing consonants, U+3099/U+309A,
+ * and vowel signs of Indic and Southeast Asian scripts above U+0900), so a
+ * code point outside them takes the full algorithm. Inert flags are
+ * computed through unicode_normalize, 256 code points at a time, when a
+ * block is first seen. */
+#define SCR_NQC_BLOCKS 0x200
+typedef struct {
+  uint8_t known[4][SCR_NQC_BLOCKS];
+  uint8_t inert[4][SCR_NQC_BLOCKS][32];
+  uint8_t starter[4][SCR_NQC_BLOCKS][32]; /* class 0, never a second element */
+  struct {
+    uint32_t cp;
+    uint8_t len;
+    char bytes[15];
+  } single[4][256]; /* a lone code point's normalized UTF-8 */
+} ScrNormalizeQuick;
+static SCR_TL ScrNormalizeQuick *scr_nqc;
+
+static void scr_nqc_free(void) {
+  free(scr_nqc);
+  scr_nqc = NULL;
+}
+
+static bool scr_nqc_in_ranges(uint32_t c) {
+  return c < 0x900 || (c >= 0x2000 && c < 0x3099) || (c >= 0x309B && c < 0xA000) ||
+         (c >= 0xAC00 && c < 0xD7A4) || (c >= 0xF900 && c < 0x10000) ||
+         (c >= 0x1F000 && c < 0x1FB00);
+}
+
+/* Bit 0: inert. Bit 1: a starter, with class 0 and never the second element
+ * of a composition, so nothing before it reorders or composes with it. */
+/* Whether c has class 0 and lies in the ranges above. U+0345 is the one
+ * code point of class 240, which the reordering test cannot tell from
+ * class 0; a mark of class 1..239 after U+0345 moves before it. */
+static bool scr_nqc_class0(uint32_t c) {
+  if (c == 0x345 || !scr_nqc_in_ranges(c)) return false;
+  uint32_t pair[2] = {0x345, c};
+  uint32_t *out = NULL;
+  int n = unicode_normalize(&out, pair, 2, UNICODE_NFD, NULL, scr_normalize_realloc);
+  bool starter = n >= 2 && out[0] == 0x345;
+  free(out);
+  return starter;
+}
+
+static unsigned scr_nqc_compute(uint32_t c, UnicodeNormalizationEnum mode) {
+  if (!scr_nqc_class0(c)) return 0;
+  uint32_t *out = NULL;
+  int n = unicode_normalize(&out, &c, 1, mode, NULL, scr_normalize_realloc);
+  bool same = n == 1 && out[0] == c;
+  free(out);
+  if (same) return 3u;
+  /* A starter's decomposition must begin with one too (U+FF9E's
+   * compatibility decomposition is U+3099, which composes with the code
+   * point before it). */
+  UnicodeNormalizationEnum decompose =
+      mode == UNICODE_NFKC || mode == UNICODE_NFKD ? UNICODE_NFKD : UNICODE_NFD;
+  out = NULL;
+  n = unicode_normalize(&out, &c, 1, decompose, NULL, scr_normalize_realloc);
+  bool leads = n >= 1 && scr_nqc_class0(out[0]);
+  free(out);
+  return leads ? 2u : 0u;
+}
+
+/* scr_nqc_compute's bits for c, from its block's flags. */
+static unsigned scr_nqc_class(uint32_t c, UnicodeNormalizationEnum mode) {
+  if (c >= (uint32_t)SCR_NQC_BLOCKS << 8) return 0;
+  if (!scr_nqc) {
+    scr_nqc = calloc(1, sizeof *scr_nqc);
+    if (!scr_nqc) return 0;
+    scr_atexit(scr_nqc_free);
+  }
+  uint32_t block = c >> 8;
+  uint8_t *inert = scr_nqc->inert[mode][block];
+  uint8_t *starter = scr_nqc->starter[mode][block];
+  if (!scr_nqc->known[mode][block]) {
+    for (uint32_t i = 0; i < 256; i++) {
+      unsigned bits = scr_nqc_compute((block << 8) | i, mode);
+      if (bits & 1) inert[i >> 3] |= (uint8_t)(1u << (i & 7));
+      if (bits & 2) starter[i >> 3] |= (uint8_t)(1u << (i & 7));
+    }
+    scr_nqc->known[mode][block] = 1;
+  }
+  unsigned bit = c & 0xff;
+  return ((inert[bit >> 3] >> (bit & 7)) & 1) | (((starter[bit >> 3] >> (bit & 7)) & 1) << 1);
+}
+
+/* Normalize the code points of data[from, to) and append them. */
+static void scr_nqc_segment(ScrJsonBuf *b, const char *data, size_t from, size_t to,
+                            UnicodeNormalizationEnum mode) {
+  size_t first_step;
+  uint32_t first = scr_case_decode(data + from, &first_step);
+  if (first_step == to - from && scr_nqc) {
+    /* One code point (a precomposed letter between inert ones): its
+     * normalized bytes repeat, so they are kept per form. */
+    __typeof__(scr_nqc->single[0][0]) *slot = &scr_nqc->single[mode][first & 0xff];
+    if (slot->cp != first || slot->len == 0) {
+      uint32_t *output = NULL;
+      int count = unicode_normalize(&output, &first, 1, mode, NULL, scr_normalize_realloc);
+      if (count < 0) scr_regex_oom();
+      ScrJsonBuf one;
+      scr_jb_init(&one);
+      for (int i = 0; i < count; i++) scr_case_put_cp(&one, output[i]);
+      free(output);
+      ScrStr *bytes = scr_jb_finish(&one);
+      if (bytes->len == 0 || bytes->len > sizeof slot->bytes) {
+        scr_jb_write(b, bytes->data, bytes->len);
+        scr_str_release(bytes);
+        return;
+      }
+      slot->cp = first;
+      slot->len = (uint8_t)bytes->len;
+      memcpy(slot->bytes, bytes->data, bytes->len);
+      scr_str_release(bytes);
+    }
+    scr_jb_write(b, slot->bytes, slot->len);
+    return;
+  }
+  uint32_t local[32];
+  uint32_t *input = to - from <= 32 ? local : malloc((to - from) * sizeof *input);
+  if (!input) scr_regex_oom();
+  int length = 0;
+  for (size_t offset = from; offset < to;) {
+    size_t step;
+    input[length++] = scr_case_decode(data + offset, &step);
+    offset += step;
+  }
+  uint32_t *output = NULL;
+  int count = scr_unicode_normalize(&output, input, length, mode);
+  if (input != local) free(input);
+  if (count < 0) scr_regex_oom();
+  for (int i = 0; i < count; i++) scr_case_put_cp(b, output[i]);
+  free(output);
+}
+
 ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) {
  UnicodeNormalizationEnum mode;
  if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC;
@@ -2483,23 +2623,58 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) {
  /* Every ASCII string is already normalized in all four forms. Validate
   * the form first: even an empty subject must reject an invalid form. */
  if (scr_text_ascii(s)) return scr_str_retain((ScrStr *)s);
-  if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom();
-  uint32_t *input = malloc((s->len + 1) * sizeof(uint32_t));
-  if (!input) scr_regex_oom();
-  int length = 0;
+  bool inert = true;
  for (size_t offset = 0; offset < s->len;) {
+    if ((unsigned char)s->data[offset] < 0x80) {
+      offset++;
+      continue;
+    }
    size_t step;
-    input[length++] = scr_case_decode(s->data + offset, &step);
+    uint32_t c = scr_case_decode(s->data + offset, &step);
    offset += step;
+    if (!(scr_nqc_class(c, mode) & 1)) {
+      inert = false;
+      break;
+    }
  }
-  uint32_t *output = NULL;
-  int count = scr_unicode_normalize(&output, input, length, mode);
-  free(input);
-  if (count < 0) scr_regex_oom();
+  if (inert) return scr_str_retain((ScrStr *)s);
+  if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom();
+  /* An inert code point bounds reordering and composition, and so does any
+   * other starter (class 0, never a composition's second element), so the
+   * text normalizes segment by segment. A segment starts at a starter: runs
+   * of inert code points copy as they are, and a segment with any other
+   * code point goes through scr_unicode_normalize. */
  ScrJsonBuf buffer;
  scr_jb_init(&buffer);
-  for (int i = 0; i < count; i++) scr_case_put_cp(&buffer, output[i]);
-  free(output);
+  size_t seg = 0, last_inert = 0;
+  bool plain = true;
+  for (size_t offset = 0; offset < s->len;) {
+    unsigned char lead = (unsigned char)s->data[offset];
+    size_t step = 1;
+    unsigned kind = 3;
+    if (lead >= 0x80) kind = scr_nqc_class(scr_case_decode(s->data + offset, &step), mode);
+    if (kind & 1) {
+      if (!plain) {
+        scr_nqc_segment(&buffer, s->data, seg, offset, mode);
+        seg = offset;
+        plain = true;
+      }
+      last_inert = offset;
+    } else if (kind & 2) {
+      if (plain) scr_jb_write(&buffer, s->data + seg, offset - seg);
+      else scr_nqc_segment(&buffer, s->data, seg, offset, mode);
+      seg = offset;
+      plain = false;
+    } else if (plain) {
+      size_t start = last_inert > seg ? last_inert : seg;
+      scr_jb_write(&buffer, s->data + seg, start - seg);
+      seg = start;
+      plain = false;
+    }
+    offset += step;
+  }
+  if (plain) scr_jb_write(&buffer, s->data + seg, s->len - seg);
+  else scr_nqc_segment(&buffer, s->data, seg, s->len, mode);
  return scr_jb_finish(&buffer);
 }

@@ -83,6 +83,12 @@ typedef struct {
  size_t cu; /* UTF-16 code-unit offset, always a code-point boundary */
  size_t cb; /* matching UTF-8 byte offset, never a continuation byte */
 } ScrSidxPoint;
+/* One allocation: the 32-bit base of every 64th unit follows the deltas. */
+typedef struct {
+  size_t n;              /* the UTF-16 length the offsets describe */
+  uint32_t *base;        /* byte offset of every 64th unit */
+  uint8_t delta[];       /* each unit's byte offset past its block's base */
+} ScrSidxOffsets;
 typedef struct ScrSidx {
  const ScrStr *s;       /* NULL = empty slot */
  size_t u16len;         /* SCR_U16_UNKNOWN until the whole current string */
@@ -94,10 +100,12 @@ typedef struct ScrSidx {
  size_t indexed_cb;
  bool no_more_points;   /* metadata allocation failed/overflowed: fail open */
  bool points_complete;  /* points cover every stride of indexed prefix */
+  uint32_t walks;        /* index walks before the offsets exist */
  uint16_t *units;       /* the UTF-16 code units, once reads warrant them */
-  size_t nunits;
-  size_t reads;          /* indexed reads that missed the unit array */
-} ScrSidx;
+  uint32_t nunits;       /* units and offsets need at most 1 << 24 bytes */
+  uint32_t reads;        /* indexed reads that missed the unit array */
+  ScrSidxOffsets *offsets; /* every unit's byte offset, once walks warrant */
+} ScrSidx; /* 128 bytes on 64-bit targets: every lookup clears or copies one */
 /* Keep short-string cursor traffic out of the sparse cache. A short-lived
 * one-byte receiver can be far more common than a large indexed one; sharing
 * the round-robin slots would otherwise rebuild a warm index every few calls.
@@ -136,6 +144,7 @@ static void scr_sidx_clear(ScrSidx *e, unsigned *live) {
    scr_str_scan.n = 0;
  }
  free(e->units);
+  if (e->offsets) free(e->offsets); /* rare: skip the call on the hot clear */
  memset(e, 0, sizeof(*e));
 }

@@ -337,6 +346,9 @@ static void scr_sidx_concat_append(const ScrStr *s, size_t oldlen) {
      e->units = NULL;
      e->nunits = 0;
      e->reads = 0;
+      free(e->offsets); /* and so do the offsets */
+      e->offsets = NULL;
+      e->walks = 0;
    }
  }
 }
@@ -1576,6 +1588,8 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) {
 /* How far past its end a forward access may grow the cached ASCII run, and
 * how far beyond the access the growth looks ahead. */
 #define SCR_SIDX_ASCII_REACH ((size_t)4096)
+/* How far from the hot cursor an index walks from it. */
+#define SCR_SIDX_NEAR_UNITS ((size_t)64)
 #define SCR_SIDX_ASCII_AHEAD ((size_t)256)

 /* Grow the cached ASCII run forward until it covers byte `want` (plus a
@@ -1624,8 +1638,118 @@ static inline size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16,
  return scr_u16_to_byte_walk(s, e, u16, mid);
 }

+/* Index walks on one long non-ASCII receiver (slices, substrings and
+ * indexed reads taken across it) each step through bytes from the nearest
+ * known position. Once those walks would have paid for one pass over the
+ * receiver (a walk per 32 bytes, at least 64 walks), a receiver from
+ * SCR_SIDX_MIN_BYTES to SCR_SIDX_OFFSETS_MAX bytes keeps every unit's byte
+ * offset: a 32-bit base per 64 units and one byte per unit past it (64 units
+ * span at most 192 bytes). The low half of an astral character has its high
+ * half's offset. The offsets are freed with the entry, and an append drops
+ * them (scr_sidx_concat_append). */
+#define SCR_SIDX_OFFSET_BLOCK ((size_t)64)
+#define SCR_SIDX_OFFSETS_MAX ((size_t)1 << 24)
+static bool scr_sidx_offsets(const ScrStr *s, ScrSidx *e) {
+  if (e->offsets) return true;
+  if (s->len < SCR_SIDX_MIN_BYTES || s->len > SCR_SIDX_OFFSETS_MAX) return false;
+  if (++e->walks < 64 || (size_t)e->walks * 32 < s->len) return false;
+  size_t n = scr_sidx_len(s, e);
+  if (s->ascii || e->u16len == s->len) return false;
+  size_t ndelta = (n + 1 + 3) & ~(size_t)3;
+  ScrSidxOffsets *o =
+      malloc(sizeof *o + ndelta + (n / SCR_SIDX_OFFSET_BLOCK + 1) * sizeof(uint32_t));
+  if (!o) {
+    e->walks = 0;
+    return false;
+  }
+  o->n = n;
+  o->base = (uint32_t *)(o->delta + ndelta);
+  uint32_t *base = o->base;
+  uint8_t *delta = o->delta;
+  size_t cu = 0, cb = 0;
+  while (cb < s->len && cu < n) {
+    size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]);
+    for (size_t w = seq == 4 ? 2 : 1; w > 0 && cu < n; w--, cu++) {
+      if (cu % SCR_SIDX_OFFSET_BLOCK == 0) base[cu / SCR_SIDX_OFFSET_BLOCK] = (uint32_t)cb;
+      delta[cu] = (uint8_t)(cb - base[cu / SCR_SIDX_OFFSET_BLOCK]);
+    }
+    cb += seq;
+  }
+  e->offsets = o;
+  return true;
+}
+
+static inline size_t scr_sidx_offset_at(const ScrSidx *e, size_t u16) {
+  return e->offsets->base[u16 / SCR_SIDX_OFFSET_BLOCK] + e->offsets->delta[u16];
+}
+
+/* Walk from a known code-point boundary (cu, cb) to UTF-16 index u16 and
+ * leave the hot cursor there. Same contract as scr_u16_to_byte_walk. */
+static size_t scr_sidx_walk_from(const ScrStr *s, ScrSidx *e, size_t cu, size_t cb, size_t u16,
+                                 bool *mid, bool refresh_run) {
+  while (cu > u16) {
+    /* Eight bytes starting inside a sequence can reach back at most three
+     * more bytes to its lead. Such a span contains at most nine UTF-16
+     * units (seven ASCII bytes plus an astral pair), so it cannot overshoot
+     * a target at least nine units away. */
+    if (cu - u16 >= 9 && cb >= 8) {
+      size_t start = cb - 8;
+      while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80)
+        start--;
+      cu -= scr_utf16_units_span(s->data + start, cb - start, NULL);
+      cb = start;
+    } else {
+      scr_sidx_back(s, &cu, &cb);
+    }
+  }
+  bool m = false;
+  while (cu < u16 && cb < s->len) {
+    if (u16 - cu >= 9 && s->len - cb >= 8) {
+      size_t end = cb + 8;
+      while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80)
+        end++;
+      cu += scr_utf16_units_span(s->data + cb, end - cb, NULL);
+      cb = end;
+      continue;
+    }
+    SCR_SIDX_STEP();
+    size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]);
+    size_t w = seq == 4 ? 2 : 1;
+    if (cu + w > u16) { /* u16 lands between the halves of an astral char */
+      m = true;
+      break;
+    }
+    cu += w;
+    cb += seq;
+  }
+  e->cu = cu;
+  e->cb = cb;
+  if (refresh_run && cb < s->len && (unsigned char)s->data[cb] < 0x80)
+    scr_sidx_ascii_run(s, e);
+  *mid = m;
+  return cb;
+}
+
 static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, ScrSidx *e,
                                                             size_t u16, bool *mid) {
+  if (scr_sidx_offsets(s, e)) {
+    *mid = false;
+    if (u16 >= e->offsets->n) return s->len;
+    size_t cb = scr_sidx_offset_at(e, u16);
+    size_t cu = u16;
+    if (u16 > 0 && scr_sidx_offset_at(e, u16 - 1) == cb) {
+      *mid = true; /* the low half of an astral character */
+      cu = u16 - 1;
+    }
+    e->cu = cu;
+    e->cb = cb;
+    return cb;
+  }
+  /* Within a short distance of the hot cursor (a scanner
+   * taking slices near its position), walk from that cursor: no index
+   * search, extension or ASCII-run growth is cheaper. */
+  if (e->cb <= s->len && scr_sidx_abs_diff(e->cu, u16) <= SCR_SIDX_NEAR_UNITS)
+    return scr_sidx_walk_from(s, e, e->cu, e->cb, u16, mid, false);
  /* A short step forward from the hot cursor (a sequential scan of
   * non-ASCII text) walks from the cursor itself. */
  if (u16 >= e->cu && u16 - e->cu <= 8) {
@@ -1665,48 +1789,7 @@ static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, Sc
  }
  scr_sidx_materialize_identity_prefix(s, e);
  ScrSidxPoint near = scr_sidx_near_u16(s, e, u16);
-  size_t cu = near.cu, cb = near.cb;
-  while (cu > u16) {
-    /* Eight bytes starting inside a sequence can reach back at most three
-     * more bytes to its lead. Such a span contains at most nine UTF-16
-     * units (seven ASCII bytes plus an astral pair), so it cannot overshoot
-     * a target at least nine units away. */
-    if (cu - u16 >= 9 && cb >= 8) {
-      size_t start = cb - 8;
-      while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80)
-        start--;
-      cu -= scr_utf16_units_span(s->data + start, cb - start, NULL);
-      cb = start;
-    } else {
-      scr_sidx_back(s, &cu, &cb);
-    }
-  }
-  bool m = false;
-  while (cu < u16 && cb < s->len) {
-    if (u16 - cu >= 9 && s->len - cb >= 8) {
-      size_t end = cb + 8;
-      while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80)
-        end++;
-      cu += scr_utf16_units_span(s->data + cb, end - cb, NULL);
-      cb = end;
-      continue;
-    }
-    SCR_SIDX_STEP();
-    size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]);
-    size_t w = seq == 4 ? 2 : 1;
-    if (cu + w > u16) { /* u16 lands between the halves of an astral char */
-      m = true;
-      break;
-    }
-    cu += w;
-    cb += seq;
-  }
-  e->cu = cu;
-  e->cb = cb;
-  if (cb < s->len && (unsigned char)s->data[cb] < 0x80)
-    scr_sidx_ascii_run(s, e);
-  *mid = m;
-  return cb;
+  return scr_sidx_walk_from(s, e, near.cu, near.cb, u16, mid, true);
 }

 static size_t scr_byte_to_u16_walk(const ScrStr *s, ScrSidx *e, size_t byte_off);
@@ -1875,10 +1958,17 @@ static uint16_t *scr_str_units(const ScrStr *s, size_t *n) {
 * megabytes, which scanners read in full. The array is freed with the
 * entry, and an append drops it (scr_sidx_concat_append). */
 #define SCR_SIDX_UNITS_MAX ((size_t)1 << 24)
+static void scr_sidx_take_units(const ScrStr *s, ScrSidx *e) {
+  size_t n = 0;
+  e->units = scr_str_units(s, &n);
+  e->nunits = (uint32_t)n;
+}
+
 static bool scr_sidx_units(const ScrStr *s, ScrSidx *e) {
  if (!e->units) {
-    if (++e->reads < 32 || e->reads * 16 < s->len || s->len > SCR_SIDX_UNITS_MAX) return false;
-    e->units = scr_str_units(s, &e->nunits);
+    if (s->len > SCR_SIDX_UNITS_MAX || ++e->reads < 32 || (size_t)e->reads * 16 < s->len)
+      return false;
+    scr_sidx_take_units(s, e);
  }
  /* Publish the scan's receiver for emitted reads (scr_runtime.h). */
  if (scr_str_scan.s != s) {
@@ -1905,7 +1995,7 @@ void scr_str_scan_units_for(ScrStr *s, double from, double to) {
      (void)scr_sidx_units(s, e);
      return;
    }
-    e->units = scr_str_units(s, &e->nunits);
+    scr_sidx_take_units(s, e);
  }
  if (scr_str_scan.s != s) {
    scr_str_scan.s = s;
@@ -1980,8 +2070,9 @@ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) {
  size_t start_byte = scr_u16_to_byte_c(s, e, start16, &mid);
  size_t max_start = s->len - needle->len;
  if (start_byte < max_start) max_start = start_byte;
+  char first = needle->data[0];
  for (size_t i = max_start + 1; i-- > 0;) {
-    if (memcmp(s->data + i, needle->data, needle->len) == 0) {
+    if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) {
      return (double)scr_byte_to_u16_c(s, e, i);
    }
  }
@@ -0,0 +1,102 @@
+// UTF-16 index operations on long non-ASCII strings: slices and substrings
+// walking forward and backward, interleaved on two strings, at astral
+// characters and their halves, with negative and out-of-range indexes;
+// lastIndexOf, startsWith and endsWith with positions; and normalize on
+// text already in each form, composed and decomposed text, Hangul,
+// compatibility characters and combining marks.
+
+const unit = "Ünïcödé tëxt with ß and 日本語 and émoji 😀 mixed in, ";
+const big = unit.repeat(3000);
+const other = "ascii words then 中文 and 🎉🎉 and ü, ".repeat(4000);
+
+function digest(text: string): number {
+  let h = 0;
+  for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0;
+  return h;
+}
+
+let forward = 0;
+for (let i = 0; i + 30 < big.length; i += 7) {
+  forward = (forward * 33 + digest(big.slice(i, i + 20)) + digest(big.substring(i + 3, i + 9))) >>> 0;
+}
+let backward = 0;
+for (let i = big.length - 1; i > 30; i -= 11) {
+  backward = (backward * 33 + digest(big.slice(i - 25, i)) + big.substring(i, i - 5).length) >>> 0;
+}
+let interleaved = 0;
+for (let i = 0; i + 40 < other.length && i + 40 < big.length; i += 13) {
+  interleaved = (interleaved * 33 + digest(big.slice(i, i + 9)) + digest(other.slice(i + 1, i + 40))) >>> 0;
+}
+console.log(big.length, forward, backward, interleaved);
+
+// Indexes at astral characters and their halves.
+const emojiAt = big.indexOf("😀", 100000);
+console.log(
+  emojiAt,
+  JSON.stringify(big.slice(emojiAt, emojiAt + 1)),
+  JSON.stringify(big.slice(emojiAt + 1, emojiAt + 3)),
+  JSON.stringify(big.substring(emojiAt - 1, emojiAt + 1)),
+  big.charCodeAt(emojiAt + 1),
+  JSON.stringify(big.at(emojiAt + 1)),
+  JSON.stringify(big.charAt(emojiAt)),
+);
+console.log(
+  JSON.stringify(big.slice(-5)),
+  JSON.stringify(big.slice(-30, -20)),
+  big.slice(big.length - 3, big.length + 10).length,
+  JSON.stringify(big.substring(-10, 4)),
+  big.slice(5, 2).length,
+  JSON.stringify(big.slice(NaN, 3)),
+);
+for (const at of [0, 1, 52, 99999, 150000, big.length - 1, big.length, -1, -2, -big.length]) {
+  console.log(at, JSON.stringify(big.at(at)), JSON.stringify(big.charAt(at)), big.charCodeAt(at));
+}
+
+// lastIndexOf, startsWith and endsWith with positions.
+console.log(
+  big.lastIndexOf("日本"),
+  big.lastIndexOf("日本", 100000),
+  big.lastIndexOf("😀", 50),
+  big.lastIndexOf("absent"),
+  big.lastIndexOf("é", 0),
+  big.lastIndexOf("Ü"),
+  big.lastIndexOf(""),
+  big.lastIndexOf("", 10),
+);
+let starts = 0;
+let ends = 0;
+for (let i = 0; i < big.length; i += 17) {
+  if (big.startsWith("tëxt", i)) starts++;
+  if (big.endsWith("😀", i)) ends++;
+}
+console.log(starts, ends, big.startsWith("Ü"), big.endsWith("in, "), big.startsWith("ünï"), big.endsWith("in, ", big.length - 1));
+
+// normalize.
+const samples = [
+  "plain ascii",
+  "précomposé café naïve",
+  "décomposé café",
+  "日本語のテキスト 한국어 문장",
+  "각 각",
+  "ｆｕｌｌｗｉｄｔｈ ｶﾀｶﾅ ﬁ ① ℌ ²",
+  "ậ ậ ͅ",
+  "Ω Å Å Ω K",
+  "が が か゚",
+  "emoji 😀 and \ud800 lone",
+  unit,
+];
+for (const sample of samples) {
+  const forms = ["NFC", "NFD", "NFKC", "NFKD"].map((form) => sample.normalize(form));
+  console.log(
+    forms.map((text) => `${text.length}:${digest(text)}`).join(" "),
+    forms.map((text) => text === sample).join(","),
+  );
+}
+const longNfc = big.normalize();
+const longNfd = big.normalize("NFD");
+console.log(longNfc === big, longNfc.length, longNfd.length, digest(longNfd.slice(0, 100)), longNfd.normalize("NFC") === big);
+try {
+  "x".normalize("NFX");
+} catch (error) {
+  console.log((error as Error).name, (error as Error).message);
+}
@@ -0,0 +1 @@
+{ "plain": 4784, "san": 2792 }