Decode short non-ASCII strings directly in charCodeAt (#2388)

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/short-char-codes.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/short-char-codes.ts"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_string.c

@@ -2131,6 +2131,39 @@ void scr_str_note_utf16_len(ScrStr *s, size_t units) {
   *scr_u16len_slot(s) = (ScrU16Len){s, units};
 }
 
+/* A non-ASCII receiver up to this many bytes (a character, a short word)
+ * is decoded from its start on each read, as scr_str_utf16_len counts a
+ * short receiver: an index entry for it would cost more than the walk, and
+ * would evict the cursor of a long receiver being scanned meanwhile. */
+#define SCR_STR_DECODE_SHORT 16
+
+/* The UTF-16 unit at index i of a short receiver, or NaN past its end. */
+static double scr_str_char_code_short(const ScrStr *s, size_t i) {
+  size_t b = 0;
+  while (b < s->len) {
+    unsigned char c = (unsigned char)s->data[b];
+    if (c < 0x80) {
+      if (i == 0) return (double)c;
+      i--;
+      b++;
+      continue;
+    }
+    size_t adv;
+    uint32_t cp = scr_utf8_decode(s->data + b, &adv);
+    if (cp < 0x10000) {
+      if (i == 0) return (double)cp;
+      i--;
+    } else {
+      uint32_t v = cp - 0x10000;
+      if (i == 0) return (double)(0xD800 + (v >> 10));
+      if (i == 1) return (double)(0xDC00 + (v & 0x3FF));
+      i -= 2;
+    }
+    b += adv;
+  }
+  return NAN;
+}
+
 double scr_str_char_code_at(ScrStr *s, double i) {
   /* A sequential scan of a non-ASCII receiver stays inside the bounded
    * ASCII span cached at its last resolved position. Its unit offset is an
@@ -2169,6 +2202,7 @@ double scr_str_char_code_at(ScrStr *s, double i) {
   /* UTF-16 length <= byte length always, so this also fences the cast. */
   if (idx >= (double)s->len) return NAN;
   if (s->ascii) return (double)(unsigned char)s->data[(size_t)idx];
+  if (s->len <= SCR_STR_DECODE_SHORT) return scr_str_char_code_short(s, (size_t)idx);
   ScrSidx *e = scr_sidx(s);
   if (s->ascii || e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx];
   if (scr_sidx_units(s, e)) return idx < (double)e->nunits ? (double)e->units[(size_t)idx] : NAN;

packages/runtime/test/test_string.c

@@ -605,10 +605,15 @@ static void sparse_slice_churn_asserts(void) {
 static void index_activity_asserts(void) {
   scr_sidx_test_reset_cache();
   ScrStr *strings[9];
+  /* A receiver of up to 16 bytes is decoded on each indexed read. */
+  ScrStr *shortest = scr_str_new("\xc3\xa9x", 3);
+  if (scr_str_char_code_at(shortest, 1) != 'x' || scr_sidx_test_active() != 0)
+    sidx_fail("short indexed read claimed a cursor");
+  scr_str_release(shortest);
   for (size_t i = 0; i < 9; i++) {
-    strings[i] = scr_str_new("\xc3\xa9x", 3);
+    strings[i] = scr_str_new("\xc3\xa9xxxxxxxxxxxxxxxxx", 19);
     /* Short lengths count directly; an indexed read claims a cursor. */
-    if (scr_str_utf16_len(strings[i]) != 2 || scr_sidx_test_active() != (i < 4 ? i : 4))
+    if (scr_str_utf16_len(strings[i]) != 18 || scr_sidx_test_active() != (i < 4 ? i : 4))
       sidx_fail("short length claimed a cursor");
     if (scr_str_char_code_at(strings[i], 1) != 'x') sidx_fail("cursor activation read");
     if (scr_sidx_test_active() != (i < 4 ? i + 1 : 4))
@@ -616,7 +621,7 @@ static void index_activity_asserts(void) {
   }
   for (size_t i = 0; i < 9; i++) scr_str_release(strings[i]);
   if (scr_sidx_test_active() != 0) sidx_fail("last cursor release stayed active");
-  ScrStr *s = scr_str_new("\xe4\xb8\xad", 3);
+  ScrStr *s = scr_str_new("\xe4\xb8\xadxxxxxxxxxxxxxxxx", 19);
   (void)scr_str_utf16_len(s);
   scr_sidx_test_reset_cache();
   if (scr_sidx_test_active() != 0) sidx_fail("cache reset stayed active");

tests/corpus/short-char-codes.ts

@@ -0,0 +1,50 @@
+// charCodeAt and codePointAt on short non-ASCII strings: characters of each
+// UTF-8 length, astral pairs, lone surrogates, out-of-range and non-integer
+// indexes, strings at the 16-byte boundary, and a long receiver scanned in
+// between.
+
+const samples = [
+  "é", "ß", "日", "語", "😀", "🚀", "a", "",
+  "café", "naïve", "straße", "日本語", "テキスト", "한국어",
+  "a😀b", "😀😀", "xéy日z",
+  "\ud800", "\udc00", "a\ud800b", "\ud83d", "\ude00x",
+  "0123456789abcdé", "0123456789abcdeé", "0123456789abcdefé", "ééééééé", "éééééééé", "ééééééééé",
+];
+
+function codes(s: string): string {
+  const out: string[] = [];
+  for (let i = -1; i <= s.length + 1; i++) out.push(String(s.charCodeAt(i)));
+  out.push(String(s.charCodeAt(0.5)), String(s.charCodeAt(NaN)), String(s.charCodeAt(1.9)));
+  return out.join(",");
+}
+
+function points(s: string): string {
+  const out: string[] = [];
+  for (let i = -1; i <= s.length + 1; i++) out.push(String(s.codePointAt(i)));
+  return out.join(",");
+}
+
+for (const s of samples) {
+  console.log(JSON.stringify(s), s.length, codes(s));
+  console.log("  points", points(s));
+}
+
+// A long receiver scanned while short ones are read.
+let long = "";
+for (let i = 0; i < 200; i++) long += i % 3 === 0 ? "é" : i % 3 === 1 ? "日" : "x";
+let sum = 0;
+for (let i = 0; i < long.length; i++) {
+  sum += long.charCodeAt(i);
+  sum += samples[i % samples.length]!.charCodeAt(0) || 0;
+}
+console.log("long", long.length, sum);
+
+// Characters of a for-of loop classified by their code points.
+let letters = 0;
+let others = 0;
+for (const c of "the café, 日本 😀 naïve!") {
+  const code = c.codePointAt(0)!;
+  if ((code >= 0x61 && code <= 0x7a) || code >= 0xc0) letters++;
+  else others++;
+}
+console.log("letters", letters, "others", others);

tests/harness/corpus-costs/short-char-codes.ts.json

@@ -0,0 +1 @@
+{ "plain": 4800, "san": 4600 }