diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/short-char-codes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/short-char-codes.ts.json new file mode 100644 index 000000000..79390f594 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/short-char-codes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/short-char-codes.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 62a155bd1..cfbb0f980 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -2131,6 +2131,39 @@ void scr_str_note_utf16_len(ScrStr *s, size_t units) { *scr_u16len_slot(s) = (ScrU16Len){s, units}; } +/* A non-ASCII receiver up to this many bytes (a character, a short word) + * is decoded from its start on each read, as scr_str_utf16_len counts a + * short receiver: an index entry for it would cost more than the walk, and + * would evict the cursor of a long receiver being scanned meanwhile. */ +#define SCR_STR_DECODE_SHORT 16 + +/* The UTF-16 unit at index i of a short receiver, or NaN past its end. */ +static double scr_str_char_code_short(const ScrStr *s, size_t i) { + size_t b = 0; + while (b < s->len) { + unsigned char c = (unsigned char)s->data[b]; + if (c < 0x80) { + if (i == 0) return (double)c; + i--; + b++; + continue; + } + size_t adv; + uint32_t cp = scr_utf8_decode(s->data + b, &adv); + if (cp < 0x10000) { + if (i == 0) return (double)cp; + i--; + } else { + uint32_t v = cp - 0x10000; + if (i == 0) return (double)(0xD800 + (v >> 10)); + if (i == 1) return (double)(0xDC00 + (v & 0x3FF)); + i -= 2; + } + b += adv; + } + return NAN; +} + double scr_str_char_code_at(ScrStr *s, double i) { /* A sequential scan of a non-ASCII receiver stays inside the bounded * ASCII span cached at its last resolved position. Its unit offset is an @@ -2169,6 +2202,7 @@ double scr_str_char_code_at(ScrStr *s, double i) { /* UTF-16 length <= byte length always, so this also fences the cast. */ if (idx >= (double)s->len) return NAN; if (s->ascii) return (double)(unsigned char)s->data[(size_t)idx]; + if (s->len <= SCR_STR_DECODE_SHORT) return scr_str_char_code_short(s, (size_t)idx); ScrSidx *e = scr_sidx(s); if (s->ascii || e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx]; if (scr_sidx_units(s, e)) return idx < (double)e->nunits ? (double)e->units[(size_t)idx] : NAN; diff --git a/packages/runtime/test/test_string.c b/packages/runtime/test/test_string.c index d0dbebf0d..5a3bdfe5f 100644 --- a/packages/runtime/test/test_string.c +++ b/packages/runtime/test/test_string.c @@ -605,10 +605,15 @@ static void sparse_slice_churn_asserts(void) { static void index_activity_asserts(void) { scr_sidx_test_reset_cache(); ScrStr *strings[9]; + /* A receiver of up to 16 bytes is decoded on each indexed read. */ + ScrStr *shortest = scr_str_new("\xc3\xa9x", 3); + if (scr_str_char_code_at(shortest, 1) != 'x' || scr_sidx_test_active() != 0) + sidx_fail("short indexed read claimed a cursor"); + scr_str_release(shortest); for (size_t i = 0; i < 9; i++) { - strings[i] = scr_str_new("\xc3\xa9x", 3); + strings[i] = scr_str_new("\xc3\xa9xxxxxxxxxxxxxxxxx", 19); /* Short lengths count directly; an indexed read claims a cursor. */ - if (scr_str_utf16_len(strings[i]) != 2 || scr_sidx_test_active() != (i < 4 ? i : 4)) + if (scr_str_utf16_len(strings[i]) != 18 || scr_sidx_test_active() != (i < 4 ? i : 4)) sidx_fail("short length claimed a cursor"); if (scr_str_char_code_at(strings[i], 1) != 'x') sidx_fail("cursor activation read"); if (scr_sidx_test_active() != (i < 4 ? i + 1 : 4)) @@ -616,7 +621,7 @@ static void index_activity_asserts(void) { } for (size_t i = 0; i < 9; i++) scr_str_release(strings[i]); if (scr_sidx_test_active() != 0) sidx_fail("last cursor release stayed active"); - ScrStr *s = scr_str_new("\xe4\xb8\xad", 3); + ScrStr *s = scr_str_new("\xe4\xb8\xadxxxxxxxxxxxxxxxx", 19); (void)scr_str_utf16_len(s); scr_sidx_test_reset_cache(); if (scr_sidx_test_active() != 0) sidx_fail("cache reset stayed active"); diff --git a/tests/corpus/short-char-codes.ts b/tests/corpus/short-char-codes.ts new file mode 100644 index 000000000..14c858bb5 --- /dev/null +++ b/tests/corpus/short-char-codes.ts @@ -0,0 +1,50 @@ +// charCodeAt and codePointAt on short non-ASCII strings: characters of each +// UTF-8 length, astral pairs, lone surrogates, out-of-range and non-integer +// indexes, strings at the 16-byte boundary, and a long receiver scanned in +// between. + +const samples = [ + "é", "ß", "日", "語", "😀", "🚀", "a", "", + "café", "naïve", "straße", "日本語", "テキスト", "한국어", + "a😀b", "😀😀", "xéy日z", + "\ud800", "\udc00", "a\ud800b", "\ud83d", "\ude00x", + "0123456789abcdé", "0123456789abcdeé", "0123456789abcdefé", "ééééééé", "éééééééé", "ééééééééé", +]; + +function codes(s: string): string { + const out: string[] = []; + for (let i = -1; i <= s.length + 1; i++) out.push(String(s.charCodeAt(i))); + out.push(String(s.charCodeAt(0.5)), String(s.charCodeAt(NaN)), String(s.charCodeAt(1.9))); + return out.join(","); +} + +function points(s: string): string { + const out: string[] = []; + for (let i = -1; i <= s.length + 1; i++) out.push(String(s.codePointAt(i))); + return out.join(","); +} + +for (const s of samples) { + console.log(JSON.stringify(s), s.length, codes(s)); + console.log(" points", points(s)); +} + +// A long receiver scanned while short ones are read. +let long = ""; +for (let i = 0; i < 200; i++) long += i % 3 === 0 ? "é" : i % 3 === 1 ? "日" : "x"; +let sum = 0; +for (let i = 0; i < long.length; i++) { + sum += long.charCodeAt(i); + sum += samples[i % samples.length]!.charCodeAt(0) || 0; +} +console.log("long", long.length, sum); + +// Characters of a for-of loop classified by their code points. +let letters = 0; +let others = 0; +for (const c of "the café, 日本 😀 naïve!") { + const code = c.codePointAt(0)!; + if ((code >= 0x61 && code <= 0x7a) || code >= 0xc0) letters++; + else others++; +} +console.log("letters", letters, "others", others); diff --git a/tests/harness/corpus-costs/short-char-codes.ts.json b/tests/harness/corpus-costs/short-char-codes.ts.json new file mode 100644 index 000000000..621e15852 --- /dev/null +++ b/tests/harness/corpus-costs/short-char-codes.ts.json @@ -0,0 +1 @@ +{ "plain": 4800, "san": 4600 }