From 5f76d41a7181348e2e14d60a2ec3377f825bc3d5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:57:11 -0500 Subject: [PATCH] Keep U+11A7 after a Hangul LV syllable in NFC and NFKC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libunicode composes an LV syllable with U+11A7 as if it were a trailing consonant (U+11A8..U+11C2), which drops it: "가ᆧ".normalize() gave "가" where Node gives "가ᆧ". U+11A7 is a class-0 vowel that composes with nothing, so for the composing forms the code points between occurrences of it normalize on their own and it is kept as it is. The guard sits in front of the vendored library (scr_unicode_normalize), for String.prototype.normalize and the NFC step of IDNA mapping. 8009-normalize-hangul-jungseong-o-yae prints the code points of all four forms for U+11A7 after LV and LVT syllables, conjoining jamo, other starters and combining marks. --- ...9-normalize-hangul-jungseong-o-yae.ts.json | 6 ++ packages/runtime/src/scr_regex.c | 55 ++++++++++++++++++- .../8009-normalize-hangul-jungseong-o-yae.ts | 28 ++++++++++ ...9-normalize-hangul-jungseong-o-yae.ts.json | 1 + 4 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json create mode 100644 tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts create mode 100644 tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json new file mode 100644 index 000000000..872a85396 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 2a98e92c6..37f94bd65 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -2418,6 +2418,57 @@ static void *scr_normalize_realloc(void *opaque, void *ptr, size_t size) { return realloc(ptr, size); } +/* unicode_normalize, except that U+11A7 never composes. libunicode composes + * an LV syllable with U+11A7 as a trailing consonant (those are U+11A8 to + * U+11C2), which drops it: "\uAC00\u11A7" became "\uAC00". U+11A7 is a + * class-0 vowel that composes with nothing, so for the composing forms the + * code points between occurrences of it normalize on their own and it is + * kept as it is, as in Node. */ +static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len, + UnicodeNormalizationEnum mode) { + int at = 0; + if (mode == UNICODE_NFC || mode == UNICODE_NFKC) + while (at < len && src[at] != 0x11A7) at++; + else + at = len; + if (at == len) return unicode_normalize(pdst, src, len, mode, NULL, scr_normalize_realloc); + uint32_t *out = NULL; + size_t count = 0, cap = 0; + int start = 0; + for (int i = 0; i <= len; i++) { + if (i < len && src[i] != 0x11A7) continue; + uint32_t *part = NULL; + int n = 0; + if (i > start) { + n = unicode_normalize(&part, src + start, i - start, mode, NULL, scr_normalize_realloc); + if (n < 0) { + free(out); + return -1; + } + } + size_t need = count + (size_t)n + 1; + if (need > cap) { + size_t grown = cap ? cap * 2 : 16; + if (grown < need) grown = need; + uint32_t *bigger = realloc(out, grown * sizeof *out); + if (!bigger) { + free(part); + free(out); + return -1; + } + out = bigger; + cap = grown; + } + if (n > 0) memcpy(out + count, part, (size_t)n * sizeof *out); + count += (size_t)n; + free(part); + if (i < len) out[count++] = 0x11A7; + start = i + 1; + } + *pdst = out; + return (int)count; +} + ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { UnicodeNormalizationEnum mode; if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC; @@ -2442,7 +2493,7 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { offset += step; } uint32_t *output = NULL; - int count = unicode_normalize(&output, input, length, mode, NULL, scr_normalize_realloc); + int count = scr_unicode_normalize(&output, input, length, mode); free(input); if (count < 0) scr_regex_oom(); ScrJsonBuf buffer; @@ -2455,7 +2506,7 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { uint32_t *scr_nfc_code_points(const uint32_t *input, size_t length, size_t *out_len) { if (length > INT32_MAX) scr_regex_oom(); uint32_t *output = NULL; - int count = unicode_normalize(&output, input, (int)length, UNICODE_NFC, NULL, scr_normalize_realloc); + int count = scr_unicode_normalize(&output, input, (int)length, UNICODE_NFC); if (count < 0) scr_regex_oom(); *out_len = (size_t)count; return output; diff --git a/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts b/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts new file mode 100644 index 000000000..b7cb5418a --- /dev/null +++ b/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts @@ -0,0 +1,28 @@ +// U+11A7 (HANGUL JUNGSEONG O-YAE) is a vowel that composes with nothing: +// after an LV syllable, after a leading consonant, and around trailing +// consonants it stays as it is in every normalization form, as in Node. + +const samples = [ + "가ᆧ", + "가ᆧ", + "가ᆧᆨ", + "각ᆧ", + "ᄀᆧ", + "ᆧ", + "ᆧᆧ", + "x가ᆧy각z", + "café 가ᆧ é", + "ｶﾞᆧ가", +]; + +function points(text: string): string { + return Array.from(text, (ch) => ch.codePointAt(0)!.toString(16)).join("."); +} + +for (const sample of samples) { + console.log( + points(sample), + ["NFC", "NFD", "NFKC", "NFKD"].map((form) => points(sample.normalize(form))).join(" "), + ); +} +console.log("가ᆧ".normalize() === "가ᆧ", "가ᆧ".normalize().length); diff --git a/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json b/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json new file mode 100644 index 000000000..6c20e6ad7 --- /dev/null +++ b/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json @@ -0,0 +1 @@ +{ "plain": 2503, "san": 2110 }