From 820c4eaf77605e78de723682822d4d0fd4488ba9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 14 Oct 2026 23:49:11 -0500 Subject: [PATCH] =?UTF-8?q?Match=20Node's=20collation=20for=20l=C2=B7,=20U?= =?UTF-8?q?+0130=20and=20ligatures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated localeCompare table now agrees with Node on spellings that tie at the primary and secondary levels. "l·" and "L·" collate as one unit, equal to U+0140 and U+013F, through a contraction table. Precomposed characters take their decompositions' elements after the case weights are assigned, so U+0130 equals I + U+0307. Ligatures whose spelled-out letters tie with them (ij and ij) get the case weight Node places them at among their first letter's case forms, and ligatures with their own secondary (Æ) keep their own case. The generator's check now also compares every covered character's decompositions, case forms and ligature spellings in four contexts. A per-thread class table lets the lazy primary comparison find characters that need element lists with one lookup. --- .../5460-locale-compare-contractions.ts.json | 6 + .../5461-locale-compare-equivalents.ts.json | 6 + .../5462-locale-compare-probe-sweep.ts.json | 6 + packages/runtime/src/scr_collation_data.h | 650 +++++++++--------- packages/runtime/src/scr_string.c | 157 +++-- scripts/gen-collation-table.mjs | 156 ++++- .../5460-locale-compare-contractions.ts | 17 + .../corpus/5461-locale-compare-equivalents.ts | 29 + .../corpus/5462-locale-compare-probe-sweep.ts | 34 + 9 files changed, 664 insertions(+), 397 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json create mode 100644 tests/corpus/5460-locale-compare-contractions.ts create mode 100644 tests/corpus/5461-locale-compare-equivalents.ts create mode 100644 tests/corpus/5462-locale-compare-probe-sweep.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json new file mode 100644 index 000000000..c4d1f9e8f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5460-locale-compare-contractions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json new file mode 100644 index 000000000..dedaced3d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5461-locale-compare-equivalents.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json new file mode 100644 index 000000000..d6199005c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5462-locale-compare-probe-sweep.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_collation_data.h b/packages/runtime/src/scr_collation_data.h index 8004ca861..61972a4a7 100644 --- a/packages/runtime/src/scr_collation_data.h +++ b/packages/runtime/src/scr_collation_data.h @@ -4,341 +4,342 @@ * three collation elements {primary, secondary, tertiary, combining class * rank}; zero primaries are skipped at the primary level, and runs of * marks with a nonzero class rank are put in canonical order before - * comparing. n == 255 marks an uncovered code point. */ + * comparing. n == 255 marks an uncovered code point. A pair listed in + * scr_coll_contractions collates as its one entry. */ #ifndef SCR_COLLATION_DATA_H #define SCR_COLLATION_DATA_H #include typedef struct { uint16_t p; uint8_t s; uint8_t t; uint8_t ccc; } ScrCollElem; typedef struct { uint8_t n; ScrCollElem e[3]; } ScrCollEntry; static const ScrCollEntry scr_coll_0009[5] = { - {1, {{1, 1, 1, 0}}}, /* U+0009 */ - {1, {{2, 1, 1, 0}}}, /* U+000A */ - {1, {{3, 1, 1, 0}}}, /* U+000B */ - {1, {{4, 1, 1, 0}}}, /* U+000C */ - {1, {{5, 1, 1, 0}}}, /* U+000D */ + {1, {{1, 1, 2, 0}}}, /* U+0009 */ + {1, {{2, 1, 2, 0}}}, /* U+000A */ + {1, {{3, 1, 2, 0}}}, /* U+000B */ + {1, {{4, 1, 2, 0}}}, /* U+000C */ + {1, {{5, 1, 2, 0}}}, /* U+000D */ }; static const ScrCollEntry scr_coll_0020[95] = { - {1, {{6, 1, 1, 0}}}, /* U+0020 */ - {1, {{12, 1, 1, 0}}}, /* U+0021 */ - {1, {{19, 1, 1, 0}}}, /* U+0022 */ - {1, {{35, 1, 1, 0}}}, /* U+0023 */ - {1, {{59, 1, 1, 0}}}, /* U+0024 */ - {1, {{36, 1, 1, 0}}}, /* U+0025 */ - {1, {{34, 1, 1, 0}}}, /* U+0026 */ - {1, {{18, 1, 1, 0}}}, /* U+0027 */ - {1, {{22, 1, 1, 0}}}, /* U+0028 */ - {1, {{23, 1, 1, 0}}}, /* U+0029 */ - {1, {{31, 1, 1, 0}}}, /* U+002A */ - {1, {{46, 1, 1, 0}}}, /* U+002B */ - {1, {{9, 1, 1, 0}}}, /* U+002C */ - {1, {{8, 1, 1, 0}}}, /* U+002D */ - {1, {{16, 1, 1, 0}}}, /* U+002E */ - {1, {{32, 1, 1, 0}}}, /* U+002F */ - {1, {{62, 1, 1, 0}}}, /* U+0030 */ - {1, {{63, 1, 1, 0}}}, /* U+0031 */ - {1, {{64, 1, 1, 0}}}, /* U+0032 */ - {1, {{65, 1, 1, 0}}}, /* U+0033 */ - {1, {{66, 1, 1, 0}}}, /* U+0034 */ - {1, {{67, 1, 1, 0}}}, /* U+0035 */ - {1, {{68, 1, 1, 0}}}, /* U+0036 */ - {1, {{69, 1, 1, 0}}}, /* U+0037 */ - {1, {{70, 1, 1, 0}}}, /* U+0038 */ - {1, {{71, 1, 1, 0}}}, /* U+0039 */ - {1, {{11, 1, 1, 0}}}, /* U+003A */ - {1, {{10, 1, 1, 0}}}, /* U+003B */ - {1, {{50, 1, 1, 0}}}, /* U+003C */ - {1, {{51, 1, 1, 0}}}, /* U+003D */ - {1, {{52, 1, 1, 0}}}, /* U+003E */ - {1, {{14, 1, 1, 0}}}, /* U+003F */ - {1, {{30, 1, 1, 0}}}, /* U+0040 */ - {1, {{72, 1, 2, 0}}}, /* U+0041 */ - {1, {{73, 1, 2, 0}}}, /* U+0042 */ - {1, {{74, 1, 2, 0}}}, /* U+0043 */ - {1, {{75, 1, 2, 0}}}, /* U+0044 */ - {1, {{76, 1, 2, 0}}}, /* U+0045 */ - {1, {{77, 1, 2, 0}}}, /* U+0046 */ - {1, {{78, 1, 2, 0}}}, /* U+0047 */ - {1, {{79, 1, 2, 0}}}, /* U+0048 */ - {1, {{80, 1, 2, 0}}}, /* U+0049 */ - {1, {{82, 1, 2, 0}}}, /* U+004A */ - {1, {{83, 1, 2, 0}}}, /* U+004B */ - {1, {{84, 1, 2, 0}}}, /* U+004C */ - {1, {{85, 1, 2, 0}}}, /* U+004D */ - {1, {{86, 1, 2, 0}}}, /* U+004E */ - {1, {{88, 1, 2, 0}}}, /* U+004F */ - {1, {{89, 1, 2, 0}}}, /* U+0050 */ - {1, {{90, 1, 2, 0}}}, /* U+0051 */ - {1, {{92, 1, 2, 0}}}, /* U+0052 */ - {1, {{93, 1, 2, 0}}}, /* U+0053 */ - {1, {{94, 1, 2, 0}}}, /* U+0054 */ - {1, {{96, 1, 2, 0}}}, /* U+0055 */ - {1, {{97, 1, 2, 0}}}, /* U+0056 */ - {1, {{98, 1, 2, 0}}}, /* U+0057 */ - {1, {{99, 1, 2, 0}}}, /* U+0058 */ - {1, {{100, 1, 2, 0}}}, /* U+0059 */ - {1, {{101, 1, 2, 0}}}, /* U+005A */ - {1, {{24, 1, 1, 0}}}, /* U+005B */ - {1, {{33, 1, 1, 0}}}, /* U+005C */ - {1, {{25, 1, 1, 0}}}, /* U+005D */ - {1, {{39, 1, 1, 0}}}, /* U+005E */ - {1, {{7, 1, 1, 0}}}, /* U+005F */ - {1, {{37, 1, 1, 0}}}, /* U+0060 */ - {1, {{72, 1, 1, 0}}}, /* U+0061 */ - {1, {{73, 1, 1, 0}}}, /* U+0062 */ - {1, {{74, 1, 1, 0}}}, /* U+0063 */ - {1, {{75, 1, 1, 0}}}, /* U+0064 */ - {1, {{76, 1, 1, 0}}}, /* U+0065 */ - {1, {{77, 1, 1, 0}}}, /* U+0066 */ - {1, {{78, 1, 1, 0}}}, /* U+0067 */ - {1, {{79, 1, 1, 0}}}, /* U+0068 */ - {1, {{80, 1, 1, 0}}}, /* U+0069 */ - {1, {{82, 1, 1, 0}}}, /* U+006A */ - {1, {{83, 1, 1, 0}}}, /* U+006B */ - {1, {{84, 1, 1, 0}}}, /* U+006C */ - {1, {{85, 1, 1, 0}}}, /* U+006D */ - {1, {{86, 1, 1, 0}}}, /* U+006E */ - {1, {{88, 1, 1, 0}}}, /* U+006F */ - {1, {{89, 1, 1, 0}}}, /* U+0070 */ - {1, {{90, 1, 1, 0}}}, /* U+0071 */ - {1, {{92, 1, 1, 0}}}, /* U+0072 */ - {1, {{93, 1, 1, 0}}}, /* U+0073 */ - {1, {{94, 1, 1, 0}}}, /* U+0074 */ - {1, {{96, 1, 1, 0}}}, /* U+0075 */ - {1, {{97, 1, 1, 0}}}, /* U+0076 */ - {1, {{98, 1, 1, 0}}}, /* U+0077 */ - {1, {{99, 1, 1, 0}}}, /* U+0078 */ - {1, {{100, 1, 1, 0}}}, /* U+0079 */ - {1, {{101, 1, 1, 0}}}, /* U+007A */ - {1, {{26, 1, 1, 0}}}, /* U+007B */ - {1, {{54, 1, 1, 0}}}, /* U+007C */ - {1, {{27, 1, 1, 0}}}, /* U+007D */ - {1, {{56, 1, 1, 0}}}, /* U+007E */ + {1, {{6, 1, 2, 0}}}, /* U+0020 */ + {1, {{12, 1, 2, 0}}}, /* U+0021 */ + {1, {{19, 1, 2, 0}}}, /* U+0022 */ + {1, {{35, 1, 2, 0}}}, /* U+0023 */ + {1, {{59, 1, 2, 0}}}, /* U+0024 */ + {1, {{36, 1, 2, 0}}}, /* U+0025 */ + {1, {{34, 1, 2, 0}}}, /* U+0026 */ + {1, {{18, 1, 2, 0}}}, /* U+0027 */ + {1, {{22, 1, 2, 0}}}, /* U+0028 */ + {1, {{23, 1, 2, 0}}}, /* U+0029 */ + {1, {{31, 1, 2, 0}}}, /* U+002A */ + {1, {{46, 1, 2, 0}}}, /* U+002B */ + {1, {{9, 1, 2, 0}}}, /* U+002C */ + {1, {{8, 1, 2, 0}}}, /* U+002D */ + {1, {{16, 1, 2, 0}}}, /* U+002E */ + {1, {{32, 1, 2, 0}}}, /* U+002F */ + {1, {{62, 1, 2, 0}}}, /* U+0030 */ + {1, {{63, 1, 2, 0}}}, /* U+0031 */ + {1, {{64, 1, 2, 0}}}, /* U+0032 */ + {1, {{65, 1, 2, 0}}}, /* U+0033 */ + {1, {{66, 1, 2, 0}}}, /* U+0034 */ + {1, {{67, 1, 2, 0}}}, /* U+0035 */ + {1, {{68, 1, 2, 0}}}, /* U+0036 */ + {1, {{69, 1, 2, 0}}}, /* U+0037 */ + {1, {{70, 1, 2, 0}}}, /* U+0038 */ + {1, {{71, 1, 2, 0}}}, /* U+0039 */ + {1, {{11, 1, 2, 0}}}, /* U+003A */ + {1, {{10, 1, 2, 0}}}, /* U+003B */ + {1, {{50, 1, 2, 0}}}, /* U+003C */ + {1, {{51, 1, 2, 0}}}, /* U+003D */ + {1, {{52, 1, 2, 0}}}, /* U+003E */ + {1, {{14, 1, 2, 0}}}, /* U+003F */ + {1, {{30, 1, 2, 0}}}, /* U+0040 */ + {1, {{72, 1, 4, 0}}}, /* U+0041 */ + {1, {{73, 1, 4, 0}}}, /* U+0042 */ + {1, {{74, 1, 4, 0}}}, /* U+0043 */ + {1, {{75, 1, 4, 0}}}, /* U+0044 */ + {1, {{76, 1, 4, 0}}}, /* U+0045 */ + {1, {{77, 1, 4, 0}}}, /* U+0046 */ + {1, {{78, 1, 4, 0}}}, /* U+0047 */ + {1, {{79, 1, 4, 0}}}, /* U+0048 */ + {1, {{80, 1, 4, 0}}}, /* U+0049 */ + {1, {{82, 1, 4, 0}}}, /* U+004A */ + {1, {{83, 1, 4, 0}}}, /* U+004B */ + {1, {{84, 1, 4, 0}}}, /* U+004C */ + {1, {{85, 1, 4, 0}}}, /* U+004D */ + {1, {{86, 1, 4, 0}}}, /* U+004E */ + {1, {{88, 1, 4, 0}}}, /* U+004F */ + {1, {{89, 1, 4, 0}}}, /* U+0050 */ + {1, {{90, 1, 4, 0}}}, /* U+0051 */ + {1, {{92, 1, 4, 0}}}, /* U+0052 */ + {1, {{93, 1, 4, 0}}}, /* U+0053 */ + {1, {{94, 1, 4, 0}}}, /* U+0054 */ + {1, {{96, 1, 4, 0}}}, /* U+0055 */ + {1, {{97, 1, 4, 0}}}, /* U+0056 */ + {1, {{98, 1, 4, 0}}}, /* U+0057 */ + {1, {{99, 1, 4, 0}}}, /* U+0058 */ + {1, {{100, 1, 4, 0}}}, /* U+0059 */ + {1, {{101, 1, 4, 0}}}, /* U+005A */ + {1, {{24, 1, 2, 0}}}, /* U+005B */ + {1, {{33, 1, 2, 0}}}, /* U+005C */ + {1, {{25, 1, 2, 0}}}, /* U+005D */ + {1, {{39, 1, 2, 0}}}, /* U+005E */ + {1, {{7, 1, 2, 0}}}, /* U+005F */ + {1, {{37, 1, 2, 0}}}, /* U+0060 */ + {1, {{72, 1, 2, 0}}}, /* U+0061 */ + {1, {{73, 1, 2, 0}}}, /* U+0062 */ + {1, {{74, 1, 2, 0}}}, /* U+0063 */ + {1, {{75, 1, 2, 0}}}, /* U+0064 */ + {1, {{76, 1, 2, 0}}}, /* U+0065 */ + {1, {{77, 1, 2, 0}}}, /* U+0066 */ + {1, {{78, 1, 2, 0}}}, /* U+0067 */ + {1, {{79, 1, 2, 0}}}, /* U+0068 */ + {1, {{80, 1, 2, 0}}}, /* U+0069 */ + {1, {{82, 1, 2, 0}}}, /* U+006A */ + {1, {{83, 1, 2, 0}}}, /* U+006B */ + {1, {{84, 1, 2, 0}}}, /* U+006C */ + {1, {{85, 1, 2, 0}}}, /* U+006D */ + {1, {{86, 1, 2, 0}}}, /* U+006E */ + {1, {{88, 1, 2, 0}}}, /* U+006F */ + {1, {{89, 1, 2, 0}}}, /* U+0070 */ + {1, {{90, 1, 2, 0}}}, /* U+0071 */ + {1, {{92, 1, 2, 0}}}, /* U+0072 */ + {1, {{93, 1, 2, 0}}}, /* U+0073 */ + {1, {{94, 1, 2, 0}}}, /* U+0074 */ + {1, {{96, 1, 2, 0}}}, /* U+0075 */ + {1, {{97, 1, 2, 0}}}, /* U+0076 */ + {1, {{98, 1, 2, 0}}}, /* U+0077 */ + {1, {{99, 1, 2, 0}}}, /* U+0078 */ + {1, {{100, 1, 2, 0}}}, /* U+0079 */ + {1, {{101, 1, 2, 0}}}, /* U+007A */ + {1, {{26, 1, 2, 0}}}, /* U+007B */ + {1, {{54, 1, 2, 0}}}, /* U+007C */ + {1, {{27, 1, 2, 0}}}, /* U+007D */ + {1, {{56, 1, 2, 0}}}, /* U+007E */ }; static const ScrCollEntry scr_coll_00a0[224] = { - {1, {{6, 1, 2, 0}}}, /* U+00A0 */ - {1, {{13, 1, 1, 0}}}, /* U+00A1 */ - {1, {{58, 1, 1, 0}}}, /* U+00A2 */ - {1, {{60, 1, 1, 0}}}, /* U+00A3 */ - {1, {{57, 1, 1, 0}}}, /* U+00A4 */ - {1, {{61, 1, 1, 0}}}, /* U+00A5 */ - {1, {{55, 1, 1, 0}}}, /* U+00A6 */ - {1, {{28, 1, 1, 0}}}, /* U+00A7 */ - {1, {{41, 1, 1, 0}}}, /* U+00A8 */ - {1, {{44, 1, 1, 0}}}, /* U+00A9 */ - {1, {{72, 1, 3, 0}}}, /* U+00AA */ - {1, {{20, 1, 1, 0}}}, /* U+00AB */ - {1, {{53, 1, 1, 0}}}, /* U+00AC */ + {1, {{6, 1, 4, 0}}}, /* U+00A0 */ + {1, {{13, 1, 2, 0}}}, /* U+00A1 */ + {1, {{58, 1, 2, 0}}}, /* U+00A2 */ + {1, {{60, 1, 2, 0}}}, /* U+00A3 */ + {1, {{57, 1, 2, 0}}}, /* U+00A4 */ + {1, {{61, 1, 2, 0}}}, /* U+00A5 */ + {1, {{55, 1, 2, 0}}}, /* U+00A6 */ + {1, {{28, 1, 2, 0}}}, /* U+00A7 */ + {1, {{41, 1, 2, 0}}}, /* U+00A8 */ + {1, {{44, 1, 2, 0}}}, /* U+00A9 */ + {1, {{72, 1, 6, 0}}}, /* U+00AA */ + {1, {{20, 1, 2, 0}}}, /* U+00AB */ + {1, {{53, 1, 2, 0}}}, /* U+00AC */ {0, {}}, /* U+00AD */ - {1, {{45, 1, 1, 0}}}, /* U+00AE */ - {1, {{40, 1, 1, 0}}}, /* U+00AF */ - {1, {{43, 1, 1, 0}}}, /* U+00B0 */ - {1, {{47, 1, 1, 0}}}, /* U+00B1 */ - {1, {{64, 1, 2, 0}}}, /* U+00B2 */ - {1, {{65, 1, 2, 0}}}, /* U+00B3 */ - {1, {{38, 1, 1, 0}}}, /* U+00B4 */ - {1, {{104, 1, 1, 0}}}, /* U+00B5 */ - {1, {{29, 1, 1, 0}}}, /* U+00B6 */ - {1, {{17, 1, 1, 0}}}, /* U+00B7 */ - {1, {{42, 1, 1, 0}}}, /* U+00B8 */ - {1, {{63, 1, 2, 0}}}, /* U+00B9 */ - {1, {{88, 1, 3, 0}}}, /* U+00BA */ - {1, {{21, 1, 1, 0}}}, /* U+00BB */ + {1, {{45, 1, 2, 0}}}, /* U+00AE */ + {1, {{40, 1, 2, 0}}}, /* U+00AF */ + {1, {{43, 1, 2, 0}}}, /* U+00B0 */ + {1, {{47, 1, 2, 0}}}, /* U+00B1 */ + {1, {{64, 1, 4, 0}}}, /* U+00B2 */ + {1, {{65, 1, 4, 0}}}, /* U+00B3 */ + {1, {{38, 1, 2, 0}}}, /* U+00B4 */ + {1, {{104, 1, 2, 0}}}, /* U+00B5 */ + {1, {{29, 1, 2, 0}}}, /* U+00B6 */ + {1, {{17, 1, 2, 0}}}, /* U+00B7 */ + {1, {{42, 1, 2, 0}}}, /* U+00B8 */ + {1, {{63, 1, 4, 0}}}, /* U+00B9 */ + {1, {{88, 1, 6, 0}}}, /* U+00BA */ + {1, {{21, 1, 2, 0}}}, /* U+00BB */ {255, {{0, 0, 0, 0}}}, /* U+00BC not covered */ {255, {{0, 0, 0, 0}}}, /* U+00BD not covered */ {255, {{0, 0, 0, 0}}}, /* U+00BE not covered */ - {1, {{15, 1, 1, 0}}}, /* U+00BF */ - {2, {{72, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00C0 */ - {2, {{72, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00C1 */ - {2, {{72, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00C2 */ - {2, {{72, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00C3 */ - {2, {{72, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00C4 */ - {2, {{72, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+00C5 */ - {3, {{72, 1, 2, 0}, {0, 51, 1, 0}, {76, 1, 2, 0}}}, /* U+00C6 */ - {2, {{74, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+00C7 */ - {2, {{76, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00C8 */ - {2, {{76, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00C9 */ - {2, {{76, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00CA */ - {2, {{76, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00CB */ - {2, {{80, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00CC */ - {2, {{80, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00CD */ - {2, {{80, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00CE */ - {2, {{80, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00CF */ - {2, {{75, 1, 2, 0}, {0, 48, 1, 0}}}, /* U+00D0 */ - {2, {{86, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00D1 */ - {2, {{88, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00D2 */ - {2, {{88, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00D3 */ - {2, {{88, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00D4 */ - {2, {{88, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00D5 */ - {2, {{88, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00D6 */ - {1, {{49, 1, 1, 0}}}, /* U+00D7 */ - {2, {{88, 1, 2, 0}, {0, 18, 1, 0}}}, /* U+00D8 */ - {2, {{96, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00D9 */ - {2, {{96, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00DA */ - {2, {{96, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00DB */ - {2, {{96, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00DC */ - {2, {{100, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00DD */ - {1, {{102, 1, 2, 0}}}, /* U+00DE */ - {3, {{93, 1, 1, 0}, {0, 52, 1, 0}, {93, 1, 1, 0}}}, /* U+00DF */ - {2, {{72, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00E0 */ - {2, {{72, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00E1 */ - {2, {{72, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00E2 */ - {2, {{72, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00E3 */ - {2, {{72, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00E4 */ - {2, {{72, 1, 1, 0}, {0, 11, 1, 5}}}, /* U+00E5 */ - {3, {{72, 1, 1, 0}, {0, 51, 1, 0}, {76, 1, 1, 0}}}, /* U+00E6 */ - {2, {{74, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+00E7 */ - {2, {{76, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00E8 */ - {2, {{76, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00E9 */ - {2, {{76, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00EA */ - {2, {{76, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00EB */ - {2, {{80, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00EC */ - {2, {{80, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00ED */ - {2, {{80, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00EE */ - {2, {{80, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00EF */ - {2, {{75, 1, 1, 0}, {0, 48, 1, 0}}}, /* U+00F0 */ - {2, {{86, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00F1 */ - {2, {{88, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00F2 */ - {2, {{88, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00F3 */ - {2, {{88, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00F4 */ - {2, {{88, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00F5 */ - {2, {{88, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00F6 */ - {1, {{48, 1, 1, 0}}}, /* U+00F7 */ - {2, {{88, 1, 1, 0}, {0, 18, 1, 0}}}, /* U+00F8 */ - {2, {{96, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00F9 */ - {2, {{96, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00FA */ - {2, {{96, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00FB */ - {2, {{96, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00FC */ - {2, {{100, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00FD */ - {1, {{102, 1, 1, 0}}}, /* U+00FE */ - {2, {{100, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00FF */ - {2, {{72, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0100 */ - {2, {{72, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+0101 */ - {2, {{72, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0102 */ - {2, {{72, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+0103 */ - {2, {{72, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0104 */ - {2, {{72, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0105 */ - {2, {{74, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0106 */ - {2, {{74, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0107 */ - {2, {{74, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0108 */ - {2, {{74, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0109 */ - {2, {{74, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+010A */ - {2, {{74, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+010B */ - {2, {{74, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010C */ - {2, {{74, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+010D */ - {2, {{75, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010E */ - {2, {{75, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+010F */ - {2, {{75, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0110 */ - {2, {{75, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0111 */ - {2, {{76, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0112 */ - {2, {{76, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+0113 */ - {2, {{76, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0114 */ - {2, {{76, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+0115 */ - {2, {{76, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0116 */ - {2, {{76, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0117 */ - {2, {{76, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0118 */ - {2, {{76, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0119 */ - {2, {{76, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+011A */ - {2, {{76, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+011B */ - {2, {{78, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+011C */ - {2, {{78, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+011D */ - {2, {{78, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+011E */ - {2, {{78, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+011F */ - {2, {{78, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0120 */ - {2, {{78, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0121 */ - {2, {{78, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0122 */ - {2, {{78, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0123 */ - {2, {{79, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0124 */ - {2, {{79, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0125 */ - {2, {{79, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0126 */ - {2, {{79, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0127 */ - {2, {{80, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0128 */ - {2, {{80, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+0129 */ - {2, {{80, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+012A */ - {2, {{80, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+012B */ - {2, {{80, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+012C */ - {2, {{80, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+012D */ - {2, {{80, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+012E */ - {2, {{80, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+012F */ - {2, {{80, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0130 */ - {1, {{81, 1, 1, 0}}}, /* U+0131 */ - {2, {{80, 1, 2, 0}, {82, 1, 2, 0}}}, /* U+0132 */ - {2, {{80, 1, 1, 0}, {82, 1, 1, 0}}}, /* U+0133 */ - {2, {{82, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0134 */ - {2, {{82, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0135 */ - {2, {{83, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0136 */ - {2, {{83, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0137 */ - {1, {{91, 1, 1, 0}}}, /* U+0138 */ - {2, {{84, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0139 */ - {2, {{84, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+013A */ - {2, {{84, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+013B */ - {2, {{84, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+013C */ - {2, {{84, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+013D */ - {2, {{84, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+013E */ - {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}, /* U+013F */ - {2, {{84, 1, 1, 0}, {0, 49, 1, 0}}}, /* U+0140 */ - {2, {{84, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0141 */ - {2, {{84, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0142 */ - {2, {{86, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0143 */ - {2, {{86, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0144 */ - {2, {{86, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0145 */ - {2, {{86, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0146 */ - {2, {{86, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0147 */ - {2, {{86, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0148 */ - {1, {{103, 1, 1, 0}}}, /* U+0149 */ - {1, {{87, 1, 2, 0}}}, /* U+014A */ - {1, {{87, 1, 1, 0}}}, /* U+014B */ - {2, {{88, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+014C */ - {2, {{88, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+014D */ - {2, {{88, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+014E */ - {2, {{88, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+014F */ - {2, {{88, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0150 */ - {2, {{88, 1, 1, 0}, {0, 15, 1, 5}}}, /* U+0151 */ - {3, {{88, 1, 2, 0}, {0, 53, 1, 0}, {76, 1, 2, 0}}}, /* U+0152 */ - {3, {{88, 1, 1, 0}, {0, 53, 1, 0}, {76, 1, 1, 0}}}, /* U+0153 */ - {2, {{92, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0154 */ - {2, {{92, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0155 */ - {2, {{92, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0156 */ - {2, {{92, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0157 */ - {2, {{92, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0158 */ - {2, {{92, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0159 */ - {2, {{93, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+015A */ - {2, {{93, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+015B */ - {2, {{93, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+015C */ - {2, {{93, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+015D */ - {2, {{93, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+015E */ - {2, {{93, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+015F */ - {2, {{93, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0160 */ - {2, {{93, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0161 */ - {2, {{94, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0162 */ - {2, {{94, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0163 */ - {2, {{94, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0164 */ - {2, {{94, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0165 */ - {1, {{95, 1, 2, 0}}}, /* U+0166 */ - {1, {{95, 1, 1, 0}}}, /* U+0167 */ - {2, {{96, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0168 */ - {2, {{96, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+0169 */ - {2, {{96, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+016A */ - {2, {{96, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+016B */ - {2, {{96, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+016C */ - {2, {{96, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+016D */ - {2, {{96, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+016E */ - {2, {{96, 1, 1, 0}, {0, 11, 1, 5}}}, /* U+016F */ - {2, {{96, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0170 */ - {2, {{96, 1, 1, 0}, {0, 15, 1, 5}}}, /* U+0171 */ - {2, {{96, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0172 */ - {2, {{96, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0173 */ - {2, {{98, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0174 */ - {2, {{98, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0175 */ - {2, {{100, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0176 */ - {2, {{100, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0177 */ - {2, {{100, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+0178 */ - {2, {{101, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0179 */ - {2, {{101, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+017A */ - {2, {{101, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+017B */ - {2, {{101, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+017C */ - {2, {{101, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+017D */ - {2, {{101, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+017E */ - {2, {{93, 1, 1, 0}, {0, 50, 1, 0}}}, /* U+017F */ + {1, {{15, 1, 2, 0}}}, /* U+00BF */ + {2, {{72, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00C0 */ + {2, {{72, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00C1 */ + {2, {{72, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00C2 */ + {2, {{72, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00C3 */ + {2, {{72, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00C4 */ + {2, {{72, 1, 4, 0}, {0, 11, 1, 5}}}, /* U+00C5 */ + {3, {{72, 1, 4, 0}, {0, 51, 1, 0}, {76, 1, 4, 0}}}, /* U+00C6 */ + {2, {{74, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+00C7 */ + {2, {{76, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00C8 */ + {2, {{76, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00C9 */ + {2, {{76, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00CA */ + {2, {{76, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00CB */ + {2, {{80, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00CC */ + {2, {{80, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00CD */ + {2, {{80, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00CE */ + {2, {{80, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00CF */ + {2, {{75, 1, 4, 0}, {0, 48, 1, 0}}}, /* U+00D0 */ + {2, {{86, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00D1 */ + {2, {{88, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00D2 */ + {2, {{88, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00D3 */ + {2, {{88, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00D4 */ + {2, {{88, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00D5 */ + {2, {{88, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00D6 */ + {1, {{49, 1, 2, 0}}}, /* U+00D7 */ + {2, {{88, 1, 4, 0}, {0, 18, 1, 0}}}, /* U+00D8 */ + {2, {{96, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00D9 */ + {2, {{96, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00DA */ + {2, {{96, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00DB */ + {2, {{96, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00DC */ + {2, {{100, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00DD */ + {1, {{102, 1, 4, 0}}}, /* U+00DE */ + {3, {{93, 1, 2, 0}, {0, 52, 1, 0}, {93, 1, 2, 0}}}, /* U+00DF */ + {2, {{72, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00E0 */ + {2, {{72, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00E1 */ + {2, {{72, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00E2 */ + {2, {{72, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00E3 */ + {2, {{72, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00E4 */ + {2, {{72, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+00E5 */ + {3, {{72, 1, 2, 0}, {0, 51, 1, 0}, {76, 1, 2, 0}}}, /* U+00E6 */ + {2, {{74, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+00E7 */ + {2, {{76, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00E8 */ + {2, {{76, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00E9 */ + {2, {{76, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00EA */ + {2, {{76, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00EB */ + {2, {{80, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00EC */ + {2, {{80, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00ED */ + {2, {{80, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00EE */ + {2, {{80, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00EF */ + {2, {{75, 1, 2, 0}, {0, 48, 1, 0}}}, /* U+00F0 */ + {2, {{86, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00F1 */ + {2, {{88, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00F2 */ + {2, {{88, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00F3 */ + {2, {{88, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00F4 */ + {2, {{88, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00F5 */ + {2, {{88, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00F6 */ + {1, {{48, 1, 2, 0}}}, /* U+00F7 */ + {2, {{88, 1, 2, 0}, {0, 18, 1, 0}}}, /* U+00F8 */ + {2, {{96, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00F9 */ + {2, {{96, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00FA */ + {2, {{96, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00FB */ + {2, {{96, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00FC */ + {2, {{100, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00FD */ + {1, {{102, 1, 2, 0}}}, /* U+00FE */ + {2, {{100, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00FF */ + {2, {{72, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+0100 */ + {2, {{72, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0101 */ + {2, {{72, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+0102 */ + {2, {{72, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0103 */ + {2, {{72, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0104 */ + {2, {{72, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0105 */ + {2, {{74, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0106 */ + {2, {{74, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0107 */ + {2, {{74, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0108 */ + {2, {{74, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0109 */ + {2, {{74, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+010A */ + {2, {{74, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+010B */ + {2, {{74, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+010C */ + {2, {{74, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010D */ + {2, {{75, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+010E */ + {2, {{75, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010F */ + {2, {{75, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0110 */ + {2, {{75, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0111 */ + {2, {{76, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+0112 */ + {2, {{76, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0113 */ + {2, {{76, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+0114 */ + {2, {{76, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0115 */ + {2, {{76, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0116 */ + {2, {{76, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0117 */ + {2, {{76, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0118 */ + {2, {{76, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0119 */ + {2, {{76, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+011A */ + {2, {{76, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+011B */ + {2, {{78, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+011C */ + {2, {{78, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+011D */ + {2, {{78, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+011E */ + {2, {{78, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+011F */ + {2, {{78, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0120 */ + {2, {{78, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0121 */ + {2, {{78, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0122 */ + {2, {{78, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0123 */ + {2, {{79, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0124 */ + {2, {{79, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0125 */ + {2, {{79, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0126 */ + {2, {{79, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0127 */ + {2, {{80, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+0128 */ + {2, {{80, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0129 */ + {2, {{80, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+012A */ + {2, {{80, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+012B */ + {2, {{80, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+012C */ + {2, {{80, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+012D */ + {2, {{80, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+012E */ + {2, {{80, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+012F */ + {2, {{80, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0130 */ + {1, {{81, 1, 2, 0}}}, /* U+0131 */ + {2, {{80, 1, 5, 0}, {82, 1, 5, 0}}}, /* U+0132 */ + {2, {{80, 1, 3, 0}, {82, 1, 3, 0}}}, /* U+0133 */ + {2, {{82, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0134 */ + {2, {{82, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0135 */ + {2, {{83, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0136 */ + {2, {{83, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0137 */ + {1, {{91, 1, 2, 0}}}, /* U+0138 */ + {2, {{84, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0139 */ + {2, {{84, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+013A */ + {2, {{84, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+013B */ + {2, {{84, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+013C */ + {2, {{84, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+013D */ + {2, {{84, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+013E */ + {2, {{84, 1, 4, 0}, {0, 49, 1, 0}}}, /* U+013F */ + {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}, /* U+0140 */ + {2, {{84, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0141 */ + {2, {{84, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0142 */ + {2, {{86, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0143 */ + {2, {{86, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0144 */ + {2, {{86, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0145 */ + {2, {{86, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0146 */ + {2, {{86, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0147 */ + {2, {{86, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0148 */ + {1, {{103, 1, 2, 0}}}, /* U+0149 */ + {1, {{87, 1, 4, 0}}}, /* U+014A */ + {1, {{87, 1, 2, 0}}}, /* U+014B */ + {2, {{88, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+014C */ + {2, {{88, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+014D */ + {2, {{88, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+014E */ + {2, {{88, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+014F */ + {2, {{88, 1, 4, 0}, {0, 15, 1, 5}}}, /* U+0150 */ + {2, {{88, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0151 */ + {3, {{88, 1, 4, 0}, {0, 53, 1, 0}, {76, 1, 4, 0}}}, /* U+0152 */ + {3, {{88, 1, 2, 0}, {0, 53, 1, 0}, {76, 1, 2, 0}}}, /* U+0153 */ + {2, {{92, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0154 */ + {2, {{92, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0155 */ + {2, {{92, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0156 */ + {2, {{92, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0157 */ + {2, {{92, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0158 */ + {2, {{92, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0159 */ + {2, {{93, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+015A */ + {2, {{93, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+015B */ + {2, {{93, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+015C */ + {2, {{93, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+015D */ + {2, {{93, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+015E */ + {2, {{93, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+015F */ + {2, {{93, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0160 */ + {2, {{93, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0161 */ + {2, {{94, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0162 */ + {2, {{94, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0163 */ + {2, {{94, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0164 */ + {2, {{94, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0165 */ + {1, {{95, 1, 4, 0}}}, /* U+0166 */ + {1, {{95, 1, 2, 0}}}, /* U+0167 */ + {2, {{96, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+0168 */ + {2, {{96, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0169 */ + {2, {{96, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+016A */ + {2, {{96, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+016B */ + {2, {{96, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+016C */ + {2, {{96, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+016D */ + {2, {{96, 1, 4, 0}, {0, 11, 1, 5}}}, /* U+016E */ + {2, {{96, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+016F */ + {2, {{96, 1, 4, 0}, {0, 15, 1, 5}}}, /* U+0170 */ + {2, {{96, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0171 */ + {2, {{96, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0172 */ + {2, {{96, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0173 */ + {2, {{98, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0174 */ + {2, {{98, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0175 */ + {2, {{100, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0176 */ + {2, {{100, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0177 */ + {2, {{100, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+0178 */ + {2, {{101, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0179 */ + {2, {{101, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+017A */ + {2, {{101, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+017B */ + {2, {{101, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+017C */ + {2, {{101, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+017D */ + {2, {{101, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+017E */ + {2, {{93, 1, 2, 0}, {0, 50, 1, 0}}}, /* U+017F */ }; static const ScrCollEntry scr_coll_0300[112] = { {1, {{0, 7, 1, 5}}}, /* U+0300 */ @@ -461,4 +462,11 @@ static const ScrCollEntry *scr_coll_lookup(uint32_t c) { if (c >= 0x300 && c <= 0x36f) return &scr_coll_0300[c - 0x300]; return 0; } +/* Two-code-point contractions: the pair collates as one entry (l· as ŀ). */ +typedef struct { uint32_t first, second; ScrCollEntry entry; } ScrCollContraction; +#define SCR_COLL_CONTRACTIONS 2 +static const ScrCollContraction scr_coll_contractions[2] = { + {0x4c, 0xb7, {2, {{84, 1, 4, 0}, {0, 49, 1, 0}}}}, /* U+004C U+00B7 */ + {0x6c, 0xb7, {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}}, /* U+006C U+00B7 */ +}; #endif diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 3a4ca9e7e..61c0481ed 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -3289,30 +3289,106 @@ typedef struct { bool ok; } ScrCollAscii; static SCR_TL ScrCollAscii scr_coll_ascii_table[128]; +/* Covered bytes that are part of no contraction: one element each. */ +static SCR_TL bool scr_coll_ascii_plain[128]; static SCR_TL bool scr_coll_ascii_ready; /* The same table for the single-pass primary comparison: covered bytes as * a bit set, their primaries, and whether any covered byte has a zero * primary (then the level-0 skip makes the byte-by-byte pass inexact). */ static SCR_TL uint64_t scr_coll_ascii_mask[2]; static SCR_TL uint16_t scr_coll_ascii_primaries[128]; -static SCR_TL bool scr_coll_ascii_zero_primary; +/* Which byte-by-byte ASCII paths are exact: a contraction of two ASCII code + * points turns both off, and a covered byte with a zero primary turns off + * the single-pass primary comparison. */ +static SCR_TL bool scr_coll_ascii_one_pass; +static SCR_TL bool scr_coll_ascii_levels; +/* Code points below 0x180 that start a contraction, as a bit set; higher + * ones are looked up in scr_coll_contractions. */ +#define SCR_COLL_BITS_LIMIT 0x180 +static SCR_TL uint64_t scr_coll_starts_bits[SCR_COLL_BITS_LIMIT / 64]; +/* scr_coll_scan's answer for one code point below SCR_COLL_SCAN_LIMIT + * (every code point the table covers): 0 for plain covered, 1 for covered + * but needing the element lists (a mark with a primary weight, or the end + * of a contraction), 2 for uncovered. */ +#define SCR_COLL_SCAN_LIMIT 0x370 +static SCR_TL uint8_t scr_coll_scan_class[SCR_COLL_SCAN_LIMIT]; -static const ScrCollAscii *scr_coll_ascii(void) { - if (__builtin_expect(!scr_coll_ascii_ready, 0)) { - for (uint32_t c = 0; c < 128; c++) { - const ScrCollEntry *entry = scr_coll_lookup(c); - bool ok = entry != NULL && entry->n == 1 && entry->e[0].ccc == 0; - scr_coll_ascii_table[c] = (ScrCollAscii){ok ? entry->e[0].p : 0, ok ? entry->e[0].s : 0, - ok ? entry->e[0].t : 0, ok}; - if (ok) scr_coll_ascii_mask[c >> 6] |= UINT64_C(1) << (c & 63); - scr_coll_ascii_primaries[c] = ok ? entry->e[0].p : 0; - if (ok && entry->e[0].p == 0) scr_coll_ascii_zero_primary = true; - } - scr_coll_ascii_ready = true; +static __attribute__((noinline)) void scr_coll_ascii_init(void) { + bool zero_primary = false, contraction = false, in_contraction[128] = {false}; + for (uint32_t c = 0; c < 128; c++) { + const ScrCollEntry *entry = scr_coll_lookup(c); + bool ok = entry != NULL && entry->n == 1 && entry->e[0].ccc == 0; + scr_coll_ascii_table[c] = (ScrCollAscii){ok ? entry->e[0].p : 0, ok ? entry->e[0].s : 0, + ok ? entry->e[0].t : 0, ok}; + if (ok) scr_coll_ascii_mask[c >> 6] |= UINT64_C(1) << (c & 63); + scr_coll_ascii_primaries[c] = ok ? entry->e[0].p : 0; + if (ok && entry->e[0].p == 0) zero_primary = true; } + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) { + uint32_t first = scr_coll_contractions[k].first, second = scr_coll_contractions[k].second; + if (first < 128) in_contraction[first] = true; + if (second < 128) in_contraction[second] = true; + if (first < 128 && second < 128) contraction = true; + if (first < SCR_COLL_BITS_LIMIT) scr_coll_starts_bits[first >> 6] |= UINT64_C(1) << (first & 63); + } + for (uint32_t c = 0; c < 128; c++) scr_coll_ascii_plain[c] = scr_coll_ascii_table[c].ok && !in_contraction[c]; + scr_coll_ascii_one_pass = !zero_primary && !contraction; + scr_coll_ascii_levels = !contraction; + for (uint32_t c = 0; c < SCR_COLL_SCAN_LIMIT; c++) { + const ScrCollEntry *entry = scr_coll_lookup(c); + uint8_t klass = entry == NULL || entry->n == 255 ? 2 : 0; + for (uint8_t i = 0; klass == 0 && i < entry->n; i++) + if (entry->e[i].ccc != 0 && entry->e[i].p != 0) klass = 1; + for (size_t k = 0; klass == 0 && k < SCR_COLL_CONTRACTIONS; k++) + if (scr_coll_contractions[k].second == c) klass = 1; + scr_coll_scan_class[c] = klass; + } + scr_coll_ascii_ready = true; +} + +/* The per-thread tables, built on first use. */ +static inline __attribute__((always_inline)) const ScrCollAscii *scr_coll_ascii(void) { + if (__builtin_expect(!scr_coll_ascii_ready, 0)) scr_coll_ascii_init(); return scr_coll_ascii_table; } +static inline __attribute__((always_inline)) uint32_t scr_coll_decode(const unsigned char **pp, + const unsigned char *end) { + const unsigned char *p = *pp; + uint32_t c = *p++; + if (c >= 0x80) { + int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; + c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; + for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); + } + *pp = p; + return c; +} + +/* The contraction of code point c with the code point at p, setting *after + * past that second code point; NULL when the pair is not a contraction. */ +static const ScrCollEntry *scr_coll_contraction(uint32_t c, const unsigned char *p, + const unsigned char *end, + const unsigned char **after) { + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) { + if (scr_coll_contractions[k].first != c || p == end) continue; + const unsigned char *q = p; + if (scr_coll_decode(&q, end) == scr_coll_contractions[k].second) { + *after = q; + return &scr_coll_contractions[k].entry; + } + } + return NULL; +} + +/* Whether code point c starts a contraction. */ +static inline bool scr_coll_starts_contraction(uint32_t c) { + if (c < SCR_COLL_BITS_LIMIT) return (scr_coll_starts_bits[c >> 6] >> (c & 63)) & 1; + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) + if (scr_coll_contractions[k].first == c) return true; + return false; +} + /* Elements of a UTF-8 string, marks in canonical order; false when a code * point is outside the table. */ static bool scr_coll_elements(const ScrStr *s, ScrCollBuf *b) { @@ -3331,17 +3407,16 @@ static bool scr_coll_elements(const ScrStr *s, ScrCollBuf *b) { const unsigned char *p = (const unsigned char *)s->data; const unsigned char *end = p + s->len; while (p < end) { - uint32_t c = *p++; - if (c < 0x80 && ascii[c].ok) { + uint32_t c = *p; + if (c < 0x80 && scr_coll_ascii_plain[c]) { + p++; e[n++] = (ScrCollElem){ascii[c].p, ascii[c].s, ascii[c].t, 0}; continue; } - if (c >= 0x80) { - int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; - c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; - for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); - } - const ScrCollEntry *entry = scr_coll_lookup(c); + c = scr_coll_decode(&p, end); + const ScrCollEntry *entry = NULL; + if (scr_coll_starts_contraction(c)) entry = scr_coll_contraction(c, p, end, &p); + if (entry == NULL) entry = scr_coll_lookup(c); if (entry == NULL || entry->n == 255) { b->n = n; return false; @@ -3374,32 +3449,27 @@ typedef struct { uint8_t next; /* its next element */ } ScrCollCursor; -static uint32_t scr_coll_decode(const unsigned char **pp, const unsigned char *end) { - const unsigned char *p = *pp; - uint32_t c = *p++; - if (c >= 0x80) { - int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; - c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; - for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); - } - *pp = p; - return c; -} - /* 0: every code point is covered and no mark carries a primary weight; * 1: covered, with such a mark; 2: a code point outside the table. */ -static int scr_coll_scan(const ScrCollAscii *ascii, const ScrStr *s) { +static int scr_coll_scan(const ScrStr *s) { const unsigned char *p = (const unsigned char *)s->data, *end = p + s->len; int result = 0; while (p < end) { - if (*p < 0x80 && ascii[*p].ok) { + if (*p < 0x80 && scr_coll_ascii_plain[*p]) { p++; continue; } - const ScrCollEntry *entry = scr_coll_lookup(scr_coll_decode(&p, end)); - if (entry == NULL || entry->n == 255) return 2; - for (uint8_t i = 0; i < entry->n; i++) - if (entry->e[i].ccc != 0 && entry->e[i].p != 0) result = 1; + uint32_t c = scr_coll_decode(&p, end); + if (c >= SCR_COLL_SCAN_LIMIT) { + /* Above the class table: covered ones take the element lists. */ + const ScrCollEntry *entry = scr_coll_lookup(c); + if (entry == NULL || entry->n == 255) return 2; + result = 1; + continue; + } + uint8_t klass = scr_coll_scan_class[c]; + if (klass == 2) return 2; + result |= klass; } return result; } @@ -3518,20 +3588,21 @@ double scr_str_locale_compare(ScrStr *a, ScrStr *b) { /* Equal bytes have equal elements at every level. */ if (a == b || (a->len == b->len && memcmp(a->data, b->data, a->len) == 0)) return 0; const ScrCollAscii *table = scr_coll_ascii(); - if (!scr_coll_ascii_zero_primary) { + if (scr_coll_ascii_one_pass) { int r = scr_coll_ascii_primary(a, b); if (r != 2) { for (int level = 1; level < 3 && r == 0; level++) r = scr_coll_ascii_level(table, a, b, level); return (double)r; } - } else if (scr_coll_ascii_covered(table, a) && scr_coll_ascii_covered(table, b)) { + } else if (scr_coll_ascii_levels && scr_coll_ascii_covered(table, a) && + scr_coll_ascii_covered(table, b)) { int r = 0; for (int level = 0; level < 3 && r == 0; level++) r = scr_coll_ascii_level(table, a, b, level); return (double)r; } - int scan_a = scr_coll_scan(table, a); + int scan_a = scr_coll_scan(a); if (scan_a == 2) return scr_str_cmp(a, b) < 0 ? -1 : 1; /* bytes differ (checked above) */ - int scan_b = scr_coll_scan(table, b); + int scan_b = scr_coll_scan(b); if (scan_b == 2) return scr_str_cmp(a, b) < 0 ? -1 : 1; if (scan_a == 0 && scan_b == 0) { int r = scr_coll_primary(table, a, b); diff --git a/scripts/gen-collation-table.mjs b/scripts/gen-collation-table.mjs index 9afb25438..a79bd0c9a 100644 --- a/scripts/gen-collation-table.mjs +++ b/scripts/gen-collation-table.mjs @@ -186,7 +186,8 @@ for (const [c, pair] of expansion) { for (const [c, m] of variantMark) extraSecondary.set(c, markWeight.get(m)); for (const [c, m] of expansionMark) expansionSecondary.set(c, markWeight.get(m)); -// 6. Elements per code point (tertiary assigned below). +// 6. Elements of marks, ignorables and base characters; decomposable +// characters and ligatures are built from them in step 7. const COMMON = 1; const elements = new Map(); for (const c of covered) { @@ -199,38 +200,20 @@ for (const c of primaryReps) { if (extraSecondary.has(c)) ces.push([0, extraSecondary.get(c), COMMON, 0]); elements.set(c, ces); } -for (const c of covered) { - if (elements.has(c) || expansion.has(c)) continue; - // Canonically decomposable: the elements of its NFD atoms. - const nfd = [...ch(c).normalize("NFD")].map((x) => x.codePointAt(0)); - if (nfd.length === 1 && nfd[0] === c) throw new Error(`no elements for U+${c.toString(16)}`); - elements.set( - c, - nfd.flatMap((x) => { - const es = elements.get(x); - if (es === undefined) throw new Error(`U+${c.toString(16)} decomposes outside the table`); - return es.map((e) => [...e]); - }), - ); -} -for (const [c, pair] of expansion) { - const first = elements.get(pair.codePointAt(0)).map((e) => [...e]); - const second = elements.get(pair.codePointAt(1)).map((e) => [...e]); - // A ligature differs from its spelled-out pair at the secondary level: - // find the mark-equivalent secondary between the two halves, if any. - const middle = expansionSecondary.has(c) ? [[0, expansionSecondary.get(c), COMMON, 0]] : []; - elements.set(c, [...first, ...middle, ...second]); -} - -// 7. Tertiary: rank code points whose primary+secondary elements agree. +// 7. Tertiary. Base characters (primaryReps) are ranked within each group +// whose primary+secondary elements agree, at even weights. Canonically +// decomposable characters then take their atoms' elements, so they equal +// their decompositions at every level, as canonical equivalence requires. +// A ligature whose spelled-out pair ties with it at the secondary level +// (ij against ij) gets the weight Node places it at among the case forms +// of its first letter; odd weights fall between two of them. const psKey = (c) => elements .get(c) .map((e) => `${e[0]}.${e[1]}`) .join(" "); const groups = new Map(); -for (const c of covered) { - if (ignorable.has(c) || isMark(c)) continue; +for (const c of primaryReps) { const k = psKey(c); if (!groups.has(k)) groups.set(k, []); groups.get(k).push(c); @@ -241,18 +224,81 @@ for (const members of groups.values()) { let t = 1; for (let i = 0; i < members.length; i++) { if (i > 0 && variant.compare(ch(members[i - 1]), ch(members[i])) !== 0) t++; - tertiaryOf.set(members[i], t); + tertiaryOf.set(members[i], 2 * t); } } -for (const c of covered) { - if (ignorable.has(c) || isMark(c)) continue; +for (const c of primaryReps) for (const e of elements.get(c)) if (e[0] !== 0) e[2] = tertiaryOf.get(c); +const atomsOf = (c) => + [...ch(c).normalize("NFD")].flatMap((x) => elements.get(x.codePointAt(0)).map((e) => [...e])); +for (const c of covered) { + if (ignorable.has(c) || primaryReps.includes(c) || expansion.has(c)) continue; + if (isMark(c) && ch(c).normalize("NFD") === ch(c)) continue; + elements.set(c, atomsOf(c)); +} +const groupOf = (c) => { + const members = groups.get(psKey(c)) ?? [c]; + return [...members].sort((a, b) => tertiaryOf.get(a) - tertiaryOf.get(b)); +}; +for (const [c, pair] of expansion) { + const [x, y] = [...pair].map((q) => q.codePointAt(0)); + const middle = expansionSecondary.has(c) ? [[0, expansionSecondary.get(c), COMMON, 0]] : []; + const es = [...atomsOf(x), ...middle, ...atomsOf(y)]; + // A ligature with its own secondary (æ) keeps the case of its first + // letter in the ligature's case (Æ: the weight of A). + const upper = ch(c) !== ch(c).toLowerCase(); + let t = tertiaryOf.get(upper ? ch(x).toUpperCase().codePointAt(0) : x); + if (middle.length === 0) { + const gx = groupOf(x); + const gy = groupOf(y); + const low = ch(gy[0]), + high = ch(gy[gy.length - 1]); + const below = gx.filter((m) => variant.compare(ch(c), ch(m) + high) > 0); + const equal = gx.find( + (m) => variant.compare(ch(c), ch(m) + low) >= 0 && variant.compare(ch(c), ch(m) + high) <= 0, + ); + t = + equal !== undefined + ? tertiaryOf.get(equal) + : below.length > 0 + ? tertiaryOf.get(below[below.length - 1]) + 1 + : tertiaryOf.get(gx[0]) - 1; + } + for (const e of es) if (e[0] !== 0) e[2] = t; + elements.set(c, es); +} +// Contractions: a covered character that Node treats as equal to its +// two-character compatibility form (ŀ and l·) gives that pair its elements, +// where the pair's own characters would order differently ("L·a" > "La"). +const contractions = new Map(); +for (const c of covered) { + const k = ch(c).normalize("NFKD"); + const cps = [...k].map((q) => q.codePointAt(0)); + if (cps.length !== 2 || !cps.every((q) => coveredSet.has(q))) continue; + if (ch(c).normalize("NFD") === k || variant.compare(ch(c), k) !== 0) continue; + const own = JSON.stringify(elements.get(c)); + const spelled = JSON.stringify(cps.flatMap((q) => elements.get(q))); + if (own !== spelled) + contractions.set( + k, + elements.get(c).map((e) => [...e]), + ); } // The model's comparison (mirrored by scr_str_locale_compare in C). export function modelCompare(a, b) { const ces = (s) => { - const es = [...s].flatMap((x) => elements.get(x.codePointAt(0))); + const cps = [...s].map((x) => x.codePointAt(0)); + const es = []; + for (let i = 0; i < cps.length; i++) { + const pair = i + 1 < cps.length ? String.fromCodePoint(cps[i], cps[i + 1]) : ""; + if (contractions.has(pair)) { + es.push(...contractions.get(pair)); + i++; + } else { + es.push(...elements.get(cps[i])); + } + } // Canonical reordering: stable sort each run of movable marks by class. for (let i = 1; i < es.length; i++) { const e = es[i]; @@ -311,12 +357,38 @@ const check = (a, b) => { }; for (const x of pool) for (const y of pool) check(ch(x), ch(y)); for (let i = 0; i < 300000; i++) check(word(), word()); +// Equivalent and near-equivalent spellings, which random strings rarely +// produce: each covered character with its canonical and compatibility +// decompositions and case forms, ligatures with their spelled-out pairs, in +// four contexts. These pairs tie at the primary and secondary levels, so +// they check the tertiary weights, contractions and canonical equivalence. +const inTable = (t) => [...t].every((x) => coveredSet.has(x.codePointAt(0))); +const probes = new Set(); +for (const c of covered) { + const t = ch(c); + for (const v of [t, t.normalize("NFD"), t.normalize("NFKD"), t.toUpperCase(), t.toLowerCase()]) + if (inTable(v)) probes.add(v); +} +for (const [, pair] of expansion) { + const [x, y] = [...pair]; + for (const u of [x, x.toUpperCase()]) for (const v of [y, y.toUpperCase()]) probes.add(u + v); +} +const contexts = [(t) => t, (t) => t + "a", (t) => t + "B", (t) => "x" + t + "y"]; +let probeCount = 0; +for (const f of contexts) + for (const a of probes) + for (const b of probes) { + check(f(a), f(b)); + probeCount++; + } if (mismatches > 0) { console.error(`collation model disagrees with Node in ${mismatches} cases:`); for (const s of samples) console.error(` ${s}`); process.exit(1); } -console.log(`collation model matches Node over ${pool.length ** 2 + 300000} comparisons`); +console.log( + `collation model matches Node over ${pool.length ** 2 + 300000 + probeCount} comparisons`, +); if (checkOnly) process.exit(0); // 9. Emit the C table. @@ -327,7 +399,8 @@ lines.push(" * options). Do not edit by hand. Each covered code point maps to up lines.push(" * three collation elements {primary, secondary, tertiary, combining class"); lines.push(" * rank}; zero primaries are skipped at the primary level, and runs of"); lines.push(" * marks with a nonzero class rank are put in canonical order before"); -lines.push(" * comparing. n == 255 marks an uncovered code point. */"); +lines.push(" * comparing. n == 255 marks an uncovered code point. A pair listed in"); +lines.push(" * scr_coll_contractions collates as its one entry. */"); lines.push("#ifndef SCR_COLLATION_DATA_H"); lines.push("#define SCR_COLLATION_DATA_H"); lines.push("#include "); @@ -366,6 +439,23 @@ for (const [lo, hi] of RANGES) { } lines.push(" return 0;"); lines.push("}"); +lines.push("/* Two-code-point contractions: the pair collates as one entry (l· as ŀ). */"); +lines.push("typedef struct { uint32_t first, second; ScrCollEntry entry; } ScrCollContraction;"); +lines.push(`#define SCR_COLL_CONTRACTIONS ${contractions.size}`); +lines.push( + `static const ScrCollContraction scr_coll_contractions[${Math.max(contractions.size, 1)}] = {`, +); +for (const [pair, es] of contractions) { + const [x, y] = [...pair].map((q) => q.codePointAt(0)); + if (es.length > 3) throw new Error(`contraction ${pair} needs ${es.length} elements`); + const body = es.map((e) => `{${e[0]}, ${e[1]}, ${e[2]}, ${e[3]}}`).join(", "); + const hex = (q) => "U+" + q.toString(16).toUpperCase().padStart(4, "0"); + lines.push( + ` {0x${x.toString(16)}, 0x${y.toString(16)}, {${es.length}, {${body}}}}, /* ${hex(x)} ${hex(y)} */`, + ); +} +if (contractions.size === 0) lines.push(" {0, 0, {0, {{0, 0, 0, 0}}}},"); +lines.push("};"); lines.push("#endif"); await writeFile(out, lines.join("\n") + "\n"); console.log(`wrote ${out}`); diff --git a/tests/corpus/5460-locale-compare-contractions.ts b/tests/corpus/5460-locale-compare-contractions.ts new file mode 100644 index 000000000..db65f2562 --- /dev/null +++ b/tests/corpus/5460-locale-compare-contractions.ts @@ -0,0 +1,17 @@ +// localeCompare treats "l·" and "L·" (with U+00B7 MIDDLE DOT) as one unit, +// equal to U+0140 and U+013F: the dot then orders as an accent, while +// after any other character it is punctuation, as in Node. +function sign(x: number): number { + return x < 0 ? -1 : x > 0 ? 1 : 0; +} +const dot = "\u00b7"; +const pairs: [string, string][] = [ + ["l" + dot, "\u0140"], ["L" + dot, "\u013f"], ["L" + dot + "a", "La"], ["La", "L" + dot + "a"], + ["l" + dot + "l", "ll"], ["col" + dot + "lecci\u00f3", "collecci\u00f3"], ["a" + dot + "b", "ab"], + ["l" + dot, "l"], ["l" + dot, "L" + dot], ["\u0140a", "l" + dot + "a"], ["L" + dot + "B", "l" + dot + "b"], + ["x" + dot, "x"], [dot, ""], ["l" + dot + dot, "l" + dot], ["l" + dot + "\u0301", "l\u0301" + dot], + ["\u013fB", "L" + dot + "b"], ["para" + dot + "l" + dot + "lel", "paral" + dot + "lel"], +]; +console.log(pairs.map(([a, b]) => sign(a.localeCompare(b))).join(",")); +const words = ["col" + dot + "lecci\u00f3", "collecci\u00f3", "cola", "col" + dot + "la", "Col" + dot + "lecci\u00f3", "co" + dot + "la", "c\u00f2la", "\u0140", "l", "L", "l" + dot, "L" + dot, "ll", "l" + dot + "l"]; +console.log(words.slice().sort((a, b) => a.localeCompare(b)).map((w) => JSON.stringify(w)).join(" ")); diff --git a/tests/corpus/5461-locale-compare-equivalents.ts b/tests/corpus/5461-locale-compare-equivalents.ts new file mode 100644 index 000000000..53270c06f --- /dev/null +++ b/tests/corpus/5461-locale-compare-equivalents.ts @@ -0,0 +1,29 @@ +// localeCompare over canonically equivalent and near-equivalent spellings: +// U+0130 equals I + U+0307, precomposed letters equal their decompositions +// in every context, and ligatures order against their spelled-out letters +// by case (ij < iJ < ij < Ij < IJ < IJ), as in Node. +function sign(x: number): number { + return x < 0 ? -1 : x > 0 ? 1 : 0; +} +const contexts = [(s: string) => s, (s: string) => s + "a", (s: string) => s + "B", (s: string) => "x" + s + "y"]; +const groups: string[][] = [ + ["\u0130", "I\u0307", "i\u0307", "I", "i", "\u0131"], + ["\u0133", "ij", "iJ", "Ij", "IJ", "\u0132"], + ["\u00e6", "\u00c6", "ae", "AE", "a\u00e6"], + ["\u0153", "\u0152", "oe", "OE"], + ["\u00df", "ss", "sS", "Ss", "SS"], + ["\u00e1", "a\u0301", "\u00c1", "A\u0301", "a"], + ["\u00c5", "A\u030a", "a\u030a", "\u00e5"], + ["\u00aa", "a", "A", "\u00ba", "o"], +]; +for (const group of groups) { + const rows: string[] = []; + for (const f of contexts) { + let row = ""; + for (const a of group) for (const b of group) row += sign(f(a).localeCompare(f(b))) + 1; + rows.push(row); + } + console.log(rows.join(" ")); +} +const all = groups.flat(); +console.log(all.slice().sort((a, b) => a.localeCompare(b) || (a < b ? -1 : a > b ? 1 : 0)).map((w) => JSON.stringify(w)).join(" ")); diff --git a/tests/corpus/5462-locale-compare-probe-sweep.ts b/tests/corpus/5462-locale-compare-probe-sweep.ts new file mode 100644 index 000000000..3d0440760 --- /dev/null +++ b/tests/corpus/5462-locale-compare-probe-sweep.ts @@ -0,0 +1,34 @@ +// Every covered Latin character with its canonical and compatibility +// decompositions and case forms, compared pairwise in four contexts; a +// hash of all results and their counts match Node. +const ranges: [number, number][] = [[0x09, 0x0d], [0x20, 0x7e], [0xa0, 0x17f], [0x300, 0x362]]; +const covered = new Set(); +for (const [lo, hi] of ranges) for (let c = lo; c <= hi; c++) if (c < 0xbc || c > 0xbe) covered.add(c); +function inTable(s: string): boolean { + for (const ch of s) if (!covered.has(ch.codePointAt(0)!)) return false; + return true; +} +const probes = new Set(); +for (const c of covered) { + const t = String.fromCharCode(c); + for (const v of [t, t.normalize("NFD"), t.normalize("NFKD"), t.toUpperCase(), t.toLowerCase()]) if (inTable(v)) probes.add(v); +} +for (const pair of ["ae", "oe", "ij", "ss", "l\u00b7"]) { + for (const u of [pair[0]!, pair[0]!.toUpperCase()]) for (const v of [pair.slice(1), pair.slice(1).toUpperCase()]) probes.add(u + v); +} +const list = [...probes]; +const contexts = [(s: string) => s, (s: string) => s + "a", (s: string) => s + "B", (s: string) => "x" + s + "y"]; +let hash = 2166136261; +const counts = [0, 0, 0]; +for (const f of contexts) { + for (const a of list) { + const x = f(a); + for (const b of list) { + const r = x.localeCompare(f(b)); + const v = r < 0 ? 0 : r > 0 ? 2 : 1; + counts[v] = counts[v]! + 1; + hash = Math.imul(hash ^ v, 16777619) >>> 0; + } + } +} +console.log(list.length, counts.join(","), hash.toString(16));