diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json new file mode 100644 index 000000000..c4d1f9e8f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5460-locale-compare-contractions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5460-locale-compare-contractions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json new file mode 100644 index 000000000..dedaced3d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5461-locale-compare-equivalents.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5461-locale-compare-equivalents.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json new file mode 100644 index 000000000..d6199005c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5462-locale-compare-probe-sweep.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5462-locale-compare-probe-sweep.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_collation_data.h b/packages/runtime/src/scr_collation_data.h index 8004ca861..61972a4a7 100644 --- a/packages/runtime/src/scr_collation_data.h +++ b/packages/runtime/src/scr_collation_data.h @@ -4,341 +4,342 @@ * three collation elements {primary, secondary, tertiary, combining class * rank}; zero primaries are skipped at the primary level, and runs of * marks with a nonzero class rank are put in canonical order before - * comparing. n == 255 marks an uncovered code point. */ + * comparing. n == 255 marks an uncovered code point. A pair listed in + * scr_coll_contractions collates as its one entry. */ #ifndef SCR_COLLATION_DATA_H #define SCR_COLLATION_DATA_H #include typedef struct { uint16_t p; uint8_t s; uint8_t t; uint8_t ccc; } ScrCollElem; typedef struct { uint8_t n; ScrCollElem e[3]; } ScrCollEntry; static const ScrCollEntry scr_coll_0009[5] = { - {1, {{1, 1, 1, 0}}}, /* U+0009 */ - {1, {{2, 1, 1, 0}}}, /* U+000A */ - {1, {{3, 1, 1, 0}}}, /* U+000B */ - {1, {{4, 1, 1, 0}}}, /* U+000C */ - {1, {{5, 1, 1, 0}}}, /* U+000D */ + {1, {{1, 1, 2, 0}}}, /* U+0009 */ + {1, {{2, 1, 2, 0}}}, /* U+000A */ + {1, {{3, 1, 2, 0}}}, /* U+000B */ + {1, {{4, 1, 2, 0}}}, /* U+000C */ + {1, {{5, 1, 2, 0}}}, /* U+000D */ }; static const ScrCollEntry scr_coll_0020[95] = { - {1, {{6, 1, 1, 0}}}, /* U+0020 */ - {1, {{12, 1, 1, 0}}}, /* U+0021 */ - {1, {{19, 1, 1, 0}}}, /* U+0022 */ - {1, {{35, 1, 1, 0}}}, /* U+0023 */ - {1, {{59, 1, 1, 0}}}, /* U+0024 */ - {1, {{36, 1, 1, 0}}}, /* U+0025 */ - {1, {{34, 1, 1, 0}}}, /* U+0026 */ - {1, {{18, 1, 1, 0}}}, /* U+0027 */ - {1, {{22, 1, 1, 0}}}, /* U+0028 */ - {1, {{23, 1, 1, 0}}}, /* U+0029 */ - {1, {{31, 1, 1, 0}}}, /* U+002A */ - {1, {{46, 1, 1, 0}}}, /* U+002B */ - {1, {{9, 1, 1, 0}}}, /* U+002C */ - {1, {{8, 1, 1, 0}}}, /* U+002D */ - {1, {{16, 1, 1, 0}}}, /* U+002E */ - {1, {{32, 1, 1, 0}}}, /* U+002F */ - {1, {{62, 1, 1, 0}}}, /* U+0030 */ - {1, {{63, 1, 1, 0}}}, /* U+0031 */ - {1, {{64, 1, 1, 0}}}, /* U+0032 */ - {1, {{65, 1, 1, 0}}}, /* U+0033 */ - {1, {{66, 1, 1, 0}}}, /* U+0034 */ - {1, {{67, 1, 1, 0}}}, /* U+0035 */ - {1, {{68, 1, 1, 0}}}, /* U+0036 */ - {1, {{69, 1, 1, 0}}}, /* U+0037 */ - {1, {{70, 1, 1, 0}}}, /* U+0038 */ - {1, {{71, 1, 1, 0}}}, /* U+0039 */ - {1, {{11, 1, 1, 0}}}, /* U+003A */ - {1, {{10, 1, 1, 0}}}, /* U+003B */ - {1, {{50, 1, 1, 0}}}, /* U+003C */ - {1, {{51, 1, 1, 0}}}, /* U+003D */ - {1, {{52, 1, 1, 0}}}, /* U+003E */ - {1, {{14, 1, 1, 0}}}, /* U+003F */ - {1, {{30, 1, 1, 0}}}, /* U+0040 */ - {1, {{72, 1, 2, 0}}}, /* U+0041 */ - {1, {{73, 1, 2, 0}}}, /* U+0042 */ - {1, {{74, 1, 2, 0}}}, /* U+0043 */ - {1, {{75, 1, 2, 0}}}, /* U+0044 */ - {1, {{76, 1, 2, 0}}}, /* U+0045 */ - {1, {{77, 1, 2, 0}}}, /* U+0046 */ - {1, {{78, 1, 2, 0}}}, /* U+0047 */ - {1, {{79, 1, 2, 0}}}, /* U+0048 */ - {1, {{80, 1, 2, 0}}}, /* U+0049 */ - {1, {{82, 1, 2, 0}}}, /* U+004A */ - {1, {{83, 1, 2, 0}}}, /* U+004B */ - {1, {{84, 1, 2, 0}}}, /* U+004C */ - {1, {{85, 1, 2, 0}}}, /* U+004D */ - {1, {{86, 1, 2, 0}}}, /* U+004E */ - {1, {{88, 1, 2, 0}}}, /* U+004F */ - {1, {{89, 1, 2, 0}}}, /* U+0050 */ - {1, {{90, 1, 2, 0}}}, /* U+0051 */ - {1, {{92, 1, 2, 0}}}, /* U+0052 */ - {1, {{93, 1, 2, 0}}}, /* U+0053 */ - {1, {{94, 1, 2, 0}}}, /* U+0054 */ - {1, {{96, 1, 2, 0}}}, /* U+0055 */ - {1, {{97, 1, 2, 0}}}, /* U+0056 */ - {1, {{98, 1, 2, 0}}}, /* U+0057 */ - {1, {{99, 1, 2, 0}}}, /* U+0058 */ - {1, {{100, 1, 2, 0}}}, /* U+0059 */ - {1, {{101, 1, 2, 0}}}, /* U+005A */ - {1, {{24, 1, 1, 0}}}, /* U+005B */ - {1, {{33, 1, 1, 0}}}, /* U+005C */ - {1, {{25, 1, 1, 0}}}, /* U+005D */ - {1, {{39, 1, 1, 0}}}, /* U+005E */ - {1, {{7, 1, 1, 0}}}, /* U+005F */ - {1, {{37, 1, 1, 0}}}, /* U+0060 */ - {1, {{72, 1, 1, 0}}}, /* U+0061 */ - {1, {{73, 1, 1, 0}}}, /* U+0062 */ - {1, {{74, 1, 1, 0}}}, /* U+0063 */ - {1, {{75, 1, 1, 0}}}, /* U+0064 */ - {1, {{76, 1, 1, 0}}}, /* U+0065 */ - {1, {{77, 1, 1, 0}}}, /* U+0066 */ - {1, {{78, 1, 1, 0}}}, /* U+0067 */ - {1, {{79, 1, 1, 0}}}, /* U+0068 */ - {1, {{80, 1, 1, 0}}}, /* U+0069 */ - {1, {{82, 1, 1, 0}}}, /* U+006A */ - {1, {{83, 1, 1, 0}}}, /* U+006B */ - {1, {{84, 1, 1, 0}}}, /* U+006C */ - {1, {{85, 1, 1, 0}}}, /* U+006D */ - {1, {{86, 1, 1, 0}}}, /* U+006E */ - {1, {{88, 1, 1, 0}}}, /* U+006F */ - {1, {{89, 1, 1, 0}}}, /* U+0070 */ - {1, {{90, 1, 1, 0}}}, /* U+0071 */ - {1, {{92, 1, 1, 0}}}, /* U+0072 */ - {1, {{93, 1, 1, 0}}}, /* U+0073 */ - {1, {{94, 1, 1, 0}}}, /* U+0074 */ - {1, {{96, 1, 1, 0}}}, /* U+0075 */ - {1, {{97, 1, 1, 0}}}, /* U+0076 */ - {1, {{98, 1, 1, 0}}}, /* U+0077 */ - {1, {{99, 1, 1, 0}}}, /* U+0078 */ - {1, {{100, 1, 1, 0}}}, /* U+0079 */ - {1, {{101, 1, 1, 0}}}, /* U+007A */ - {1, {{26, 1, 1, 0}}}, /* U+007B */ - {1, {{54, 1, 1, 0}}}, /* U+007C */ - {1, {{27, 1, 1, 0}}}, /* U+007D */ - {1, {{56, 1, 1, 0}}}, /* U+007E */ + {1, {{6, 1, 2, 0}}}, /* U+0020 */ + {1, {{12, 1, 2, 0}}}, /* U+0021 */ + {1, {{19, 1, 2, 0}}}, /* U+0022 */ + {1, {{35, 1, 2, 0}}}, /* U+0023 */ + {1, {{59, 1, 2, 0}}}, /* U+0024 */ + {1, {{36, 1, 2, 0}}}, /* U+0025 */ + {1, {{34, 1, 2, 0}}}, /* U+0026 */ + {1, {{18, 1, 2, 0}}}, /* U+0027 */ + {1, {{22, 1, 2, 0}}}, /* U+0028 */ + {1, {{23, 1, 2, 0}}}, /* U+0029 */ + {1, {{31, 1, 2, 0}}}, /* U+002A */ + {1, {{46, 1, 2, 0}}}, /* U+002B */ + {1, {{9, 1, 2, 0}}}, /* U+002C */ + {1, {{8, 1, 2, 0}}}, /* U+002D */ + {1, {{16, 1, 2, 0}}}, /* U+002E */ + {1, {{32, 1, 2, 0}}}, /* U+002F */ + {1, {{62, 1, 2, 0}}}, /* U+0030 */ + {1, {{63, 1, 2, 0}}}, /* U+0031 */ + {1, {{64, 1, 2, 0}}}, /* U+0032 */ + {1, {{65, 1, 2, 0}}}, /* U+0033 */ + {1, {{66, 1, 2, 0}}}, /* U+0034 */ + {1, {{67, 1, 2, 0}}}, /* U+0035 */ + {1, {{68, 1, 2, 0}}}, /* U+0036 */ + {1, {{69, 1, 2, 0}}}, /* U+0037 */ + {1, {{70, 1, 2, 0}}}, /* U+0038 */ + {1, {{71, 1, 2, 0}}}, /* U+0039 */ + {1, {{11, 1, 2, 0}}}, /* U+003A */ + {1, {{10, 1, 2, 0}}}, /* U+003B */ + {1, {{50, 1, 2, 0}}}, /* U+003C */ + {1, {{51, 1, 2, 0}}}, /* U+003D */ + {1, {{52, 1, 2, 0}}}, /* U+003E */ + {1, {{14, 1, 2, 0}}}, /* U+003F */ + {1, {{30, 1, 2, 0}}}, /* U+0040 */ + {1, {{72, 1, 4, 0}}}, /* U+0041 */ + {1, {{73, 1, 4, 0}}}, /* U+0042 */ + {1, {{74, 1, 4, 0}}}, /* U+0043 */ + {1, {{75, 1, 4, 0}}}, /* U+0044 */ + {1, {{76, 1, 4, 0}}}, /* U+0045 */ + {1, {{77, 1, 4, 0}}}, /* U+0046 */ + {1, {{78, 1, 4, 0}}}, /* U+0047 */ + {1, {{79, 1, 4, 0}}}, /* U+0048 */ + {1, {{80, 1, 4, 0}}}, /* U+0049 */ + {1, {{82, 1, 4, 0}}}, /* U+004A */ + {1, {{83, 1, 4, 0}}}, /* U+004B */ + {1, {{84, 1, 4, 0}}}, /* U+004C */ + {1, {{85, 1, 4, 0}}}, /* U+004D */ + {1, {{86, 1, 4, 0}}}, /* U+004E */ + {1, {{88, 1, 4, 0}}}, /* U+004F */ + {1, {{89, 1, 4, 0}}}, /* U+0050 */ + {1, {{90, 1, 4, 0}}}, /* U+0051 */ + {1, {{92, 1, 4, 0}}}, /* U+0052 */ + {1, {{93, 1, 4, 0}}}, /* U+0053 */ + {1, {{94, 1, 4, 0}}}, /* U+0054 */ + {1, {{96, 1, 4, 0}}}, /* U+0055 */ + {1, {{97, 1, 4, 0}}}, /* U+0056 */ + {1, {{98, 1, 4, 0}}}, /* U+0057 */ + {1, {{99, 1, 4, 0}}}, /* U+0058 */ + {1, {{100, 1, 4, 0}}}, /* U+0059 */ + {1, {{101, 1, 4, 0}}}, /* U+005A */ + {1, {{24, 1, 2, 0}}}, /* U+005B */ + {1, {{33, 1, 2, 0}}}, /* U+005C */ + {1, {{25, 1, 2, 0}}}, /* U+005D */ + {1, {{39, 1, 2, 0}}}, /* U+005E */ + {1, {{7, 1, 2, 0}}}, /* U+005F */ + {1, {{37, 1, 2, 0}}}, /* U+0060 */ + {1, {{72, 1, 2, 0}}}, /* U+0061 */ + {1, {{73, 1, 2, 0}}}, /* U+0062 */ + {1, {{74, 1, 2, 0}}}, /* U+0063 */ + {1, {{75, 1, 2, 0}}}, /* U+0064 */ + {1, {{76, 1, 2, 0}}}, /* U+0065 */ + {1, {{77, 1, 2, 0}}}, /* U+0066 */ + {1, {{78, 1, 2, 0}}}, /* U+0067 */ + {1, {{79, 1, 2, 0}}}, /* U+0068 */ + {1, {{80, 1, 2, 0}}}, /* U+0069 */ + {1, {{82, 1, 2, 0}}}, /* U+006A */ + {1, {{83, 1, 2, 0}}}, /* U+006B */ + {1, {{84, 1, 2, 0}}}, /* U+006C */ + {1, {{85, 1, 2, 0}}}, /* U+006D */ + {1, {{86, 1, 2, 0}}}, /* U+006E */ + {1, {{88, 1, 2, 0}}}, /* U+006F */ + {1, {{89, 1, 2, 0}}}, /* U+0070 */ + {1, {{90, 1, 2, 0}}}, /* U+0071 */ + {1, {{92, 1, 2, 0}}}, /* U+0072 */ + {1, {{93, 1, 2, 0}}}, /* U+0073 */ + {1, {{94, 1, 2, 0}}}, /* U+0074 */ + {1, {{96, 1, 2, 0}}}, /* U+0075 */ + {1, {{97, 1, 2, 0}}}, /* U+0076 */ + {1, {{98, 1, 2, 0}}}, /* U+0077 */ + {1, {{99, 1, 2, 0}}}, /* U+0078 */ + {1, {{100, 1, 2, 0}}}, /* U+0079 */ + {1, {{101, 1, 2, 0}}}, /* U+007A */ + {1, {{26, 1, 2, 0}}}, /* U+007B */ + {1, {{54, 1, 2, 0}}}, /* U+007C */ + {1, {{27, 1, 2, 0}}}, /* U+007D */ + {1, {{56, 1, 2, 0}}}, /* U+007E */ }; static const ScrCollEntry scr_coll_00a0[224] = { - {1, {{6, 1, 2, 0}}}, /* U+00A0 */ - {1, {{13, 1, 1, 0}}}, /* U+00A1 */ - {1, {{58, 1, 1, 0}}}, /* U+00A2 */ - {1, {{60, 1, 1, 0}}}, /* U+00A3 */ - {1, {{57, 1, 1, 0}}}, /* U+00A4 */ - {1, {{61, 1, 1, 0}}}, /* U+00A5 */ - {1, {{55, 1, 1, 0}}}, /* U+00A6 */ - {1, {{28, 1, 1, 0}}}, /* U+00A7 */ - {1, {{41, 1, 1, 0}}}, /* U+00A8 */ - {1, {{44, 1, 1, 0}}}, /* U+00A9 */ - {1, {{72, 1, 3, 0}}}, /* U+00AA */ - {1, {{20, 1, 1, 0}}}, /* U+00AB */ - {1, {{53, 1, 1, 0}}}, /* U+00AC */ + {1, {{6, 1, 4, 0}}}, /* U+00A0 */ + {1, {{13, 1, 2, 0}}}, /* U+00A1 */ + {1, {{58, 1, 2, 0}}}, /* U+00A2 */ + {1, {{60, 1, 2, 0}}}, /* U+00A3 */ + {1, {{57, 1, 2, 0}}}, /* U+00A4 */ + {1, {{61, 1, 2, 0}}}, /* U+00A5 */ + {1, {{55, 1, 2, 0}}}, /* U+00A6 */ + {1, {{28, 1, 2, 0}}}, /* U+00A7 */ + {1, {{41, 1, 2, 0}}}, /* U+00A8 */ + {1, {{44, 1, 2, 0}}}, /* U+00A9 */ + {1, {{72, 1, 6, 0}}}, /* U+00AA */ + {1, {{20, 1, 2, 0}}}, /* U+00AB */ + {1, {{53, 1, 2, 0}}}, /* U+00AC */ {0, {}}, /* U+00AD */ - {1, {{45, 1, 1, 0}}}, /* U+00AE */ - {1, {{40, 1, 1, 0}}}, /* U+00AF */ - {1, {{43, 1, 1, 0}}}, /* U+00B0 */ - {1, {{47, 1, 1, 0}}}, /* U+00B1 */ - {1, {{64, 1, 2, 0}}}, /* U+00B2 */ - {1, {{65, 1, 2, 0}}}, /* U+00B3 */ - {1, {{38, 1, 1, 0}}}, /* U+00B4 */ - {1, {{104, 1, 1, 0}}}, /* U+00B5 */ - {1, {{29, 1, 1, 0}}}, /* U+00B6 */ - {1, {{17, 1, 1, 0}}}, /* U+00B7 */ - {1, {{42, 1, 1, 0}}}, /* U+00B8 */ - {1, {{63, 1, 2, 0}}}, /* U+00B9 */ - {1, {{88, 1, 3, 0}}}, /* U+00BA */ - {1, {{21, 1, 1, 0}}}, /* U+00BB */ + {1, {{45, 1, 2, 0}}}, /* U+00AE */ + {1, {{40, 1, 2, 0}}}, /* U+00AF */ + {1, {{43, 1, 2, 0}}}, /* U+00B0 */ + {1, {{47, 1, 2, 0}}}, /* U+00B1 */ + {1, {{64, 1, 4, 0}}}, /* U+00B2 */ + {1, {{65, 1, 4, 0}}}, /* U+00B3 */ + {1, {{38, 1, 2, 0}}}, /* U+00B4 */ + {1, {{104, 1, 2, 0}}}, /* U+00B5 */ + {1, {{29, 1, 2, 0}}}, /* U+00B6 */ + {1, {{17, 1, 2, 0}}}, /* U+00B7 */ + {1, {{42, 1, 2, 0}}}, /* U+00B8 */ + {1, {{63, 1, 4, 0}}}, /* U+00B9 */ + {1, {{88, 1, 6, 0}}}, /* U+00BA */ + {1, {{21, 1, 2, 0}}}, /* U+00BB */ {255, {{0, 0, 0, 0}}}, /* U+00BC not covered */ {255, {{0, 0, 0, 0}}}, /* U+00BD not covered */ {255, {{0, 0, 0, 0}}}, /* U+00BE not covered */ - {1, {{15, 1, 1, 0}}}, /* U+00BF */ - {2, {{72, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00C0 */ - {2, {{72, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00C1 */ - {2, {{72, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00C2 */ - {2, {{72, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00C3 */ - {2, {{72, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00C4 */ - {2, {{72, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+00C5 */ - {3, {{72, 1, 2, 0}, {0, 51, 1, 0}, {76, 1, 2, 0}}}, /* U+00C6 */ - {2, {{74, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+00C7 */ - {2, {{76, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00C8 */ - {2, {{76, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00C9 */ - {2, {{76, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00CA */ - {2, {{76, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00CB */ - {2, {{80, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00CC */ - {2, {{80, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00CD */ - {2, {{80, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00CE */ - {2, {{80, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00CF */ - {2, {{75, 1, 2, 0}, {0, 48, 1, 0}}}, /* U+00D0 */ - {2, {{86, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00D1 */ - {2, {{88, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00D2 */ - {2, {{88, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00D3 */ - {2, {{88, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00D4 */ - {2, {{88, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00D5 */ - {2, {{88, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00D6 */ - {1, {{49, 1, 1, 0}}}, /* U+00D7 */ - {2, {{88, 1, 2, 0}, {0, 18, 1, 0}}}, /* U+00D8 */ - {2, {{96, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00D9 */ - {2, {{96, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00DA */ - {2, {{96, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00DB */ - {2, {{96, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00DC */ - {2, {{100, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00DD */ - {1, {{102, 1, 2, 0}}}, /* U+00DE */ - {3, {{93, 1, 1, 0}, {0, 52, 1, 0}, {93, 1, 1, 0}}}, /* U+00DF */ - {2, {{72, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00E0 */ - {2, {{72, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00E1 */ - {2, {{72, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00E2 */ - {2, {{72, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00E3 */ - {2, {{72, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00E4 */ - {2, {{72, 1, 1, 0}, {0, 11, 1, 5}}}, /* U+00E5 */ - {3, {{72, 1, 1, 0}, {0, 51, 1, 0}, {76, 1, 1, 0}}}, /* U+00E6 */ - {2, {{74, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+00E7 */ - {2, {{76, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00E8 */ - {2, {{76, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00E9 */ - {2, {{76, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00EA */ - {2, {{76, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00EB */ - {2, {{80, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00EC */ - {2, {{80, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00ED */ - {2, {{80, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00EE */ - {2, {{80, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00EF */ - {2, {{75, 1, 1, 0}, {0, 48, 1, 0}}}, /* U+00F0 */ - {2, {{86, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00F1 */ - {2, {{88, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00F2 */ - {2, {{88, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00F3 */ - {2, {{88, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00F4 */ - {2, {{88, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+00F5 */ - {2, {{88, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00F6 */ - {1, {{48, 1, 1, 0}}}, /* U+00F7 */ - {2, {{88, 1, 1, 0}, {0, 18, 1, 0}}}, /* U+00F8 */ - {2, {{96, 1, 1, 0}, {0, 7, 1, 5}}}, /* U+00F9 */ - {2, {{96, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00FA */ - {2, {{96, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+00FB */ - {2, {{96, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00FC */ - {2, {{100, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+00FD */ - {1, {{102, 1, 1, 0}}}, /* U+00FE */ - {2, {{100, 1, 1, 0}, {0, 13, 1, 5}}}, /* U+00FF */ - {2, {{72, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0100 */ - {2, {{72, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+0101 */ - {2, {{72, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0102 */ - {2, {{72, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+0103 */ - {2, {{72, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0104 */ - {2, {{72, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0105 */ - {2, {{74, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0106 */ - {2, {{74, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0107 */ - {2, {{74, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0108 */ - {2, {{74, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0109 */ - {2, {{74, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+010A */ - {2, {{74, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+010B */ - {2, {{74, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010C */ - {2, {{74, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+010D */ - {2, {{75, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010E */ - {2, {{75, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+010F */ - {2, {{75, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0110 */ - {2, {{75, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0111 */ - {2, {{76, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0112 */ - {2, {{76, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+0113 */ - {2, {{76, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0114 */ - {2, {{76, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+0115 */ - {2, {{76, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0116 */ - {2, {{76, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0117 */ - {2, {{76, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0118 */ - {2, {{76, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0119 */ - {2, {{76, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+011A */ - {2, {{76, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+011B */ - {2, {{78, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+011C */ - {2, {{78, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+011D */ - {2, {{78, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+011E */ - {2, {{78, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+011F */ - {2, {{78, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0120 */ - {2, {{78, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0121 */ - {2, {{78, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0122 */ - {2, {{78, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0123 */ - {2, {{79, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0124 */ - {2, {{79, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0125 */ - {2, {{79, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0126 */ - {2, {{79, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0127 */ - {2, {{80, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0128 */ - {2, {{80, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+0129 */ - {2, {{80, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+012A */ - {2, {{80, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+012B */ - {2, {{80, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+012C */ - {2, {{80, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+012D */ - {2, {{80, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+012E */ - {2, {{80, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+012F */ - {2, {{80, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+0130 */ - {1, {{81, 1, 1, 0}}}, /* U+0131 */ - {2, {{80, 1, 2, 0}, {82, 1, 2, 0}}}, /* U+0132 */ - {2, {{80, 1, 1, 0}, {82, 1, 1, 0}}}, /* U+0133 */ - {2, {{82, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0134 */ - {2, {{82, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0135 */ - {2, {{83, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0136 */ - {2, {{83, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0137 */ - {1, {{91, 1, 1, 0}}}, /* U+0138 */ - {2, {{84, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0139 */ - {2, {{84, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+013A */ - {2, {{84, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+013B */ - {2, {{84, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+013C */ - {2, {{84, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+013D */ - {2, {{84, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+013E */ - {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}, /* U+013F */ - {2, {{84, 1, 1, 0}, {0, 49, 1, 0}}}, /* U+0140 */ - {2, {{84, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0141 */ - {2, {{84, 1, 1, 0}, {0, 25, 1, 0}}}, /* U+0142 */ - {2, {{86, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0143 */ - {2, {{86, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0144 */ - {2, {{86, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0145 */ - {2, {{86, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0146 */ - {2, {{86, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0147 */ - {2, {{86, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0148 */ - {1, {{103, 1, 1, 0}}}, /* U+0149 */ - {1, {{87, 1, 2, 0}}}, /* U+014A */ - {1, {{87, 1, 1, 0}}}, /* U+014B */ - {2, {{88, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+014C */ - {2, {{88, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+014D */ - {2, {{88, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+014E */ - {2, {{88, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+014F */ - {2, {{88, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0150 */ - {2, {{88, 1, 1, 0}, {0, 15, 1, 5}}}, /* U+0151 */ - {3, {{88, 1, 2, 0}, {0, 53, 1, 0}, {76, 1, 2, 0}}}, /* U+0152 */ - {3, {{88, 1, 1, 0}, {0, 53, 1, 0}, {76, 1, 1, 0}}}, /* U+0153 */ - {2, {{92, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0154 */ - {2, {{92, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+0155 */ - {2, {{92, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0156 */ - {2, {{92, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0157 */ - {2, {{92, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0158 */ - {2, {{92, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0159 */ - {2, {{93, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+015A */ - {2, {{93, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+015B */ - {2, {{93, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+015C */ - {2, {{93, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+015D */ - {2, {{93, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+015E */ - {2, {{93, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+015F */ - {2, {{93, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0160 */ - {2, {{93, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0161 */ - {2, {{94, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0162 */ - {2, {{94, 1, 1, 0}, {0, 19, 1, 2}}}, /* U+0163 */ - {2, {{94, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0164 */ - {2, {{94, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+0165 */ - {1, {{95, 1, 2, 0}}}, /* U+0166 */ - {1, {{95, 1, 1, 0}}}, /* U+0167 */ - {2, {{96, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0168 */ - {2, {{96, 1, 1, 0}, {0, 16, 1, 5}}}, /* U+0169 */ - {2, {{96, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+016A */ - {2, {{96, 1, 1, 0}, {0, 21, 1, 5}}}, /* U+016B */ - {2, {{96, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+016C */ - {2, {{96, 1, 1, 0}, {0, 8, 1, 5}}}, /* U+016D */ - {2, {{96, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+016E */ - {2, {{96, 1, 1, 0}, {0, 11, 1, 5}}}, /* U+016F */ - {2, {{96, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0170 */ - {2, {{96, 1, 1, 0}, {0, 15, 1, 5}}}, /* U+0171 */ - {2, {{96, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0172 */ - {2, {{96, 1, 1, 0}, {0, 20, 1, 2}}}, /* U+0173 */ - {2, {{98, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0174 */ - {2, {{98, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0175 */ - {2, {{100, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0176 */ - {2, {{100, 1, 1, 0}, {0, 9, 1, 5}}}, /* U+0177 */ - {2, {{100, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+0178 */ - {2, {{101, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0179 */ - {2, {{101, 1, 1, 0}, {0, 6, 1, 5}}}, /* U+017A */ - {2, {{101, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+017B */ - {2, {{101, 1, 1, 0}, {0, 17, 1, 5}}}, /* U+017C */ - {2, {{101, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+017D */ - {2, {{101, 1, 1, 0}, {0, 10, 1, 5}}}, /* U+017E */ - {2, {{93, 1, 1, 0}, {0, 50, 1, 0}}}, /* U+017F */ + {1, {{15, 1, 2, 0}}}, /* U+00BF */ + {2, {{72, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00C0 */ + {2, {{72, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00C1 */ + {2, {{72, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00C2 */ + {2, {{72, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00C3 */ + {2, {{72, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00C4 */ + {2, {{72, 1, 4, 0}, {0, 11, 1, 5}}}, /* U+00C5 */ + {3, {{72, 1, 4, 0}, {0, 51, 1, 0}, {76, 1, 4, 0}}}, /* U+00C6 */ + {2, {{74, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+00C7 */ + {2, {{76, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00C8 */ + {2, {{76, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00C9 */ + {2, {{76, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00CA */ + {2, {{76, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00CB */ + {2, {{80, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00CC */ + {2, {{80, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00CD */ + {2, {{80, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00CE */ + {2, {{80, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00CF */ + {2, {{75, 1, 4, 0}, {0, 48, 1, 0}}}, /* U+00D0 */ + {2, {{86, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00D1 */ + {2, {{88, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00D2 */ + {2, {{88, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00D3 */ + {2, {{88, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00D4 */ + {2, {{88, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+00D5 */ + {2, {{88, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00D6 */ + {1, {{49, 1, 2, 0}}}, /* U+00D7 */ + {2, {{88, 1, 4, 0}, {0, 18, 1, 0}}}, /* U+00D8 */ + {2, {{96, 1, 4, 0}, {0, 7, 1, 5}}}, /* U+00D9 */ + {2, {{96, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00DA */ + {2, {{96, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+00DB */ + {2, {{96, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+00DC */ + {2, {{100, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+00DD */ + {1, {{102, 1, 4, 0}}}, /* U+00DE */ + {3, {{93, 1, 2, 0}, {0, 52, 1, 0}, {93, 1, 2, 0}}}, /* U+00DF */ + {2, {{72, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00E0 */ + {2, {{72, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00E1 */ + {2, {{72, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00E2 */ + {2, {{72, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00E3 */ + {2, {{72, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00E4 */ + {2, {{72, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+00E5 */ + {3, {{72, 1, 2, 0}, {0, 51, 1, 0}, {76, 1, 2, 0}}}, /* U+00E6 */ + {2, {{74, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+00E7 */ + {2, {{76, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00E8 */ + {2, {{76, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00E9 */ + {2, {{76, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00EA */ + {2, {{76, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00EB */ + {2, {{80, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00EC */ + {2, {{80, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00ED */ + {2, {{80, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00EE */ + {2, {{80, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00EF */ + {2, {{75, 1, 2, 0}, {0, 48, 1, 0}}}, /* U+00F0 */ + {2, {{86, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00F1 */ + {2, {{88, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00F2 */ + {2, {{88, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00F3 */ + {2, {{88, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00F4 */ + {2, {{88, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+00F5 */ + {2, {{88, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00F6 */ + {1, {{48, 1, 2, 0}}}, /* U+00F7 */ + {2, {{88, 1, 2, 0}, {0, 18, 1, 0}}}, /* U+00F8 */ + {2, {{96, 1, 2, 0}, {0, 7, 1, 5}}}, /* U+00F9 */ + {2, {{96, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00FA */ + {2, {{96, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+00FB */ + {2, {{96, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00FC */ + {2, {{100, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+00FD */ + {1, {{102, 1, 2, 0}}}, /* U+00FE */ + {2, {{100, 1, 2, 0}, {0, 13, 1, 5}}}, /* U+00FF */ + {2, {{72, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+0100 */ + {2, {{72, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0101 */ + {2, {{72, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+0102 */ + {2, {{72, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0103 */ + {2, {{72, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0104 */ + {2, {{72, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0105 */ + {2, {{74, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0106 */ + {2, {{74, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0107 */ + {2, {{74, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0108 */ + {2, {{74, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0109 */ + {2, {{74, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+010A */ + {2, {{74, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+010B */ + {2, {{74, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+010C */ + {2, {{74, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010D */ + {2, {{75, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+010E */ + {2, {{75, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+010F */ + {2, {{75, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0110 */ + {2, {{75, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0111 */ + {2, {{76, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+0112 */ + {2, {{76, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+0113 */ + {2, {{76, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+0114 */ + {2, {{76, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+0115 */ + {2, {{76, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0116 */ + {2, {{76, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0117 */ + {2, {{76, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0118 */ + {2, {{76, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0119 */ + {2, {{76, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+011A */ + {2, {{76, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+011B */ + {2, {{78, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+011C */ + {2, {{78, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+011D */ + {2, {{78, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+011E */ + {2, {{78, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+011F */ + {2, {{78, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0120 */ + {2, {{78, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+0121 */ + {2, {{78, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0122 */ + {2, {{78, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0123 */ + {2, {{79, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0124 */ + {2, {{79, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0125 */ + {2, {{79, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0126 */ + {2, {{79, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0127 */ + {2, {{80, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+0128 */ + {2, {{80, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0129 */ + {2, {{80, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+012A */ + {2, {{80, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+012B */ + {2, {{80, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+012C */ + {2, {{80, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+012D */ + {2, {{80, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+012E */ + {2, {{80, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+012F */ + {2, {{80, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+0130 */ + {1, {{81, 1, 2, 0}}}, /* U+0131 */ + {2, {{80, 1, 5, 0}, {82, 1, 5, 0}}}, /* U+0132 */ + {2, {{80, 1, 3, 0}, {82, 1, 3, 0}}}, /* U+0133 */ + {2, {{82, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0134 */ + {2, {{82, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0135 */ + {2, {{83, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0136 */ + {2, {{83, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0137 */ + {1, {{91, 1, 2, 0}}}, /* U+0138 */ + {2, {{84, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0139 */ + {2, {{84, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+013A */ + {2, {{84, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+013B */ + {2, {{84, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+013C */ + {2, {{84, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+013D */ + {2, {{84, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+013E */ + {2, {{84, 1, 4, 0}, {0, 49, 1, 0}}}, /* U+013F */ + {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}, /* U+0140 */ + {2, {{84, 1, 4, 0}, {0, 25, 1, 0}}}, /* U+0141 */ + {2, {{84, 1, 2, 0}, {0, 25, 1, 0}}}, /* U+0142 */ + {2, {{86, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0143 */ + {2, {{86, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0144 */ + {2, {{86, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0145 */ + {2, {{86, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0146 */ + {2, {{86, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0147 */ + {2, {{86, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0148 */ + {1, {{103, 1, 2, 0}}}, /* U+0149 */ + {1, {{87, 1, 4, 0}}}, /* U+014A */ + {1, {{87, 1, 2, 0}}}, /* U+014B */ + {2, {{88, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+014C */ + {2, {{88, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+014D */ + {2, {{88, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+014E */ + {2, {{88, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+014F */ + {2, {{88, 1, 4, 0}, {0, 15, 1, 5}}}, /* U+0150 */ + {2, {{88, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0151 */ + {3, {{88, 1, 4, 0}, {0, 53, 1, 0}, {76, 1, 4, 0}}}, /* U+0152 */ + {3, {{88, 1, 2, 0}, {0, 53, 1, 0}, {76, 1, 2, 0}}}, /* U+0153 */ + {2, {{92, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0154 */ + {2, {{92, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+0155 */ + {2, {{92, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0156 */ + {2, {{92, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0157 */ + {2, {{92, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0158 */ + {2, {{92, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0159 */ + {2, {{93, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+015A */ + {2, {{93, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+015B */ + {2, {{93, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+015C */ + {2, {{93, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+015D */ + {2, {{93, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+015E */ + {2, {{93, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+015F */ + {2, {{93, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0160 */ + {2, {{93, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0161 */ + {2, {{94, 1, 4, 0}, {0, 19, 1, 2}}}, /* U+0162 */ + {2, {{94, 1, 2, 0}, {0, 19, 1, 2}}}, /* U+0163 */ + {2, {{94, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+0164 */ + {2, {{94, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+0165 */ + {1, {{95, 1, 4, 0}}}, /* U+0166 */ + {1, {{95, 1, 2, 0}}}, /* U+0167 */ + {2, {{96, 1, 4, 0}, {0, 16, 1, 5}}}, /* U+0168 */ + {2, {{96, 1, 2, 0}, {0, 16, 1, 5}}}, /* U+0169 */ + {2, {{96, 1, 4, 0}, {0, 21, 1, 5}}}, /* U+016A */ + {2, {{96, 1, 2, 0}, {0, 21, 1, 5}}}, /* U+016B */ + {2, {{96, 1, 4, 0}, {0, 8, 1, 5}}}, /* U+016C */ + {2, {{96, 1, 2, 0}, {0, 8, 1, 5}}}, /* U+016D */ + {2, {{96, 1, 4, 0}, {0, 11, 1, 5}}}, /* U+016E */ + {2, {{96, 1, 2, 0}, {0, 11, 1, 5}}}, /* U+016F */ + {2, {{96, 1, 4, 0}, {0, 15, 1, 5}}}, /* U+0170 */ + {2, {{96, 1, 2, 0}, {0, 15, 1, 5}}}, /* U+0171 */ + {2, {{96, 1, 4, 0}, {0, 20, 1, 2}}}, /* U+0172 */ + {2, {{96, 1, 2, 0}, {0, 20, 1, 2}}}, /* U+0173 */ + {2, {{98, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0174 */ + {2, {{98, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0175 */ + {2, {{100, 1, 4, 0}, {0, 9, 1, 5}}}, /* U+0176 */ + {2, {{100, 1, 2, 0}, {0, 9, 1, 5}}}, /* U+0177 */ + {2, {{100, 1, 4, 0}, {0, 13, 1, 5}}}, /* U+0178 */ + {2, {{101, 1, 4, 0}, {0, 6, 1, 5}}}, /* U+0179 */ + {2, {{101, 1, 2, 0}, {0, 6, 1, 5}}}, /* U+017A */ + {2, {{101, 1, 4, 0}, {0, 17, 1, 5}}}, /* U+017B */ + {2, {{101, 1, 2, 0}, {0, 17, 1, 5}}}, /* U+017C */ + {2, {{101, 1, 4, 0}, {0, 10, 1, 5}}}, /* U+017D */ + {2, {{101, 1, 2, 0}, {0, 10, 1, 5}}}, /* U+017E */ + {2, {{93, 1, 2, 0}, {0, 50, 1, 0}}}, /* U+017F */ }; static const ScrCollEntry scr_coll_0300[112] = { {1, {{0, 7, 1, 5}}}, /* U+0300 */ @@ -461,4 +462,11 @@ static const ScrCollEntry *scr_coll_lookup(uint32_t c) { if (c >= 0x300 && c <= 0x36f) return &scr_coll_0300[c - 0x300]; return 0; } +/* Two-code-point contractions: the pair collates as one entry (l· as ŀ). */ +typedef struct { uint32_t first, second; ScrCollEntry entry; } ScrCollContraction; +#define SCR_COLL_CONTRACTIONS 2 +static const ScrCollContraction scr_coll_contractions[2] = { + {0x4c, 0xb7, {2, {{84, 1, 4, 0}, {0, 49, 1, 0}}}}, /* U+004C U+00B7 */ + {0x6c, 0xb7, {2, {{84, 1, 2, 0}, {0, 49, 1, 0}}}}, /* U+006C U+00B7 */ +}; #endif diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 3a4ca9e7e..61c0481ed 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -3289,30 +3289,106 @@ typedef struct { bool ok; } ScrCollAscii; static SCR_TL ScrCollAscii scr_coll_ascii_table[128]; +/* Covered bytes that are part of no contraction: one element each. */ +static SCR_TL bool scr_coll_ascii_plain[128]; static SCR_TL bool scr_coll_ascii_ready; /* The same table for the single-pass primary comparison: covered bytes as * a bit set, their primaries, and whether any covered byte has a zero * primary (then the level-0 skip makes the byte-by-byte pass inexact). */ static SCR_TL uint64_t scr_coll_ascii_mask[2]; static SCR_TL uint16_t scr_coll_ascii_primaries[128]; -static SCR_TL bool scr_coll_ascii_zero_primary; +/* Which byte-by-byte ASCII paths are exact: a contraction of two ASCII code + * points turns both off, and a covered byte with a zero primary turns off + * the single-pass primary comparison. */ +static SCR_TL bool scr_coll_ascii_one_pass; +static SCR_TL bool scr_coll_ascii_levels; +/* Code points below 0x180 that start a contraction, as a bit set; higher + * ones are looked up in scr_coll_contractions. */ +#define SCR_COLL_BITS_LIMIT 0x180 +static SCR_TL uint64_t scr_coll_starts_bits[SCR_COLL_BITS_LIMIT / 64]; +/* scr_coll_scan's answer for one code point below SCR_COLL_SCAN_LIMIT + * (every code point the table covers): 0 for plain covered, 1 for covered + * but needing the element lists (a mark with a primary weight, or the end + * of a contraction), 2 for uncovered. */ +#define SCR_COLL_SCAN_LIMIT 0x370 +static SCR_TL uint8_t scr_coll_scan_class[SCR_COLL_SCAN_LIMIT]; -static const ScrCollAscii *scr_coll_ascii(void) { - if (__builtin_expect(!scr_coll_ascii_ready, 0)) { - for (uint32_t c = 0; c < 128; c++) { - const ScrCollEntry *entry = scr_coll_lookup(c); - bool ok = entry != NULL && entry->n == 1 && entry->e[0].ccc == 0; - scr_coll_ascii_table[c] = (ScrCollAscii){ok ? entry->e[0].p : 0, ok ? entry->e[0].s : 0, - ok ? entry->e[0].t : 0, ok}; - if (ok) scr_coll_ascii_mask[c >> 6] |= UINT64_C(1) << (c & 63); - scr_coll_ascii_primaries[c] = ok ? entry->e[0].p : 0; - if (ok && entry->e[0].p == 0) scr_coll_ascii_zero_primary = true; - } - scr_coll_ascii_ready = true; +static __attribute__((noinline)) void scr_coll_ascii_init(void) { + bool zero_primary = false, contraction = false, in_contraction[128] = {false}; + for (uint32_t c = 0; c < 128; c++) { + const ScrCollEntry *entry = scr_coll_lookup(c); + bool ok = entry != NULL && entry->n == 1 && entry->e[0].ccc == 0; + scr_coll_ascii_table[c] = (ScrCollAscii){ok ? entry->e[0].p : 0, ok ? entry->e[0].s : 0, + ok ? entry->e[0].t : 0, ok}; + if (ok) scr_coll_ascii_mask[c >> 6] |= UINT64_C(1) << (c & 63); + scr_coll_ascii_primaries[c] = ok ? entry->e[0].p : 0; + if (ok && entry->e[0].p == 0) zero_primary = true; } + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) { + uint32_t first = scr_coll_contractions[k].first, second = scr_coll_contractions[k].second; + if (first < 128) in_contraction[first] = true; + if (second < 128) in_contraction[second] = true; + if (first < 128 && second < 128) contraction = true; + if (first < SCR_COLL_BITS_LIMIT) scr_coll_starts_bits[first >> 6] |= UINT64_C(1) << (first & 63); + } + for (uint32_t c = 0; c < 128; c++) scr_coll_ascii_plain[c] = scr_coll_ascii_table[c].ok && !in_contraction[c]; + scr_coll_ascii_one_pass = !zero_primary && !contraction; + scr_coll_ascii_levels = !contraction; + for (uint32_t c = 0; c < SCR_COLL_SCAN_LIMIT; c++) { + const ScrCollEntry *entry = scr_coll_lookup(c); + uint8_t klass = entry == NULL || entry->n == 255 ? 2 : 0; + for (uint8_t i = 0; klass == 0 && i < entry->n; i++) + if (entry->e[i].ccc != 0 && entry->e[i].p != 0) klass = 1; + for (size_t k = 0; klass == 0 && k < SCR_COLL_CONTRACTIONS; k++) + if (scr_coll_contractions[k].second == c) klass = 1; + scr_coll_scan_class[c] = klass; + } + scr_coll_ascii_ready = true; +} + +/* The per-thread tables, built on first use. */ +static inline __attribute__((always_inline)) const ScrCollAscii *scr_coll_ascii(void) { + if (__builtin_expect(!scr_coll_ascii_ready, 0)) scr_coll_ascii_init(); return scr_coll_ascii_table; } +static inline __attribute__((always_inline)) uint32_t scr_coll_decode(const unsigned char **pp, + const unsigned char *end) { + const unsigned char *p = *pp; + uint32_t c = *p++; + if (c >= 0x80) { + int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; + c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; + for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); + } + *pp = p; + return c; +} + +/* The contraction of code point c with the code point at p, setting *after + * past that second code point; NULL when the pair is not a contraction. */ +static const ScrCollEntry *scr_coll_contraction(uint32_t c, const unsigned char *p, + const unsigned char *end, + const unsigned char **after) { + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) { + if (scr_coll_contractions[k].first != c || p == end) continue; + const unsigned char *q = p; + if (scr_coll_decode(&q, end) == scr_coll_contractions[k].second) { + *after = q; + return &scr_coll_contractions[k].entry; + } + } + return NULL; +} + +/* Whether code point c starts a contraction. */ +static inline bool scr_coll_starts_contraction(uint32_t c) { + if (c < SCR_COLL_BITS_LIMIT) return (scr_coll_starts_bits[c >> 6] >> (c & 63)) & 1; + for (size_t k = 0; k < SCR_COLL_CONTRACTIONS; k++) + if (scr_coll_contractions[k].first == c) return true; + return false; +} + /* Elements of a UTF-8 string, marks in canonical order; false when a code * point is outside the table. */ static bool scr_coll_elements(const ScrStr *s, ScrCollBuf *b) { @@ -3331,17 +3407,16 @@ static bool scr_coll_elements(const ScrStr *s, ScrCollBuf *b) { const unsigned char *p = (const unsigned char *)s->data; const unsigned char *end = p + s->len; while (p < end) { - uint32_t c = *p++; - if (c < 0x80 && ascii[c].ok) { + uint32_t c = *p; + if (c < 0x80 && scr_coll_ascii_plain[c]) { + p++; e[n++] = (ScrCollElem){ascii[c].p, ascii[c].s, ascii[c].t, 0}; continue; } - if (c >= 0x80) { - int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; - c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; - for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); - } - const ScrCollEntry *entry = scr_coll_lookup(c); + c = scr_coll_decode(&p, end); + const ScrCollEntry *entry = NULL; + if (scr_coll_starts_contraction(c)) entry = scr_coll_contraction(c, p, end, &p); + if (entry == NULL) entry = scr_coll_lookup(c); if (entry == NULL || entry->n == 255) { b->n = n; return false; @@ -3374,32 +3449,27 @@ typedef struct { uint8_t next; /* its next element */ } ScrCollCursor; -static uint32_t scr_coll_decode(const unsigned char **pp, const unsigned char *end) { - const unsigned char *p = *pp; - uint32_t c = *p++; - if (c >= 0x80) { - int extra = c >= 0xf0 ? 3 : c >= 0xe0 ? 2 : 1; - c &= extra == 3 ? 0x07 : extra == 2 ? 0x0f : 0x1f; - for (int i = 0; i < extra && p < end; i++) c = (c << 6) | (*p++ & 0x3f); - } - *pp = p; - return c; -} - /* 0: every code point is covered and no mark carries a primary weight; * 1: covered, with such a mark; 2: a code point outside the table. */ -static int scr_coll_scan(const ScrCollAscii *ascii, const ScrStr *s) { +static int scr_coll_scan(const ScrStr *s) { const unsigned char *p = (const unsigned char *)s->data, *end = p + s->len; int result = 0; while (p < end) { - if (*p < 0x80 && ascii[*p].ok) { + if (*p < 0x80 && scr_coll_ascii_plain[*p]) { p++; continue; } - const ScrCollEntry *entry = scr_coll_lookup(scr_coll_decode(&p, end)); - if (entry == NULL || entry->n == 255) return 2; - for (uint8_t i = 0; i < entry->n; i++) - if (entry->e[i].ccc != 0 && entry->e[i].p != 0) result = 1; + uint32_t c = scr_coll_decode(&p, end); + if (c >= SCR_COLL_SCAN_LIMIT) { + /* Above the class table: covered ones take the element lists. */ + const ScrCollEntry *entry = scr_coll_lookup(c); + if (entry == NULL || entry->n == 255) return 2; + result = 1; + continue; + } + uint8_t klass = scr_coll_scan_class[c]; + if (klass == 2) return 2; + result |= klass; } return result; } @@ -3518,20 +3588,21 @@ double scr_str_locale_compare(ScrStr *a, ScrStr *b) { /* Equal bytes have equal elements at every level. */ if (a == b || (a->len == b->len && memcmp(a->data, b->data, a->len) == 0)) return 0; const ScrCollAscii *table = scr_coll_ascii(); - if (!scr_coll_ascii_zero_primary) { + if (scr_coll_ascii_one_pass) { int r = scr_coll_ascii_primary(a, b); if (r != 2) { for (int level = 1; level < 3 && r == 0; level++) r = scr_coll_ascii_level(table, a, b, level); return (double)r; } - } else if (scr_coll_ascii_covered(table, a) && scr_coll_ascii_covered(table, b)) { + } else if (scr_coll_ascii_levels && scr_coll_ascii_covered(table, a) && + scr_coll_ascii_covered(table, b)) { int r = 0; for (int level = 0; level < 3 && r == 0; level++) r = scr_coll_ascii_level(table, a, b, level); return (double)r; } - int scan_a = scr_coll_scan(table, a); + int scan_a = scr_coll_scan(a); if (scan_a == 2) return scr_str_cmp(a, b) < 0 ? -1 : 1; /* bytes differ (checked above) */ - int scan_b = scr_coll_scan(table, b); + int scan_b = scr_coll_scan(b); if (scan_b == 2) return scr_str_cmp(a, b) < 0 ? -1 : 1; if (scan_a == 0 && scan_b == 0) { int r = scr_coll_primary(table, a, b); diff --git a/scripts/gen-collation-table.mjs b/scripts/gen-collation-table.mjs index 9afb25438..a79bd0c9a 100644 --- a/scripts/gen-collation-table.mjs +++ b/scripts/gen-collation-table.mjs @@ -186,7 +186,8 @@ for (const [c, pair] of expansion) { for (const [c, m] of variantMark) extraSecondary.set(c, markWeight.get(m)); for (const [c, m] of expansionMark) expansionSecondary.set(c, markWeight.get(m)); -// 6. Elements per code point (tertiary assigned below). +// 6. Elements of marks, ignorables and base characters; decomposable +// characters and ligatures are built from them in step 7. const COMMON = 1; const elements = new Map(); for (const c of covered) { @@ -199,38 +200,20 @@ for (const c of primaryReps) { if (extraSecondary.has(c)) ces.push([0, extraSecondary.get(c), COMMON, 0]); elements.set(c, ces); } -for (const c of covered) { - if (elements.has(c) || expansion.has(c)) continue; - // Canonically decomposable: the elements of its NFD atoms. - const nfd = [...ch(c).normalize("NFD")].map((x) => x.codePointAt(0)); - if (nfd.length === 1 && nfd[0] === c) throw new Error(`no elements for U+${c.toString(16)}`); - elements.set( - c, - nfd.flatMap((x) => { - const es = elements.get(x); - if (es === undefined) throw new Error(`U+${c.toString(16)} decomposes outside the table`); - return es.map((e) => [...e]); - }), - ); -} -for (const [c, pair] of expansion) { - const first = elements.get(pair.codePointAt(0)).map((e) => [...e]); - const second = elements.get(pair.codePointAt(1)).map((e) => [...e]); - // A ligature differs from its spelled-out pair at the secondary level: - // find the mark-equivalent secondary between the two halves, if any. - const middle = expansionSecondary.has(c) ? [[0, expansionSecondary.get(c), COMMON, 0]] : []; - elements.set(c, [...first, ...middle, ...second]); -} - -// 7. Tertiary: rank code points whose primary+secondary elements agree. +// 7. Tertiary. Base characters (primaryReps) are ranked within each group +// whose primary+secondary elements agree, at even weights. Canonically +// decomposable characters then take their atoms' elements, so they equal +// their decompositions at every level, as canonical equivalence requires. +// A ligature whose spelled-out pair ties with it at the secondary level +// (ij against ij) gets the weight Node places it at among the case forms +// of its first letter; odd weights fall between two of them. const psKey = (c) => elements .get(c) .map((e) => `${e[0]}.${e[1]}`) .join(" "); const groups = new Map(); -for (const c of covered) { - if (ignorable.has(c) || isMark(c)) continue; +for (const c of primaryReps) { const k = psKey(c); if (!groups.has(k)) groups.set(k, []); groups.get(k).push(c); @@ -241,18 +224,81 @@ for (const members of groups.values()) { let t = 1; for (let i = 0; i < members.length; i++) { if (i > 0 && variant.compare(ch(members[i - 1]), ch(members[i])) !== 0) t++; - tertiaryOf.set(members[i], t); + tertiaryOf.set(members[i], 2 * t); } } -for (const c of covered) { - if (ignorable.has(c) || isMark(c)) continue; +for (const c of primaryReps) for (const e of elements.get(c)) if (e[0] !== 0) e[2] = tertiaryOf.get(c); +const atomsOf = (c) => + [...ch(c).normalize("NFD")].flatMap((x) => elements.get(x.codePointAt(0)).map((e) => [...e])); +for (const c of covered) { + if (ignorable.has(c) || primaryReps.includes(c) || expansion.has(c)) continue; + if (isMark(c) && ch(c).normalize("NFD") === ch(c)) continue; + elements.set(c, atomsOf(c)); +} +const groupOf = (c) => { + const members = groups.get(psKey(c)) ?? [c]; + return [...members].sort((a, b) => tertiaryOf.get(a) - tertiaryOf.get(b)); +}; +for (const [c, pair] of expansion) { + const [x, y] = [...pair].map((q) => q.codePointAt(0)); + const middle = expansionSecondary.has(c) ? [[0, expansionSecondary.get(c), COMMON, 0]] : []; + const es = [...atomsOf(x), ...middle, ...atomsOf(y)]; + // A ligature with its own secondary (æ) keeps the case of its first + // letter in the ligature's case (Æ: the weight of A). + const upper = ch(c) !== ch(c).toLowerCase(); + let t = tertiaryOf.get(upper ? ch(x).toUpperCase().codePointAt(0) : x); + if (middle.length === 0) { + const gx = groupOf(x); + const gy = groupOf(y); + const low = ch(gy[0]), + high = ch(gy[gy.length - 1]); + const below = gx.filter((m) => variant.compare(ch(c), ch(m) + high) > 0); + const equal = gx.find( + (m) => variant.compare(ch(c), ch(m) + low) >= 0 && variant.compare(ch(c), ch(m) + high) <= 0, + ); + t = + equal !== undefined + ? tertiaryOf.get(equal) + : below.length > 0 + ? tertiaryOf.get(below[below.length - 1]) + 1 + : tertiaryOf.get(gx[0]) - 1; + } + for (const e of es) if (e[0] !== 0) e[2] = t; + elements.set(c, es); +} +// Contractions: a covered character that Node treats as equal to its +// two-character compatibility form (ŀ and l·) gives that pair its elements, +// where the pair's own characters would order differently ("L·a" > "La"). +const contractions = new Map(); +for (const c of covered) { + const k = ch(c).normalize("NFKD"); + const cps = [...k].map((q) => q.codePointAt(0)); + if (cps.length !== 2 || !cps.every((q) => coveredSet.has(q))) continue; + if (ch(c).normalize("NFD") === k || variant.compare(ch(c), k) !== 0) continue; + const own = JSON.stringify(elements.get(c)); + const spelled = JSON.stringify(cps.flatMap((q) => elements.get(q))); + if (own !== spelled) + contractions.set( + k, + elements.get(c).map((e) => [...e]), + ); } // The model's comparison (mirrored by scr_str_locale_compare in C). export function modelCompare(a, b) { const ces = (s) => { - const es = [...s].flatMap((x) => elements.get(x.codePointAt(0))); + const cps = [...s].map((x) => x.codePointAt(0)); + const es = []; + for (let i = 0; i < cps.length; i++) { + const pair = i + 1 < cps.length ? String.fromCodePoint(cps[i], cps[i + 1]) : ""; + if (contractions.has(pair)) { + es.push(...contractions.get(pair)); + i++; + } else { + es.push(...elements.get(cps[i])); + } + } // Canonical reordering: stable sort each run of movable marks by class. for (let i = 1; i < es.length; i++) { const e = es[i]; @@ -311,12 +357,38 @@ const check = (a, b) => { }; for (const x of pool) for (const y of pool) check(ch(x), ch(y)); for (let i = 0; i < 300000; i++) check(word(), word()); +// Equivalent and near-equivalent spellings, which random strings rarely +// produce: each covered character with its canonical and compatibility +// decompositions and case forms, ligatures with their spelled-out pairs, in +// four contexts. These pairs tie at the primary and secondary levels, so +// they check the tertiary weights, contractions and canonical equivalence. +const inTable = (t) => [...t].every((x) => coveredSet.has(x.codePointAt(0))); +const probes = new Set(); +for (const c of covered) { + const t = ch(c); + for (const v of [t, t.normalize("NFD"), t.normalize("NFKD"), t.toUpperCase(), t.toLowerCase()]) + if (inTable(v)) probes.add(v); +} +for (const [, pair] of expansion) { + const [x, y] = [...pair]; + for (const u of [x, x.toUpperCase()]) for (const v of [y, y.toUpperCase()]) probes.add(u + v); +} +const contexts = [(t) => t, (t) => t + "a", (t) => t + "B", (t) => "x" + t + "y"]; +let probeCount = 0; +for (const f of contexts) + for (const a of probes) + for (const b of probes) { + check(f(a), f(b)); + probeCount++; + } if (mismatches > 0) { console.error(`collation model disagrees with Node in ${mismatches} cases:`); for (const s of samples) console.error(` ${s}`); process.exit(1); } -console.log(`collation model matches Node over ${pool.length ** 2 + 300000} comparisons`); +console.log( + `collation model matches Node over ${pool.length ** 2 + 300000 + probeCount} comparisons`, +); if (checkOnly) process.exit(0); // 9. Emit the C table. @@ -327,7 +399,8 @@ lines.push(" * options). Do not edit by hand. Each covered code point maps to up lines.push(" * three collation elements {primary, secondary, tertiary, combining class"); lines.push(" * rank}; zero primaries are skipped at the primary level, and runs of"); lines.push(" * marks with a nonzero class rank are put in canonical order before"); -lines.push(" * comparing. n == 255 marks an uncovered code point. */"); +lines.push(" * comparing. n == 255 marks an uncovered code point. A pair listed in"); +lines.push(" * scr_coll_contractions collates as its one entry. */"); lines.push("#ifndef SCR_COLLATION_DATA_H"); lines.push("#define SCR_COLLATION_DATA_H"); lines.push("#include "); @@ -366,6 +439,23 @@ for (const [lo, hi] of RANGES) { } lines.push(" return 0;"); lines.push("}"); +lines.push("/* Two-code-point contractions: the pair collates as one entry (l· as ŀ). */"); +lines.push("typedef struct { uint32_t first, second; ScrCollEntry entry; } ScrCollContraction;"); +lines.push(`#define SCR_COLL_CONTRACTIONS ${contractions.size}`); +lines.push( + `static const ScrCollContraction scr_coll_contractions[${Math.max(contractions.size, 1)}] = {`, +); +for (const [pair, es] of contractions) { + const [x, y] = [...pair].map((q) => q.codePointAt(0)); + if (es.length > 3) throw new Error(`contraction ${pair} needs ${es.length} elements`); + const body = es.map((e) => `{${e[0]}, ${e[1]}, ${e[2]}, ${e[3]}}`).join(", "); + const hex = (q) => "U+" + q.toString(16).toUpperCase().padStart(4, "0"); + lines.push( + ` {0x${x.toString(16)}, 0x${y.toString(16)}, {${es.length}, {${body}}}}, /* ${hex(x)} ${hex(y)} */`, + ); +} +if (contractions.size === 0) lines.push(" {0, 0, {0, {{0, 0, 0, 0}}}},"); +lines.push("};"); lines.push("#endif"); await writeFile(out, lines.join("\n") + "\n"); console.log(`wrote ${out}`); diff --git a/tests/corpus/5460-locale-compare-contractions.ts b/tests/corpus/5460-locale-compare-contractions.ts new file mode 100644 index 000000000..db65f2562 --- /dev/null +++ b/tests/corpus/5460-locale-compare-contractions.ts @@ -0,0 +1,17 @@ +// localeCompare treats "l·" and "L·" (with U+00B7 MIDDLE DOT) as one unit, +// equal to U+0140 and U+013F: the dot then orders as an accent, while +// after any other character it is punctuation, as in Node. +function sign(x: number): number { + return x < 0 ? -1 : x > 0 ? 1 : 0; +} +const dot = "\u00b7"; +const pairs: [string, string][] = [ + ["l" + dot, "\u0140"], ["L" + dot, "\u013f"], ["L" + dot + "a", "La"], ["La", "L" + dot + "a"], + ["l" + dot + "l", "ll"], ["col" + dot + "lecci\u00f3", "collecci\u00f3"], ["a" + dot + "b", "ab"], + ["l" + dot, "l"], ["l" + dot, "L" + dot], ["\u0140a", "l" + dot + "a"], ["L" + dot + "B", "l" + dot + "b"], + ["x" + dot, "x"], [dot, ""], ["l" + dot + dot, "l" + dot], ["l" + dot + "\u0301", "l\u0301" + dot], + ["\u013fB", "L" + dot + "b"], ["para" + dot + "l" + dot + "lel", "paral" + dot + "lel"], +]; +console.log(pairs.map(([a, b]) => sign(a.localeCompare(b))).join(",")); +const words = ["col" + dot + "lecci\u00f3", "collecci\u00f3", "cola", "col" + dot + "la", "Col" + dot + "lecci\u00f3", "co" + dot + "la", "c\u00f2la", "\u0140", "l", "L", "l" + dot, "L" + dot, "ll", "l" + dot + "l"]; +console.log(words.slice().sort((a, b) => a.localeCompare(b)).map((w) => JSON.stringify(w)).join(" ")); diff --git a/tests/corpus/5461-locale-compare-equivalents.ts b/tests/corpus/5461-locale-compare-equivalents.ts new file mode 100644 index 000000000..53270c06f --- /dev/null +++ b/tests/corpus/5461-locale-compare-equivalents.ts @@ -0,0 +1,29 @@ +// localeCompare over canonically equivalent and near-equivalent spellings: +// U+0130 equals I + U+0307, precomposed letters equal their decompositions +// in every context, and ligatures order against their spelled-out letters +// by case (ij < iJ < ij < Ij < IJ < IJ), as in Node. +function sign(x: number): number { + return x < 0 ? -1 : x > 0 ? 1 : 0; +} +const contexts = [(s: string) => s, (s: string) => s + "a", (s: string) => s + "B", (s: string) => "x" + s + "y"]; +const groups: string[][] = [ + ["\u0130", "I\u0307", "i\u0307", "I", "i", "\u0131"], + ["\u0133", "ij", "iJ", "Ij", "IJ", "\u0132"], + ["\u00e6", "\u00c6", "ae", "AE", "a\u00e6"], + ["\u0153", "\u0152", "oe", "OE"], + ["\u00df", "ss", "sS", "Ss", "SS"], + ["\u00e1", "a\u0301", "\u00c1", "A\u0301", "a"], + ["\u00c5", "A\u030a", "a\u030a", "\u00e5"], + ["\u00aa", "a", "A", "\u00ba", "o"], +]; +for (const group of groups) { + const rows: string[] = []; + for (const f of contexts) { + let row = ""; + for (const a of group) for (const b of group) row += sign(f(a).localeCompare(f(b))) + 1; + rows.push(row); + } + console.log(rows.join(" ")); +} +const all = groups.flat(); +console.log(all.slice().sort((a, b) => a.localeCompare(b) || (a < b ? -1 : a > b ? 1 : 0)).map((w) => JSON.stringify(w)).join(" ")); diff --git a/tests/corpus/5462-locale-compare-probe-sweep.ts b/tests/corpus/5462-locale-compare-probe-sweep.ts new file mode 100644 index 000000000..3d0440760 --- /dev/null +++ b/tests/corpus/5462-locale-compare-probe-sweep.ts @@ -0,0 +1,34 @@ +// Every covered Latin character with its canonical and compatibility +// decompositions and case forms, compared pairwise in four contexts; a +// hash of all results and their counts match Node. +const ranges: [number, number][] = [[0x09, 0x0d], [0x20, 0x7e], [0xa0, 0x17f], [0x300, 0x362]]; +const covered = new Set(); +for (const [lo, hi] of ranges) for (let c = lo; c <= hi; c++) if (c < 0xbc || c > 0xbe) covered.add(c); +function inTable(s: string): boolean { + for (const ch of s) if (!covered.has(ch.codePointAt(0)!)) return false; + return true; +} +const probes = new Set(); +for (const c of covered) { + const t = String.fromCharCode(c); + for (const v of [t, t.normalize("NFD"), t.normalize("NFKD"), t.toUpperCase(), t.toLowerCase()]) if (inTable(v)) probes.add(v); +} +for (const pair of ["ae", "oe", "ij", "ss", "l\u00b7"]) { + for (const u of [pair[0]!, pair[0]!.toUpperCase()]) for (const v of [pair.slice(1), pair.slice(1).toUpperCase()]) probes.add(u + v); +} +const list = [...probes]; +const contexts = [(s: string) => s, (s: string) => s + "a", (s: string) => s + "B", (s: string) => "x" + s + "y"]; +let hash = 2166136261; +const counts = [0, 0, 0]; +for (const f of contexts) { + for (const a of list) { + const x = f(a); + for (const b of list) { + const r = x.localeCompare(f(b)); + const v = r < 0 ? 0 : r > 0 ? 2 : 1; + counts[v] = counts[v]! + 1; + hash = Math.imul(hash ^ v, 16777619) >>> 0; + } + } +} +console.log(list.length, counts.join(","), hash.toString(16));