diff --git a/docs/content/limitations/strings-and-sorting.mdx b/docs/content/limitations/strings-and-sorting.mdx index eacca198e..b1616f9f4 100644 --- a/docs/content/limitations/strings-and-sorting.mdx +++ b/docs/content/limitations/strings-and-sorting.mdx @@ -1,4 +1,4 @@ -Strings are stored as UTF-8, with lone surrogates kept as their own code units. Length and string methods use UTF-16 semantics, including for lone surrogates, but relational comparisons such as `<` and `>` use code-point order. `split` with a string separator and `replace` or `replaceAll` with a string pattern don't match a lone surrogate against half of a surrogate pair; regular-expression patterns do. Output, file contents, and other encoded text write each lone surrogate as U+FFFD, as Node does. File system paths and network host names that contain a lone surrogate reach the operating system unconverted. +Strings are stored as UTF-8, with lone surrogates kept as their own code units. Length, string methods and relational comparisons such as `<` and `>` use UTF-16 semantics, including for lone surrogates. `split` with a string separator and `replace` or `replaceAll` with a string pattern don't match a lone surrogate against half of a surrogate pair; regular-expression patterns do. Output, file contents, and other encoded text write each lone surrogate as U+FFFD, as Node does. File system paths and network host names that contain a lone surrogate reach the operating system unconverted. One-argument `localeCompare` follows Node's default root-locale collation for Latin-script text (ASCII, Latin-1 Supplement, Latin Extended-A, combining diacritical marks, and U+2044 FRACTION SLASH): letters order before case and accents, so `"a".localeCompare("B")` is `-1`. Strings containing other characters compare by code point, which can differ from Node's ICU collation; `SC6003` reports such string literals. The locales and options arguments are unsupported. diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 9e2d3524e..a16f129ef 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -3671,13 +3671,30 @@ export class LlEmitter { } defs.push(...helper.define(this.sizeType, FN_ATTRS, scanInline), ``); } - // Ordering also decides inline when the first bytes differ: byte order - // is the plain comparison's order, and the UTF-16 order whenever both - // bytes are ASCII. Callers only test the sign of the result. - for (const fn of ["scr_str_cmp", "scr_str_cmp_u16"]) { - if (!this.decls.has(`declare i32 @${fn}(ptr, ptr)`)) continue; + // Ordering decides inline from the first differing bytes. The bytes + // before them are equal, so both strings are at the same position of a + // character: either both bytes continue characters with the same lead + // byte, or both start a character. Byte order is the UTF-16 order for + // two continuation bytes, and for two lead bytes when one starts a + // character below U+D000, whose UTF-16 unit is below every unit of a + // character with a lead byte of at least 0xED. Between lead bytes of at + // least 0xED, a supplementary character (0xF0..0xF4, a high surrogate + // unit) orders below U+E000..U+FFFF (0xEE, 0xEF), and a lead byte of + // 0xED (U+D000..U+D7FF or a lone surrogate) takes the runtime. + // Differing words without a byte of at least 0xED decide in byte + // order; other differing words decide by their first differing byte, + // and strings shorter than a word by their first byte. Otherwise a + // proven-ASCII string on either side orders by bytes (scr_str_cmp), and + // other pairs call scr_str_cmp_u16_bytes, which only reads the two + // strings, so LLVM can merge repeated comparisons of one pair. Callers + // only test the sign of the result. + if (this.decls.has(`declare i32 @scr_str_cmp_u16(ptr, ptr)`)) { + const fn = "scr_str_cmp_u16"; this.declare("declare i64 @llvm.bswap.i64(i64)"); - const utf16 = fn === "scr_str_cmp_u16"; + this.declare("declare i64 @llvm.cttz.i64(i64, i1 immarg)"); + this.declare("declare i8 @llvm.umin.i8(i8, i8)"); + this.declare("declare i32 @scr_str_cmp(ptr, ptr)"); + this.declare("declare i32 @scr_str_cmp_u16_bytes(ptr, ptr)"); defs.push( `define internal i32 @${fn.replace(/^scr_/, "sc_")}(ptr %a, ptr %b) ${FN_ATTRS} {`, `entry:`, @@ -3692,8 +3709,7 @@ export class LlEmitter { ` %bempty = icmp eq ${this.sizeType} %blen, 0`, ` %empty = or i1 %aempty, %bempty`, ` br i1 %empty, label %order, label %words`, - // Both at least 8 bytes: the first differing byte of one word pair. - // Byte-swapped little-endian words compare in byte order. + // Both at least 8 bytes: the first word of each. `words:`, ` %along = icmp uge ${this.sizeType} %alen, 8`, ` %blong = icmp uge ${this.sizeType} %blen, 8`, @@ -3705,22 +3721,7 @@ export class LlEmitter { ` %aw = load i64, ptr %awp, align 1`, ` %bw = load i64, ptr %bwp, align 1`, ` %wdiffer = icmp ne i64 %aw, %bw`, - ` br i1 %wdiffer, label %wdiff, label %wsame`, - `wdiff:`, - ...(utf16 - ? [ - ` %wbits = or i64 %aw, %bw`, - ` %whigh = and i64 %wbits, -9187201950435737472`, - ` %wascii = icmp eq i64 %whigh, 0`, - ` br i1 %wascii, label %wordorder, label %order`, - ] - : [` br label %wordorder`]), - `wordorder:`, - ` %asw = call i64 @llvm.bswap.i64(i64 %aw)`, - ` %bsw = call i64 @llvm.bswap.i64(i64 %bw)`, - ` %wbelow = icmp ult i64 %asw, %bsw`, - ` %wsign = select i1 %wbelow, i32 -1, i32 1`, - ` ret i32 %wsign`, + ` br i1 %wdiffer, label %differ, label %wsame`, // Equal first words and one length of at most 16 bytes: the word // ending at that length covers the rest, and any difference lies // past the equal first word. @@ -3736,42 +3737,89 @@ export class LlEmitter { ` %at = load i64, ptr %atp, align 1`, ` %bt = load i64, ptr %btp, align 1`, ` %tdiffer = icmp ne i64 %at, %bt`, - ` br i1 %tdiffer, label %tdiff, label %equal`, - `tdiff:`, - ...(utf16 - ? [ - ` %tbits = or i64 %at, %bt`, - ` %thigh = and i64 %tbits, -9187201950435737472`, - ` %tascii = icmp eq i64 %thigh, 0`, - ` br i1 %tascii, label %tailorder, label %order`, - ] - : [` br label %tailorder`]), - `tailorder:`, - ` %ats = call i64 @llvm.bswap.i64(i64 %at)`, - ` %bts = call i64 @llvm.bswap.i64(i64 %bt)`, - ` %tbelow = icmp ult i64 %ats, %bts`, - ` %tsign = select i1 %tbelow, i32 -1, i32 1`, - ` ret i32 %tsign`, + ` br i1 %tdiffer, label %differ, label %equal`, `first:`, ` %adata = getelementptr inbounds %ScrStr, ptr %a, i32 1`, ` %bdata = getelementptr inbounds %ScrStr, ptr %b, i32 1`, ` %x = load i8, ptr %adata`, ` %y = load i8, ptr %bdata`, - ` %differ = icmp ne i8 %x, %y`, - ...(utf16 - ? [ - ` %bits = or i8 %x, %y`, - ` %ascii = icmp sge i8 %bits, 0`, - ` %decide = and i1 %differ, %ascii`, - ] - : [` %decide = and i1 %differ, true`]), - ` br i1 %decide, label %byte, label %order`, + ` %fdiffer = icmp ne i8 %x, %y`, + ` br i1 %fdiffer, label %bytes, label %order`, + // Two differing words. When neither has a byte of at least 0xED + // (a byte b is when its high bit and bit 7 of (b & 0x7f) + 0x13 are + // set), byte-swapped little-endian words compare in byte order. + // Otherwise the lowest set byte of their xor is the first differing + // byte. + `differ:`, + ` %dx = phi i64 [ %aw, %word ], [ %at, %tail ]`, + ` %dy = phi i64 [ %bw, %word ], [ %bt, %tail ]`, + ` %xl = and i64 %dx, 9187201950435737471`, + ` %yl = and i64 %dy, 9187201950435737471`, + ` %xa = add i64 %xl, 1374463283923456787`, + ` %ya = add i64 %yl, 1374463283923456787`, + ` %xh = and i64 %dx, %xa`, + ` %yh = and i64 %dy, %ya`, + ` %hh = or i64 %xh, %yh`, + ` %whigh = and i64 %hh, -9187201950435737472`, + ` %dlow = icmp eq i64 %whigh, 0`, + ` br i1 %dlow, label %wordorder, label %locate`, + `wordorder:`, + ` %xsw = call i64 @llvm.bswap.i64(i64 %dx)`, + ` %ysw = call i64 @llvm.bswap.i64(i64 %dy)`, + ` %wbelow = icmp ult i64 %xsw, %ysw`, + ` %wsign = select i1 %wbelow, i32 -1, i32 1`, + ` ret i32 %wsign`, + `locate:`, + ` %dxor = xor i64 %dx, %dy`, + ` %dtz = call i64 @llvm.cttz.i64(i64 %dxor, i1 true)`, + ` %dshift = and i64 %dtz, 56`, + ` %dxs = lshr i64 %dx, %dshift`, + ` %dys = lshr i64 %dy, %dshift`, + ` %dxb = trunc i64 %dxs to i8`, + ` %dyb = trunc i64 %dys to i8`, + ` br label %bytes`, + // The first differing bytes. + `bytes:`, + ` %bx = phi i8 [ %x, %first ], [ %dxb, %locate ]`, + ` %by = phi i8 [ %y, %first ], [ %dyb, %locate ]`, + ` %bmin = call i8 @llvm.umin.i8(i8 %bx, i8 %by)`, + ` %blow = icmp ult i8 %bmin, -19`, + ` br i1 %blow, label %byte, label %high`, + // Both at least 0xED. A lone surrogate (0xED) takes the runtime; + // supplementary lead bytes (0xF0..0xF4) order below 0xEE and 0xEF + // once 0x10 is subtracted from them. + `high:`, + ` %xed = icmp eq i8 %bx, -19`, + ` %yed = icmp eq i8 %by, -19`, + ` %ed = or i1 %xed, %yed`, + ` br i1 %ed, label %order, label %planes`, + `planes:`, + ` %xsupp = icmp uge i8 %bx, -16`, + ` %ysupp = icmp uge i8 %by, -16`, + ` %xlow = add i8 %bx, -16`, + ` %ylow = add i8 %by, -16`, + ` %xk = select i1 %xsupp, i8 %xlow, i8 %bx`, + ` %yk = select i1 %ysupp, i8 %ylow, i8 %by`, + ` %kbelow = icmp ult i8 %xk, %yk`, + ` %ksign = select i1 %kbelow, i32 -1, i32 1`, + ` ret i32 %ksign`, `byte:`, - ` %below = icmp ult i8 %x, %y`, + ` %below = icmp ult i8 %bx, %by`, ` %sign = select i1 %below, i32 -1, i32 1`, ` ret i32 %sign`, `order:`, - ` %c = call i32 @${fn}(ptr %a, ptr %b)`, + ` %acw.p = getelementptr inbounds %ScrStr, ptr %a, i32 0, i32 2`, + ` %bcw.p = getelementptr inbounds %ScrStr, ptr %b, i32 0, i32 2`, + ` %acw = load i32, ptr %acw.p`, + ` %bcw = load i32, ptr %bcw.p`, + ` %cws = or i32 %acw, %bcw`, + ` %anyascii = icmp slt i32 %cws, 0`, + ` br i1 %anyascii, label %memorder, label %units`, + `memorder:`, + ` %m = call i32 @scr_str_cmp(ptr %a, ptr %b)`, + ` ret i32 %m`, + `units:`, + ` %c = call i32 @scr_str_cmp_u16_bytes(ptr %a, ptr %b) nounwind willreturn memory(argmem: read)`, ` ret i32 %c`, `equal:`, ` ret i32 0`, diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 4e7d418f8..1cab02a9d 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -602,7 +602,7 @@ export function emitStringExpr( } case "strCmp": { const [l, r] = emitStringInputs(host, [e.left, e.right]); - const fn = e.utf16 === true ? "scr_str_cmp_u16" : "scr_str_cmp"; + const fn = "scr_str_cmp_u16"; host.declare(`declare i32 @${fn}(ptr, ptr)`); const c = B.tmp(); const t = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts index bf140d2e6..8898b86d3 100644 --- a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts +++ b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts @@ -432,7 +432,6 @@ test("borrowed literals keep UTF-16 string comparison without temporary owners", op: "<", left: str("a"), right: str("b"), - utf16: true, type: BOOL, loc, }); @@ -443,16 +442,14 @@ test("borrowed literals keep UTF-16 string comparison without temporary owners", }); test("string comparisons decide identity, lengths and leading bytes before the runtime", () => { - const order = (utf16: boolean) => - fn(`order${utf16 ? 16 : 8}`, [], { - kind: "strCmp", - op: "<", - left: str("apple"), - right: str("apricot"), - utf16, - type: BOOL, - loc, - }); + const order = fn("order", [], { + kind: "strCmp", + op: "<", + left: str("apple"), + right: str("apricot"), + type: BOOL, + loc, + }); const same = fn("same", ["a", "b"], { kind: "strEq", left: { kind: "varRef", localId: "a", type: STRING, loc }, @@ -462,23 +459,28 @@ test("string comparisons decide identity, lengths and leading bytes before the r loc, }); for (const bits of [32, 64] as const) { - const ir = emitLlvmModule(mod(order(false), order(true), same), { pointerBits: bits }); + const ir = emitLlvmModule(mod(order, same), { pointerBits: bits }); const helper = (name: string) => new RegExp(`^define internal [^\\n]*@${name}(?:\\.i32)?\\([^]*?^}`, "m").exec(ir)![0]; const eq = helper("sc_str_eq"); expect(eq).toContain("icmp eq ptr %a, %b"); expect(eq).toMatch(/icmp eq i(32|64) %alen, %blen/); expect(eq).toContain("call zeroext i1 @scr_str_eq(ptr %a, ptr %b)"); - const bytes = helper("sc_str_cmp"); - expect(bytes).toContain("@llvm.bswap.i64"); - expect(bytes).not.toContain("%ascii"); - expect(bytes).toContain("call i32 @scr_str_cmp(ptr %a, ptr %b)"); const units = helper("sc_str_cmp_u16"); - expect(units).toContain("%wascii = icmp eq i64 %whigh, 0"); - expect(units).toContain("%tascii = icmp eq i64 %thigh, 0"); - expect(bytes).toContain("%tailable = and i1 %samelen, %short"); - expect(units).toContain("%ascii = icmp sge i8 %bits, 0"); - expect(units).toContain("call i32 @scr_str_cmp_u16(ptr %a, ptr %b)"); + expect(units).toContain("%dx = phi i64 [ %aw, %word ], [ %at, %tail ]"); + expect(units).toContain("%dlow = icmp eq i64 %whigh, 0"); + expect(units).toContain("@llvm.bswap.i64"); + expect(units).toContain("%dtz = call i64 @llvm.cttz.i64(i64 %dxor, i1 true)"); + expect(units).toContain("%bx = phi i8 [ %x, %first ], [ %dxb, %locate ]"); + expect(units).toContain("%blow = icmp ult i8 %bmin, -19"); + expect(units).toContain("%kbelow = icmp ult i8 %xk, %yk"); + expect(units).toContain("%below = icmp ult i8 %bx, %by"); + expect(units).toContain("%tailable = and i1 %samelen, %short"); + expect(units).toContain("%anyascii = icmp slt i32 %cws, 0"); + expect(units).toContain("call i32 @scr_str_cmp(ptr %a, ptr %b)"); + expect(units).toContain( + "call i32 @scr_str_cmp_u16_bytes(ptr %a, ptr %b) nounwind willreturn memory(argmem: read)", + ); } }); diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 6c8df7733..67622719c 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -2591,18 +2591,8 @@ export type IrExpr = | { kind: "logical"; op: "&&" | "||"; left: IrExpr; right: IrExpr; type: IrType; loc: SrcLoc } | { kind: "strConcat"; left: IrExpr; right: IrExpr; type: IrType; loc: SrcLoc } | { kind: "strEq"; negated: boolean; left: IrExpr; right: IrExpr; type: IrType; loc: SrcLoc } - /** String ordering. Ordinary source comparisons omit `utf16` and retain - * scriptc's documented code-point order; the default Array sort comparator - * sets it to request ECMAScript's UTF-16 code-unit order. */ - | { - kind: "strCmp"; - op: IrStrCmpOp; - left: IrExpr; - right: IrExpr; - utf16?: boolean; - type: IrType; - loc: SrcLoc; - } + /** String ordering in ECMAScript's UTF-16 code-unit order. */ + | { kind: "strCmp"; op: IrStrCmpOp; left: IrExpr; right: IrExpr; type: IrType; loc: SrcLoc } /** f64|bool → string, JS-exact (Number::toString / "true"/"false"). * Union operands dispatch through the per-union ToString helper (arms * fenced to unit/string/f64/bool by the frontend); a CAUGHT operand is diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6023-string-compare-utf16.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6023-string-compare-utf16.ts.json new file mode 100644 index 000000000..365709f61 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6023-string-compare-utf16.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6023-string-compare-utf16.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 752c19805..8a1bfa728 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -559,10 +559,8 @@ static ScrDyn *scr_dynh_dispatch(ScrDyn *recv, const char *method, ScrDyn *const * reaching the comparator. The ordered list writes back index by index, so * a comparator that mutates the receiver mid-sort never dangles the items * being ordered. The default comparator compares ToString images - * (scr_dyn_display_buf — join's conversion) bytewise: code-POINT order, - * where JS orders UTF-16 code units — identical through the BMP, divergent - * only across the surrogate boundary (SEMANTICS.md). A comparator result - * converts with ToNumber; NaN counts as 0. */ + * (scr_dyn_display_buf — join's conversion) in UTF-16 code-unit order. A + * comparator result converts with ToNumber; NaN counts as 0. */ static ScrStr *dyn_sort_str(const ScrDyn *e) { ScrJsonBuf b; scr_jb_init(&b); @@ -584,7 +582,7 @@ static bool dyn_sort_less(void *ctx, uint64_t xs, uint64_t ys, bool *failed) { } ScrStr *xstr = dyn_sort_str(x); ScrStr *ystr = dyn_sort_str(y); - int c = scr_str_cmp(xstr, ystr); + int c = scr_str_cmp_u16(xstr, ystr); scr_str_release(xstr); scr_str_release(ystr); return c < 0; diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 43cf21b0c..8358658b5 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -6,6 +6,11 @@ #include #include #include +#if defined(__SSE2__) +#include +#elif defined(__ARM_NEON) && defined(__aarch64__) +#include +#endif /* Live heap-string count for the RC audit lane (-DSCR_RC_AUDIT): the test * harness builds with it to prove the emitted retain/release discipline @@ -836,24 +841,93 @@ static __attribute__((noinline)) int scr_str_cmp_u16_units(const ScrStr *a, cons } } +/* The memory index of the first differing byte of two words; x is their + * xor and is not zero. */ +static inline size_t scr_word_mismatch(uint64_t x) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return (size_t)__builtin_clzll(x) >> 3; +#else + return (size_t)__builtin_ctzll(x) >> 3; +#endif +} + +static inline size_t scr_word32_mismatch(uint32_t x) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return (size_t)__builtin_clz(x) >> 3; +#else + return (size_t)__builtin_ctz(x) >> 3; +#endif +} + +/* The index of the first byte where the n-byte spans a and b differ, or n + * when they are equal. Spans of at least 8 bytes compare words up to byte + * 32, then 32-byte blocks with SSE2 or NEON where the target has them, + * then words; the last word overlaps bytes already compared. Shorter + * spans compare two overlapping 4-byte halves, or bytes. */ +static inline size_t scr_span_mismatch(const char *a, const char *b, size_t n) { + size_t i = 0; + uint64_t l, r; + if (n < sizeof(uint64_t)) { + if (n >= sizeof(uint32_t)) { + uint32_t l4, r4; + memcpy(&l4, a, 4); + memcpy(&r4, b, 4); + if (l4 != r4) return scr_word32_mismatch(l4 ^ r4); + memcpy(&l4, a + n - 4, 4); + memcpy(&r4, b + n - 4, 4); + return l4 != r4 ? n - 4 + scr_word32_mismatch(l4 ^ r4) : n; + } + while (i < n && a[i] == b[i]) i++; + return i; + } + memcpy(&l, a, 8); + memcpy(&r, b, 8); + if (l != r) return scr_word_mismatch(l ^ r); + size_t head = n < 32 ? n & ~(size_t)7 : 32; + for (i = sizeof(uint64_t); i < head; i += sizeof(uint64_t)) { + memcpy(&l, a + i, 8); + memcpy(&r, b + i, 8); + if (l != r) return i + scr_word_mismatch(l ^ r); + } +#if defined(__SSE2__) + for (; n - i >= 32; i += 32) { + __m128i e0 = _mm_cmpeq_epi8(_mm_loadu_si128((const __m128i *)(a + i)), + _mm_loadu_si128((const __m128i *)(b + i))); + __m128i e1 = _mm_cmpeq_epi8(_mm_loadu_si128((const __m128i *)(a + i + 16)), + _mm_loadu_si128((const __m128i *)(b + i + 16))); + uint32_t equal = (uint32_t)_mm_movemask_epi8(e0) | (uint32_t)_mm_movemask_epi8(e1) << 16; + if (equal != UINT32_MAX) return i + (size_t)__builtin_ctz(~equal); + } +#elif defined(__ARM_NEON) && defined(__aarch64__) + /* A differing block leaves the loop; the word loop below finds the byte. */ + for (; n - i >= 32; i += 32) { + uint8x16_t e0 = vceqq_u8(vld1q_u8((const uint8_t *)a + i), vld1q_u8((const uint8_t *)b + i)); + uint8x16_t e1 = + vceqq_u8(vld1q_u8((const uint8_t *)a + i + 16), vld1q_u8((const uint8_t *)b + i + 16)); + if (vminvq_u8(vandq_u8(e0, e1)) != 0xff) break; + } +#endif + for (; n - i >= sizeof(uint64_t); i += sizeof(uint64_t)) { + memcpy(&l, a + i, 8); + memcpy(&r, b + i, 8); + if (l != r) return i + scr_word_mismatch(l ^ r); + } + if (i == n) return n; + size_t j = n - sizeof(uint64_t); + memcpy(&l, a + j, 8); + memcpy(&r, b + j, 8); + return l != r ? j + scr_word_mismatch(l ^ r) : n; +} + /* UTF-16 ordering over canonical WTF-8. Equal bytes can be skipped even * inside a multibyte character: unequal continuation bytes preserve code * point order. Differing lead bytes for U+E000..U+FFFF versus a * supplementary character reverse byte order, because the latter starts * with a UTF-16 high surrogate below U+E000. A lone surrogate against a * supplementary character compares their units. */ -int scr_str_cmp_u16(ScrStr *a, ScrStr *b) { - if (a == b) return 0; +__attribute__((noinline)) int scr_str_cmp_u16_bytes(const ScrStr *a, const ScrStr *b) { size_t common = a->len < b->len ? a->len : b->len; - size_t i = 0; - while (common - i >= sizeof(uint64_t)) { - uint64_t left, right; - memcpy(&left, a->data + i, sizeof(left)); - memcpy(&right, b->data + i, sizeof(right)); - if (left != right) break; - i += sizeof(left); - } - while (i < common && a->data[i] == b->data[i]) i++; + size_t i = scr_span_mismatch(a->data, b->data, common); if (i == common) return (a->len > b->len) - (a->len < b->len); unsigned char left = (unsigned char)a->data[i]; unsigned char right = (unsigned char)b->data[i]; @@ -866,6 +940,16 @@ int scr_str_cmp_u16(ScrStr *a, ScrStr *b) { return left < right ? -1 : 1; } +/* Byte order differs from UTF-16 order only where both differing bytes are + * at least 0xED, which a proven-ASCII string never has: such pairs compare + * with memcmp. The scan stays out of line in a global function, so program + * builds can import this body. */ +int scr_str_cmp_u16(ScrStr *a, ScrStr *b) { + if (a == b) return 0; + if (scr_str_known_ascii(a) || scr_str_known_ascii(b)) return scr_str_cmp(a, b); + return scr_str_cmp_u16_bytes(a, b); +} + /* ── interned strings ───────────────────────────────────────────────── * The empty string and every single-character ASCII string are immortal * statics (same layout the emitter uses for literals): charAt/slice churn diff --git a/packages/runtime/src/scr_string.h b/packages/runtime/src/scr_string.h index 425dfaabf..fc2f15562 100644 --- a/packages/runtime/src/scr_string.h +++ b/packages/runtime/src/scr_string.h @@ -290,16 +290,19 @@ ScrStr *scr_str_concat_f64(ScrStr *a, double x); bool scr_str_eq(ScrStr *a, ScrStr *b); -/* memcmp byte order == code-point order (see SEMANTICS.md: diverges from - * JS UTF-16 code-unit order only for non-BMP vs U+E000..U+FFFF). Returns - * <0, 0, >0. */ +/* memcmp byte order == code-point order, which differs from JS UTF-16 + * code-unit order for non-BMP characters against U+E000..U+FFFF and for + * lone surrogates. Returns <0, 0, >0. Relational string operators use + * scr_str_cmp_u16. */ int scr_str_cmp(ScrStr *a, ScrStr *b); /* a.localeCompare(b) with no locales/options: Node's root-locale order for * Latin-script text, code-point order outside it (scr_string.c). */ double scr_str_locale_compare(ScrStr *a, ScrStr *b); -/* ECMAScript string-list ordering: compare UTF-16 code units even though - * ScrStr stores WTF-8. Returns <0, 0, >0. */ +/* ECMAScript string ordering: compare UTF-16 code units even though ScrStr + * stores WTF-8. Returns <0, 0, >0. */ int scr_str_cmp_u16(ScrStr *a, ScrStr *b); +/* scr_str_cmp_u16 for two distinct strings that are not proven ASCII. */ +int scr_str_cmp_u16_bytes(const ScrStr *a, const ScrStr *b); #endif /* SCR_STRING_H */ diff --git a/tests/corpus/1536-string-array-sweep.ts b/tests/corpus/1536-string-array-sweep.ts index 0e5c12ebc..e3d5259ff 100644 --- a/tests/corpus/1536-string-array-sweep.ts +++ b/tests/corpus/1536-string-array-sweep.ts @@ -1,8 +1,7 @@ // The surface-tail sweep trio: string.substring (clamp-and-swap, the // port-suffix idiom), array findIndex (first match or -1, index-aware // callbacks), and the comparator-less sort() on string[] (JS's default -// UTF-16 ordering — a dedicated comparator because relational operators use -// scriptc's documented code-point order). +// UTF-16 code-unit ordering). const localAddr = "127.0.0.1.51823"; const lastColon = localAddr.lastIndexOf("."); console.log(localAddr.substring(lastColon + 1)); diff --git a/tests/corpus/6023-string-compare-utf16.ts b/tests/corpus/6023-string-compare-utf16.ts new file mode 100644 index 000000000..c4b9ccf91 --- /dev/null +++ b/tests/corpus/6023-string-compare-utf16.ts @@ -0,0 +1,72 @@ +// Relational string comparisons order strings by UTF-16 code units, as in +// Node: characters above U+FFFF (surrogate pairs) sort before U+E000..U+FFFF, +// lone surrogates sort by their own code unit, and ASCII and other BMP text +// keep their ordinary order. Typed comparisons, sort comparators and +// comparisons of untyped values agree. +const astral = "😀"; // U+1F600: D83D DE00 +const privateUse = "\ue000"; +const last = "\uffff"; +const highSurrogate = "\ud83d"; +const lowSurrogate = "\ude00"; +const hex = (s: string): string => { + const units: string[] = []; + for (let i = 0; i < s.length; i++) units.push(s.charCodeAt(i).toString(16)); + return units.join(" "); +}; + +const pairs: [string, string][] = [ + [astral, last], + [astral, privateUse], + ["\ud7ff", astral], + [highSurrogate, astral], + [lowSurrogate, astral], + [lowSurrogate, last], + ["é", "e"], + ["abc😀", "abc\uffff"], + ["same", "same"], + ["prefix", "prefix😀"], +]; +for (const [a, b] of pairs) + console.log(hex(a), "|", hex(b), a < b, a <= b, a > b, a >= b); + +const words = ["\uffff", "😀", "é", "e", "\ue000", "z", "\ud83d", "A"]; +const byOperator = [...words].sort((x, y) => (x < y ? -1 : x > y ? 1 : 0)); +const byDefault = [...words].sort(); +console.log(byOperator.map(hex).join(", ")); +console.log(byDefault.map(hex).join(", ")); +console.log(byOperator.join("") === byDefault.join("")); + +const untyped: unknown[] = JSON.parse('["\\uffff", "\\ud83d\\ude00", "a"]'); +const [u0, u1] = untyped as string[]; +console.log(u1! < u0!, [...untyped].sort().map((v) => hex(String(v))).join(", ")); + +let smallest = words[0]!; +for (const word of words) if (word < smallest) smallest = word; +console.log(hex(smallest), words.filter((w) => w > "\ud7ff").length); + +// Every pair of these characters after shared prefixes of 0 to 24 bytes, +// ASCII and multibyte, with and without a suffix: the first differing byte +// falls in the first word, the tail word, the first byte or past 16 bytes. +const chars = ["a", "z", "é", "한", "中", "\ud7ff", "\ue000", "\uffff"]; +chars.push(astral, highSurrogate, lowSurrogate, "\u{10ffff}"); +const hashOf = (text: string): number => { + let h = 0; + for (let i = 0; i < text.length; i++) h = (Math.imul(h, 31) + text.charCodeAt(i)) | 0; + return h >>> 0; +}; +for (const unit of ["p", "é", "中"]) { + const rows: string[] = []; + for (let n = 0; n <= 24; n += 3) { + const prefix = unit.repeat(n); + let marks = ""; + for (const x of chars) + for (const y of chars) + for (const suffix of ["", "q", "tail-of-text"]) { + const a = prefix + x + suffix; + const b = prefix + y; + marks += a < b ? "<" : a > b ? ">" : "="; + } + rows.push(`${n}:${marks.split("<").length - 1}:${hashOf(marks)}`); + } + console.log(hex(unit), rows.join(" ")); +} diff --git a/tests/harness/corpus-costs/6023-string-compare-utf16.ts.json b/tests/harness/corpus-costs/6023-string-compare-utf16.ts.json new file mode 100644 index 000000000..2d2f07e08 --- /dev/null +++ b/tests/harness/corpus-costs/6023-string-compare-utf16.ts.json @@ -0,0 +1 @@ +{ "plain": 2700, "san": 5400 } diff --git a/tests/harness/native-frontend.ts b/tests/harness/native-frontend.ts index bb76631a8..288d15481 100644 --- a/tests/harness/native-frontend.ts +++ b/tests/harness/native-frontend.ts @@ -68,6 +68,7 @@ export const programs = [ "6020-class-extends-function-constructor.js", // Re-chosen key orders rebuild the inspect helpers after lowering. "key-order-late-optional.ts", + "5888-inspect-sorted.ts", ]; interface LowerRequest {