From 302e4da3ba4c81def652561611f9742aba67e1ea Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 8 Oct 2026 23:32:23 -0700 Subject: [PATCH 1/3] Cache proven ASCII in the string header and read it inline JavaScript string reads use UTF-16 indices over scriptc's UTF-8 storage, so charCodeAt, length, slice, substring and indexOf looked every receiver up in a small global index cache (scr_sidx) on each call, and a fresh short receiver evicted an entry and rescanned its bytes. In the tsc-ts self-check that lookup and its mapping were the largest string cost. Bit 31 of the capacity word now records that a string is proven all ASCII, where units are bytes. A clear bit means "not known", so the fact only needs to be exact where it is set: scr_str_new and its users prove it while copying, concatenations combine their operands' bits, number text, ASCII slices and repeat inherit it, emitted literals and runtime statics carry it in their capacity word, and an index-cache scan that proves ASCII records it. scr_str_hash_forget clears it with the cached key hash, so every in-place mutation drops it; an in-place append that keeps the string ASCII sets it again. Capacity stops at 2^31 - 1 bytes, still above the UTF-8 size of any string V8 can represent. Proven-ASCII receivers skip the index cache in every UTF-16 mapper. Sites for length and charCodeAt call small internal helpers (string-reads.ts) that LLVM inlines: a header load and, for an in-range index, one byte load; everything else defers to the runtime with unchanged semantics. For non-ASCII receivers the cache remembers its last hit (validated by owner and tier), charCodeAt answers sequential scans from the cached ASCII span before any other work, and appends that keep a string ASCII no longer walk the cache. --- packages/compiler/src/backend/llvm/emitter.ts | 16 ++- .../src/backend/llvm/expr-containers.ts | 24 ++-- .../src/backend/llvm/expr-primitives.test.ts | 4 +- .../src/backend/llvm/string-key-hash.test.ts | 18 ++- .../src/backend/llvm/string-key-hash.ts | 24 +++- .../src/backend/llvm/string-lifetimes.test.ts | 12 +- .../compiler/src/backend/llvm/string-reads.ts | 84 +++++++++++ .../src/backend/llvm/switch-dispatch.test.ts | 2 +- .../test/ts7/baselines/order-parity.json | 12 ++ packages/runtime/src/scr_regex.c | 1 + packages/runtime/src/scr_runtime.h | 66 +++++++-- packages/runtime/src/scr_string.c | 135 ++++++++++++++---- tests/corpus/4476-string-ascii-header-fact.ts | 106 ++++++++++++++ tests/corpus/4478-string-large-index-reads.ts | 75 ++++++++++ 14 files changed, 516 insertions(+), 63 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/string-reads.ts create mode 100644 tests/corpus/4476-string-ascii-header-fact.ts create mode 100644 tests/corpus/4478-string-large-index-reads.ts diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 3adb750ed..fdc855624 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -289,8 +289,9 @@ import { LlvmUnsupportedError } from "./unsupported.js"; import { LlWalkers } from "./walkers.js"; import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js"; import { emitNullablePresent, emitUnionPeek, emitUnionTag } from "./union-repr.js"; -import { literalCapWord } from "./string-key-hash.js"; +import { literalCapWord, literalCapWord32 } from "./string-key-hash.js"; import { NUMBER_MAP_ENTRY_DECL } from "./map-number-lookup.js"; +import { STRING_READ_HELPERS } from "./string-reads.js"; import { emitConcatInputs, emitMixedConcat, @@ -1770,8 +1771,11 @@ export class LlEmitter { // the runtime ABI’s static table, retain/release skip rc == SIZE_MAX. // 64-bit (little-endian) targets split the capacity word: the high // half carries the precomputed Map key hash (SCR_STR_HASH_CACHE). + // Bit 31 marks all-ASCII literals (SCR_STR_ASCII_BIT) on every target. const capWord = - this.sizeType === "i64" ? literalCapWord(Buffer.from(text, "utf8")) : String(lit.len); + this.sizeType === "i64" + ? literalCapWord(Buffer.from(text, "utf8")) + : literalCapWord32(Buffer.from(text, "utf8")); out.push( `@${lit.sym} = internal global { ${this.sizeType}, ${this.sizeType}, ${this.sizeType}, [${lit.len + 1} x i8] } ` + `{ ${this.sizeType} -1, ${this.sizeType} ${lit.len}, ${this.sizeType} ${capWord}, [${lit.len + 1} x i8] c"${llStrBytes(text)}" }`, @@ -2643,6 +2647,14 @@ export class LlEmitter { ``, ); } + // String length and charCodeAt read proven-ASCII strings inline: bit 31 + // of the capacity word (SCR_STR_ASCII_BIT, the low half's sign bit on + // little-endian targets) makes UTF-16 units bytes. Unproven strings and + // anything but an in-range index take the runtime's UTF-16 mapping. + for (const helper of STRING_READ_HELPERS) { + if (!this.decls.has(helper.decl)) continue; + defs.push(...helper.define(this.sizeType, FN_ATTRS), ``); + } // Ordering also decides inline when the first bytes differ: byte order // is the plain comparison's order, and the UTF-16 order whenever both // bytes are ASCII. Callers only test the sign of the result. diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 0957c992f..8a58fe6fb 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -30,6 +30,7 @@ import { } from "./map-number-lookup.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; import { emitStringSliceRead } from "./string-slices.js"; +import { STRING_CHAR_CODE_AT_DECL, STRING_UTF16_LEN_DECL } from "./string-reads.js"; import { emitDenseArrayNumber, emitDenseArrayPush } from "./dense-array-access.js"; import { emitUnionWiden } from "./expr-records.js"; @@ -189,16 +190,19 @@ export function emitStrIntrinsic( }; const method = e.method; switch (method) { - case "length": - return call("scr_str_utf16_len", "double (ptr)", `ptr ${r.name}`, "double", false); - case "charCodeAt": - return call( - "scr_str_char_code_at", - "double (ptr, double)", - `ptr ${r.name}, double ${args[0]!.name}`, - "double", - false, - ); + case "length": { + // sc_str_utf16_len (string-reads.ts) answers proven-ASCII strings inline. + host.declare(STRING_UTF16_LEN_DECL); + const t = B.tmp(); + B.line(`${t} = call double @sc_str_utf16_len(ptr ${r.name})`); + return { name: t, type: e.type }; + } + case "charCodeAt": { + host.declare(STRING_CHAR_CODE_AT_DECL); + const t = B.tmp(); + B.line(`${t} = call double @sc_str_char_code_at(ptr ${r.name}, double ${args[0]!.name})`); + return { name: t, type: e.type }; + } case "charAt": return call( "scr_str_char_at", diff --git a/packages/compiler/src/backend/llvm/expr-primitives.test.ts b/packages/compiler/src/backend/llvm/expr-primitives.test.ts index 3c773f239..8e929f851 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.test.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.test.ts @@ -80,8 +80,8 @@ test.each([32, 64] as const)( const zero = /^define internal [^\n]*@sc_[bf]+_length_0\([^]*?^}/m.exec(llvm)![0]; const one = /^define internal [^\n]*@sc_[bf]+_length_1\([^]*?^}/m.exec(llvm)![0]; expect(zero).toContain(`getelementptr inbounds i${bits}`); - expect(zero).not.toContain("@scr_str_utf16_len"); - expect(one).toContain("@scr_str_utf16_len"); + expect(zero).not.toContain("_str_utf16_len"); + expect(one).toContain("@sc_str_utf16_len"); }, ); diff --git a/packages/compiler/src/backend/llvm/string-key-hash.test.ts b/packages/compiler/src/backend/llvm/string-key-hash.test.ts index 1fab9278b..5ca347a61 100644 --- a/packages/compiler/src/backend/llvm/string-key-hash.test.ts +++ b/packages/compiler/src/backend/llvm/string-key-hash.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { literalCapWord, stringKeyHash32 } from "./string-key-hash.js"; +import { literalCapWord, literalCapWord32, stringKeyHash32 } from "./string-key-hash.js"; // Shared with packages/runtime/test/test_map.c (hash_vectors): the runtime // computes these for heap strings, the compiler bakes them into literals. @@ -26,12 +26,24 @@ describe("string key hash", () => { expect(stringKeyHash32(Buffer.from("ab"))).not.toBe(stringKeyHash32(Buffer.from("abc"))); }); - test("literal capacity word keeps cap = len in the low half", () => { + test("literal capacity word keeps cap = len in the low 31 bits", () => { for (const [text, hash] of VECTORS) { const bytes = Buffer.from(text, "utf8"); const word = BigInt.asUintN(64, BigInt(literalCapWord(bytes))); - expect(word & 0xffffffffn).toBe(BigInt(bytes.length)); + expect(word & 0x7fffffffn).toBe(BigInt(bytes.length)); expect(word >> 32n).toBe(BigInt(hash)); } }); + + test("literal capacity words carry the proven-ASCII bit exactly", () => { + for (const [text] of VECTORS) { + const bytes = Buffer.from(text, "utf8"); + const ascii = bytes.every((byte) => byte < 0x80); + const word = BigInt.asUintN(64, BigInt(literalCapWord(bytes))); + expect((word & 0x80000000n) !== 0n).toBe(ascii); + const word32 = Number(literalCapWord32(bytes)) >>> 0; + expect(word32 & 0x7fffffff).toBe(bytes.length); + expect((word32 & 0x80000000) !== 0).toBe(ascii); + } + }); }); diff --git a/packages/compiler/src/backend/llvm/string-key-hash.ts b/packages/compiler/src/backend/llvm/string-key-hash.ts index dc047185a..28afb643f 100644 --- a/packages/compiler/src/backend/llvm/string-key-hash.ts +++ b/packages/compiler/src/backend/llvm/string-key-hash.ts @@ -57,10 +57,28 @@ export function stringKeyHash32(bytes: Uint8Array): number { return folded === 0 ? 1 : folded; } +/** Bit 31 of every capacity word: the string is proven all ASCII, so the + * runtime and inline string reads map UTF-16 units to bytes by identity + * (SCR_STR_ASCII_BIT in scr_runtime.h). */ +export const STRING_ASCII_BIT = 0x80000000; + +function isAscii(bytes: Uint8Array): boolean { + for (const byte of bytes) if (byte >= 0x80) return false; + return true; +} + /** The capacity word of an immortal literal on 64-bit targets: cap = len in - * the low half, the precomputed key hash in the high half, printed as the - * signed i64 LLVM expects. */ + * bits 0-30, the proven-ASCII bit, the precomputed key hash in the high + * half, printed as the signed i64 LLVM expects. */ export function literalCapWord(bytes: Uint8Array): string { - const word = BigInt(bytes.length) | (BigInt(stringKeyHash32(bytes)) << 32n); + const ascii = isAscii(bytes) ? BigInt(STRING_ASCII_BIT) : 0n; + const word = BigInt(bytes.length) | ascii | (BigInt(stringKeyHash32(bytes)) << 32n); return BigInt.asIntN(64, word).toString(); } + +/** The capacity word of an immortal literal on 32-bit targets: cap = len + * plus the proven-ASCII bit, printed as the signed i32 LLVM expects. */ +export function literalCapWord32(bytes: Uint8Array): string { + const word = bytes.length + (isAscii(bytes) ? STRING_ASCII_BIT : 0); + return String(word | 0); +} diff --git a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts index 17ceb8fea..bf54116ee 100644 --- a/packages/compiler/src/backend/llvm/string-lifetimes.test.ts +++ b/packages/compiler/src/backend/llvm/string-lifetimes.test.ts @@ -277,9 +277,11 @@ const methodCases: { args: IrExpr[]; result: IrType; target: string; + /** Sites enter through the inlined sc_str_* wrapper (string-reads.ts). */ + inline?: boolean; }[] = [ - { method: "length", args: [], result: F64, target: "utf16_len" }, - { method: "charCodeAt", args: [num(1)], result: F64, target: "char_code_at" }, + { method: "length", args: [], result: F64, target: "utf16_len", inline: true }, + { method: "charCodeAt", args: [num(1)], result: F64, target: "char_code_at", inline: true }, { method: "charAt", args: [num(1)], result: STRING, target: "char_at" }, { method: "indexOf", args: [ref("argument"), num(1)], result: F64, target: "index_of" }, { method: "includes", args: [ref("argument")], result: BOOL, target: "includes" }, @@ -314,7 +316,7 @@ const methodCases: { test.each(methodCases)( "$method passes borrowed string inputs through its runtime ABI", - ({ method, args, result, target }) => { + ({ method, args, result, target, inline }) => { const f = fn( "method", ["receiver", "argument"], @@ -322,7 +324,7 @@ test.each(methodCases)( ); for (const bits of [32, 64] as const) { const ir = body(mod(f), "sc_bf_method", bits); - expect(ir).toContain(`@scr_str_${target}(`); + expect(ir).toContain(`@${inline ? "sc" : "scr"}_str_${target}(`); expect(ir).not.toContain("@scr_str_retain_v"); // Throwing methods release their owned result on the unwind path. if (method !== "normalize" && method !== "repeat") @@ -381,7 +383,7 @@ test("boxed string lengths borrow initialized payloads and keep checked TDZ read expect(ir).toMatch(/br i1 %t\d+, label %lazy\.release\d+/); expect(ir).toContain("@scr_box_release"); expect(ir).not.toContain("@scr_box_get_ref"); - expect(ir).toContain("@scr_str_utf16_len"); + expect(ir).toContain("@sc_str_utf16_len"); inspect.locals[0]!.tdz = true; const checked = body(mod(inspect), "sc_f_inspect"); expect(checked).toContain("@scr_box_get_ref"); diff --git a/packages/compiler/src/backend/llvm/string-reads.ts b/packages/compiler/src/backend/llvm/string-reads.ts new file mode 100644 index 000000000..fe70d8a1b --- /dev/null +++ b/packages/compiler/src/backend/llvm/string-reads.ts @@ -0,0 +1,84 @@ +/* Inline UTF-16 reads of proven-ASCII strings. + * + * Strings are UTF-8, so `length` and `charCodeAt` need a UTF-16 mapping in + * general. Bit 31 of a string's capacity word (SCR_STR_ASCII_BIT in + * scr_runtime.h) proves every byte is ASCII, where units are bytes. Sites + * call the internal sc_str_* helpers below, which LLVM inlines: the proven + * case is a header load and, for charCodeAt, one byte load; every other + * string or index defers to the runtime function, which keeps the complete + * semantics (surrogate halves, out-of-range NaN, fractional, negative and + * NaN indices). On little-endian targets the bit is the sign bit of the + * capacity word's low 32 bits on both i64 and i32 layouts. + */ + +export const STRING_UTF16_LEN_DECL = "declare double @scr_str_utf16_len(ptr)"; +export const STRING_CHAR_CODE_AT_DECL = "declare double @scr_str_char_code_at(ptr, double)"; + +/** Load the proven-ASCII flag of `%s` into `%ascii`. */ +function asciiTest(): string[] { + return [ + ` %cw.p = getelementptr inbounds %ScrStr, ptr %s, i32 0, i32 2`, + ` %cw = load i32, ptr %cw.p`, + ` %ascii = icmp slt i32 %cw, 0`, + ]; +} + +function loadLength(sz: string): string[] { + return [ + ` %len.p = getelementptr inbounds %ScrStr, ptr %s, i32 0, i32 1`, + ` %len = load ${sz}, ptr %len.p`, + ]; +} + +export interface StringReadHelper { + decl: string; + define(sz: string, attrs: string): string[]; +} + +export const STRING_READ_HELPERS: readonly StringReadHelper[] = [ + { + decl: STRING_UTF16_LEN_DECL, + define: (sz, attrs) => [ + `define internal double @sc_str_utf16_len(ptr %s) ${attrs} {`, + `entry:`, + ...asciiTest(), + ` br i1 %ascii, label %fast, label %slow`, + `fast:`, + ...loadLength(sz), + ` %n = uitofp ${sz} %len to double`, + ` ret double %n`, + `slow:`, + ` %r = call double @scr_str_utf16_len(ptr %s)`, + ` ret double %r`, + `}`, + ], + }, + { + decl: STRING_CHAR_CODE_AT_DECL, + define: (sz, attrs) => [ + `define internal double @sc_str_char_code_at(ptr %s, double %i) ${attrs} {`, + `entry:`, + ...asciiTest(), + ` br i1 %ascii, label %bounds, label %slow`, + `bounds:`, + ...loadLength(sz), + ` %lenf = uitofp ${sz} %len to double`, + // NaN fails both ordered comparisons; -0 passes and truncates to 0. + ` %lo = fcmp oge double %i, 0.0`, + ` %hi = fcmp olt double %i, %lenf`, + ` %in = and i1 %lo, %hi`, + ` br i1 %in, label %read, label %slow`, + `read:`, + ` %k = fptoui double %i to ${sz}`, + ` %data = getelementptr inbounds %ScrStr, ptr %s, i32 1`, + ` %p = getelementptr inbounds i8, ptr %data, ${sz} %k`, + ` %byte = load i8, ptr %p`, + ` %unit = uitofp i8 %byte to double`, + ` ret double %unit`, + `slow:`, + ` %r = call double @scr_str_char_code_at(ptr %s, double %i)`, + ` ret double %r`, + `}`, + ], + }, +]; diff --git a/packages/compiler/src/backend/llvm/switch-dispatch.test.ts b/packages/compiler/src/backend/llvm/switch-dispatch.test.ts index 614ec75ba..e49b9e36e 100644 --- a/packages/compiler/src/backend/llvm/switch-dispatch.test.ts +++ b/packages/compiler/src/backend/llvm/switch-dispatch.test.ts @@ -81,7 +81,7 @@ test("string dispatch uses target-width byte lengths and checks equality inside const body = emit(STRING, ["", "a", "Γ©", "πŸ˜€", "tail", "Γ©"].map(string), bits); expect(body).toContain(`load i${bits}, ptr`); expect(body).toContain(`switch i${bits}`); - expect(body).not.toContain("scr_str_utf16_len"); + expect(body).not.toContain("_str_utf16_len"); expect(body.match(/call zeroext i1 @sc_str_eq/g)).toHaveLength(5); const dispatch = body.split("\n").find((line) => line.includes(`switch i${bits}`))!; expect(dispatch).toContain(`i${bits} 2, label`); diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index da1c24f82..dd4d8373e 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -15460,6 +15460,18 @@ "/tests/corpus/4453-int32-closed-locals.ts" ], "diags": [] + }, + "/tests/corpus/4476-string-ascii-header-fact.ts": { + "order": [ + "/tests/corpus/4476-string-ascii-header-fact.ts" + ], + "diags": [] + }, + "/tests/corpus/4478-string-large-index-reads.ts": { + "order": [ + "/tests/corpus/4478-string-large-index-reads.ts" + ], + "diags": [] } } } diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 5a2c670cf..e86ed3046 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -416,6 +416,7 @@ typedef struct ScrRegexInput { } ScrRegexInput; static bool scr_text_ascii(const ScrStr *s) { + if (scr_str_known_ascii(s)) return true; size_t i = 0; for (; s->len - i >= sizeof(uint64_t); i += sizeof(uint64_t)) { uint64_t word; diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 448034e1d..58b98dcf8 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -649,40 +649,86 @@ typedef struct ScrVt { * owned heap strings are ever mutated, and every such path clears the hash * (scr_str_hash_forget); immortal strings are never written, so they always * hash on demand. + * + * Bit 31 of the capacity word (on every target) is the proven-ASCII fact: + * when set, every byte is below 0x80, so UTF-16 indices equal byte offsets + * and .length == len. A clear bit means "not known", never "non-ASCII": + * constructors set it only where it is cheap, and string reads that prove + * ASCII set it later. Capacity therefore stops at 2^31 - 1 bytes, which + * still holds every string V8 can represent (2^29 - 24 UTF-16 units, at + * most three UTF-8 bytes each). Generated code reads the bit for inline + * charCodeAt/length (llvm/strings.ts); emitted literals and runtime + * statics set it in their capacity word. scr_str_hash_forget clears it + * with the hash: every in-place mutation already forgets cached facts. */ #if SIZE_MAX > UINT32_MAX && defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define SCR_STR_HASH_CACHE 1 -#define SCR_STR_MAX_CAP ((size_t)UINT32_MAX) #else #define SCR_STR_HASH_CACHE 0 -#define SCR_STR_MAX_CAP (SIZE_MAX - sizeof(size_t) * 3 - 1) #endif +#define SCR_STR_MAX_CAP ((size_t)0x7fffffff) typedef struct ScrStr { size_t rc; size_t len; #if SCR_STR_HASH_CACHE - uint32_t cap; + uint32_t cap : 31; + uint32_t ascii : 1; uint32_t hash; #else - size_t cap; + size_t cap : 31; + size_t ascii : 1; #endif char data[]; } ScrStr; -/* The capacity word of a static string: cap == len, plus the precomputed - * 32-bit key hash where the header caches one (0 leaves it to be computed). */ +/* The capacity word of a static string: cap == len, the proven-ASCII bit, + * plus the precomputed 32-bit key hash where the header caches one (0 + * leaves it to be computed). */ +#define SCR_STR_ASCII_BIT ((size_t)1 << 31) #if SCR_STR_HASH_CACHE -#define SCR_STR_CAP_WORD(len, hash32) ((size_t)(len) | ((size_t)(uint32_t)(hash32) << 32)) +#define SCR_STR_CAP_WORD(len, hash32, ascii) \ + ((size_t)(len) | ((ascii) ? SCR_STR_ASCII_BIT : 0) | \ + ((size_t)(uint32_t)(hash32) << 32)) #else -#define SCR_STR_CAP_WORD(len, hash32) ((size_t)(len)) +#define SCR_STR_CAP_WORD(len, hash32, ascii) ((size_t)(len) | ((ascii) ? SCR_STR_ASCII_BIT : 0)) #endif +/* Forget every cached content fact (key hash, proven ASCII) before a + * uniquely owned string's bytes change. */ static inline void scr_str_hash_forget(ScrStr *s) { #if SCR_STR_HASH_CACHE s->hash = 0; -#else - (void)s; #endif + s->ascii = 0; +} + +/* True only when every byte is proven below 0x80 (see above). */ +static inline bool scr_str_known_ascii(const ScrStr *s) { return s->ascii; } + +/* Record a proven-ASCII fact. Immortal strings may live in read-only + * memory and are shared across threads, so they are never written. */ +static inline void scr_str_note_ascii(ScrStr *s) { + if (s->rc != SIZE_MAX) s->ascii = 1; +} + +/* Whether a byte span is all ASCII: 32 bytes per step, stopping at the + * first chunk with a high bit. */ +static inline bool scr_bytes_all_ascii(const char *p, size_t n) { + const uint64_t high = UINT64_C(0x8080808080808080); + size_t i = 0; + for (; n - i >= 32; i += 32) { + uint64_t w[4]; + memcpy(w, p + i, sizeof w); + if ((w[0] | w[1] | w[2] | w[3]) & high) return false; + } + uint64_t acc = 0; + for (; n - i >= 8; i += 8) { + uint64_t w; + memcpy(&w, p + i, 8); + acc |= w; + } + for (; i < n; i++) acc |= (unsigned char)p[i]; + return (acc & high) == 0; } ScrStr *scr_str_new(const char *bytes, size_t len); /* returns +1 */ diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 92d5e4d74..09a6e5f10 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -200,7 +200,9 @@ static ScrSidx *scr_sidx_claim(ScrSidx *table, unsigned *clock, unsigned *live) * receivers still shed their point buffer after proving identity mapping. * Both tiers remain fixed-size and allocation-free until a non-ASCII sparse * receiver actually needs checkpoints. */ -static ScrSidx *scr_sidx(const ScrStr *s) { +static SCR_TL ScrSidx *scr_sidx_last; /* most recent hit; a slot, never freed */ + +static ScrSidx *scr_sidx_lookup(const ScrStr *s) { if (s->len >= SCR_SIDX_MIN_BYTES) { for (int i = 0; i < SCR_SIDX_N; i++) { if (scr_sidx_sparse_tab[i].s == s) return &scr_sidx_sparse_tab[i]; @@ -234,6 +236,27 @@ static ScrSidx *scr_sidx(const ScrStr *s) { return e; } +/* Sequential reads of one receiver (a scan over a source text) hit the + * same entry call after call. The memo is a pointer to a static slot, so a + * cleared, moved or reassigned entry simply fails the owner test. */ +static ScrSidx *scr_sidx(const ScrStr *s) { + ScrSidx *last = scr_sidx_last; + /* An in-place append can carry a cursor-tier receiver across the sparse + * threshold; only the lookup performs that transfer, so the memo must + * also match the receiver's current tier. */ + bool sparse = (uintptr_t)last - (uintptr_t)scr_sidx_sparse_tab < sizeof scr_sidx_sparse_tab; + if (last && last->s == s && sparse == (s->len >= SCR_SIDX_MIN_BYTES)) return last; + return scr_sidx_last = scr_sidx_lookup(s); +} + +/* The index entry for a receiver that is not proven ASCII, or NULL when it + * is: proven-ASCII strings map units to bytes by identity, and every mapper + * below checks the header bit before touching the entry. The bit only ever + * turns on during a read, so "ascii or entry" holds for the whole call. */ +static ScrSidx *scr_sidx_for(const ScrStr *s) { + return s->ascii ? NULL : scr_sidx(s); +} + /* In-place concat changes only the suffix. Keep every exact prefix anchor * (including the former end, which is now an ordinary boundary), but remove * the sole fact that described the old complete string. A threshold-crossing @@ -272,6 +295,19 @@ ScrStr *scr_str_new(const char *bytes, size_t len) { ScrStr *s = scr_str_alloc(len, len); memcpy(s->data, bytes, len); s->data[len] = '\0'; + /* The copy already streams the bytes; proving ASCII here lets every later + * UTF-16 index, length and slice skip the index cache. */ + if (scr_bytes_all_ascii(bytes, len)) s->ascii = 1; + return s; +} + +/* scr_str_new for bytes already known to be ASCII (slices of an ASCII + * string): no scan. */ +static ScrStr *scr_str_new_ascii(const char *bytes, size_t len) { + ScrStr *s = scr_str_alloc(len, len); + memcpy(s->data, bytes, len); + s->data[len] = '\0'; + s->ascii = 1; return s; } @@ -354,6 +390,7 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) { * concat chains (`a + b + c`, template literals), where each intermediate * result reaches the next concat as a sole-reference temp. Any string * with rc > 1 might be aliased and is copied, never mutated. */ + bool ascii = a->ascii && b->ascii; if (a->rc == 1 && a != b && a->cap >= newlen) { size_t oldlen = a->len; scr_short_forget(a); @@ -361,13 +398,17 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) { memcpy(a->data + a->len, b->data, b->len); a->len = newlen; a->data[newlen] = '\0'; + a->ascii = ascii; /* A cached UTF-16 length for a is stale now; checkpoints and the exact * old prefix remain valid. Its old terminal point is no longer an END * fact (u16len is invalidated below), but remains an excellent ordinary * checkpoint for accesses around the append boundary. The next mapper * lazily continues from oldlen rather than scanning the unchanged prefix - * again. */ - scr_sidx_concat_append(a, oldlen); + * again. An ASCII string that stays ASCII needs none of this: every + * mapper answers it by identity before consulting an entry, and the + * append that first adds non-ASCII bytes invalidates against its own + * old length, which bounds every older fact as well. */ + if (!ascii) scr_sidx_concat_append(a, oldlen); a->rc = 2; /* +1 for the returned reference, beside the caller's borrow */ return a; } @@ -389,6 +430,7 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) { memcpy(s->data, a->data, a->len); memcpy(s->data + a->len, b->data, b->len); s->data[newlen] = '\0'; + s->ascii = ascii; return s; } @@ -406,10 +448,12 @@ ScrStr *scr_str_concat_mixed(ScrStr *head, ScrStr *const *parts, const double *n const size_t limit = SCR_STR_MAX_CAP; size_t extra = 0; bool aliased = false; + bool ascii = !head || head->ascii; /* formatted numbers are ASCII */ for (size_t i = 0; i < count; i++) { if (parts[i]) { lens[i] = parts[i]->len; aliased |= parts[i] == head; + ascii &= parts[i]->ascii; } else { lens[i] = scr_f64_to_str(nums[i], text[i]); } @@ -448,8 +492,9 @@ ScrStr *scr_str_concat_mixed(ScrStr *head, ScrStr *const *parts, const double *n } out->len = newlen; out->data[newlen] = '\0'; + out->ascii = ascii; if (out == head) { - scr_sidx_concat_append(head, base); + if (!ascii) scr_sidx_concat_append(head, base); /* see scr_str_concat */ head->rc = 2; /* +1 for the returned reference, beside the caller's borrow */ } return out; @@ -477,12 +522,15 @@ ScrStr *scr_str_concat_parts(ScrStr *const *parts, size_t count) { if (nonempty == 1) return scr_str_retain(only); ScrStr *out = scr_str_alloc_raw(len, len); size_t offset = 0; + bool ascii = true; for (size_t i = 0; i < count; i++) { const ScrStr *part = parts[i]; memcpy(out->data + offset, part->data, part->len); offset += part->len; + ascii &= part->ascii; } out->data[len] = '\0'; + out->ascii = ascii; return out; } @@ -529,7 +577,7 @@ int scr_str_cmp_u16(ScrStr *a, ScrStr *b) { */ typedef struct { size_t rc; size_t len; size_t cap; char data[2]; } ScrChar1; #define SCR_A_WORD(c) (((uint64_t)(c) << 16) | ((uint64_t)(c) << 8) | (uint64_t)(c)) -#define SCR_A(c) {SIZE_MAX, 1, SCR_STR_CAP_WORD(1, SCR_KEY_HASH32_SHORT(1, SCR_A_WORD(c))), {(char)(c), 0}} +#define SCR_A(c) {SIZE_MAX, 1, SCR_STR_CAP_WORD(1, SCR_KEY_HASH32_SHORT(1, SCR_A_WORD(c)), 1), {(char)(c), 0}} #define SCR_A8(c) \ SCR_A(c), SCR_A(c + 1), SCR_A(c + 2), SCR_A(c + 3), \ SCR_A(c + 4), SCR_A(c + 5), SCR_A(c + 6), SCR_A(c + 7) @@ -540,7 +588,7 @@ static const ScrChar1 scr_ascii1[128] = { SCR_A8(96), SCR_A8(104), SCR_A8(112), SCR_A8(120), }; static const struct { size_t rc; size_t len; size_t cap; char data[1]; } - scr_lit_empty = {SIZE_MAX, 0, SCR_STR_CAP_WORD(0, SCR_KEY_HASH32_SHORT(0, 0)), ""}; + scr_lit_empty = {SIZE_MAX, 0, SCR_STR_CAP_WORD(0, SCR_KEY_HASH32_SHORT(0, 0), 1), ""}; static ScrStr *scr_str_empty(void) { return (ScrStr *)&scr_lit_empty; } @@ -562,6 +610,13 @@ static ScrStr *scr_str_from_span(const char *bytes, size_t len) { return scr_str_new(bytes, len); } +/* scr_str_from_span for a span of a proven-ASCII string: the result is + * ASCII without another scan. */ +static ScrStr *scr_str_from_span_ascii(const char *bytes, size_t len) { + if (len <= 4) return scr_str_from_span(bytes, len); + return scr_str_new_ascii(bytes, len); +} + /* A scalar fromCharCode needs neither the variadic argument array nor an * encoding buffer. Reuse the same empty/ASCII/tiny-span storage policy as * character indexing. A lone surrogate keeps the existing U+FFFD policy. */ @@ -794,6 +849,7 @@ static void scr_sidx_finish(const ScrStr *s, ScrSidx *e) { if (e->indexed_cb != s->len) return; e->u16len = e->indexed_cu; if (e->u16len == s->len) { /* proven all ASCII: identity needs no index */ + scr_str_note_ascii((ScrStr *)s); free(e->points); e->points = NULL; e->npoints = 0; @@ -837,6 +893,7 @@ static void scr_sidx_materialize_identity_prefix(const ScrStr *s, ScrSidx *e) { * historical single word-wise scan. `u16len == byte len` proves ASCII and * restores identity mapping with zero retained checkpoint memory. */ static size_t scr_sidx_len(const ScrStr *s, ScrSidx *e) { + if (s->ascii) return s->len; if (e->u16len != SCR_U16_UNKNOWN) return e->u16len; while (e->indexed_cb < s->len) scr_sidx_extend_one(s, e); scr_sidx_finish(s, e); @@ -970,7 +1027,7 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) { * Same contract as a from-scratch scan. */ static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, bool *mid) { - if (e->u16len == s->len) { /* all ASCII: identity mapping */ + if (s->ascii || e->u16len == s->len) { /* all ASCII: identity mapping */ *mid = false; return u16 < s->len ? u16 : s->len; } @@ -1037,7 +1094,7 @@ static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, * the closest sparse anchor or hot cursor. */ static size_t scr_byte_to_u16_c(const ScrStr *s, ScrSidx *e, size_t byte_off) { - if (e->u16len == s->len) return byte_off; /* all ASCII */ + if (s->ascii || e->u16len == s->len) return byte_off; /* all ASCII */ if (byte_off >= e->ascii_cb && byte_off < e->ascii_end) { e->cu = e->ascii_cu + (byte_off - e->ascii_cb); e->cb = byte_off; @@ -1098,7 +1155,7 @@ static const char *scr_byte_find(const char *hay, size_t hay_len, /* Search backwards from a clamped UTF-16 position. A low-surrogate position * maps to its scalar's first byte, so that scalar remains searchable. */ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); size_t len16 = scr_sidx_len(s, e); size_t start16 = isnan(position) || position >= (double)len16 ? len16 : position <= 0 ? 0 : (size_t)trunc(position); @@ -1121,16 +1178,32 @@ double scr_str_last_index_of(ScrStr *s, ScrStr *needle) { } double scr_str_utf16_len(ScrStr *s) { + if (s->ascii) return (double)s->len; return (double)scr_sidx_len(s, scr_sidx(s)); } double scr_str_char_code_at(ScrStr *s, double i) { + /* A sequential scan of a non-ASCII receiver stays inside the bounded + * ASCII span cached at its last resolved position. Its unit offset is an + * integer, so trunc(i) - ascii_cu == trunc(i - ascii_cu) for i at or past + * it; NaN and anything before the span fail the comparison. */ + ScrSidx *last = scr_sidx_last; + if (last && last->s == s && !s->ascii) { + double rel = i - (double)last->ascii_cu; + if (rel >= 0 && rel < (double)(last->ascii_end - last->ascii_cb)) { + size_t k = (size_t)rel; + last->cu = last->ascii_cu + k; + last->cb = last->ascii_cb + k; + return (double)(unsigned char)s->data[last->cb]; + } + } double idx = scr_to_integer_or_infinity(i); if (!(idx >= 0)) return NAN; /* negative or -Infinity */ /* UTF-16 length <= byte length always, so this also fences the cast. */ if (idx >= (double)s->len) return NAN; + if (s->ascii) return (double)(unsigned char)s->data[(size_t)idx]; ScrSidx *e = scr_sidx(s); - if (e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx]; + if (s->ascii || e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx]; bool mid; size_t off = scr_u16_to_byte_c(s, e, (size_t)idx, &mid); if (off >= s->len) return NAN; /* idx >= length */ @@ -1144,7 +1217,7 @@ double scr_str_char_code_at(ScrStr *s, double i) { double scr_str_index_of(ScrStr *s, ScrStr *needle, double fromIndex) { double pos = scr_to_integer_or_infinity(fromIndex); if (needle->len > s->len) return -1.0; - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); /* Per spec, the empty needle is found at the clamped fromIndex itself β€” * even when that index is between the halves of an astral pair. */ if (needle->len == 0) { @@ -1173,7 +1246,7 @@ static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e, /* Normalize once, then share slice's extraction without another cache * lookup, length query or floating-point boundary conversion. */ ScrStr *scr_str_substring(ScrStr *s, double start, double end) { - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); double len16 = (double)scr_sidx_len(s, e); double a = scr_to_integer_or_infinity(start); double b = scr_to_integer_or_infinity(end); @@ -1208,7 +1281,7 @@ bool scr_str_starts_with_from(ScrStr *s, ScrStr *needle, double position) { double pos = scr_to_integer_or_infinity(position); if (pos <= 0) return scr_str_starts_with(s, needle); if (pos >= (double)s->len || needle->len > s->len) return false; - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); bool mid; size_t start_byte = scr_u16_to_byte_c(s, e, (size_t)pos, &mid); return !mid && needle->len <= s->len - start_byte && @@ -1221,8 +1294,8 @@ bool scr_str_ends_with_from(ScrStr *s, ScrStr *needle, double end_position) { if (end_position >= (double)s->len) return scr_str_ends_with(s, needle); /* A second receiver lookup can evict the first receiver's entry. Resolve * the needle before borrowing the haystack entry for both conversions. */ - size_t needle16 = scr_sidx_len(needle, scr_sidx(needle)); - ScrSidx *e = scr_sidx(s); + size_t needle16 = scr_sidx_len(needle, scr_sidx_for(needle)); + ScrSidx *e = scr_sidx_for(s); size_t len16 = scr_sidx_len(s, e); size_t end16 = scr_str_clamp_u16_position(end_position, len16); if (needle16 > end16) return false; @@ -1245,7 +1318,7 @@ static size_t scr_slice_boundary(double v, size_t len16) { } ScrStr *scr_str_slice(ScrStr *s, double start, double end) { - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); size_t len16 = scr_sidx_len(s, e); size_t from = scr_slice_boundary(scr_to_integer_or_infinity(start), len16); size_t to = scr_slice_boundary(scr_to_integer_or_infinity(end), len16); @@ -1254,7 +1327,7 @@ ScrStr *scr_str_slice(ScrStr *s, double start, double end) { void scr_str_slice_range(ScrStr *s, double start, double end, bool substring, ScrStringSlice *out) { - ScrSidx *e = scr_sidx(s); + ScrSidx *e = scr_sidx_for(s); size_t length = scr_sidx_len(s, e); double a = scr_to_integer_or_infinity(start), b = scr_to_integer_or_infinity(end); size_t from, to; @@ -1288,6 +1361,11 @@ double scr_str_slice_char_code_at(ScrStr *s, const ScrStringSlice *range, double static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e, size_t from, size_t to) { if (from >= to) return scr_str_empty(); + if (s->ascii) { + /* Identity mapping; callers clamp both bounds to the length. */ + if (from == 0 && to == s->len) return scr_str_retain(s); + return scr_str_from_span_ascii(s->data + from, to - from); + } if (from == 0 && to == e->u16len) return scr_str_retain(s); bool from_mid, to_mid; @@ -1303,7 +1381,8 @@ static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e, /* Unit boundaries already prove the exact output length. Preserve it * instead of decoding a potentially megabyte-sized copy on .length. * Index checkpoints remain lazy and owned by the result's cache slot. */ - if (content_len >= SCR_SIDX_MIN_BYTES) scr_sidx(result)->u16len = to - from; + if (content_len >= SCR_SIDX_MIN_BYTES && !result->ascii) + scr_sidx(result)->u16len = to - from; return result; } @@ -1350,6 +1429,7 @@ ScrStr *scr_str_repeat(ScrStr *s, double count) { filled += chunk; } r->data[total] = '\0'; + r->ascii = s->ascii; return r; } @@ -1388,8 +1468,9 @@ ScrStr *scr_str_char_at(ScrStr *s, double i) { double idx = scr_to_integer_or_infinity(i); if (!(idx >= 0)) return scr_str_empty(); if (idx >= (double)s->len) return scr_str_empty(); /* len16 <= len */ + if (s->ascii) return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]]; ScrSidx *e = scr_sidx(s); - if (e->u16len == s->len) { /* all ASCII */ + if (s->ascii || e->u16len == s->len) { /* all ASCII */ return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]]; } bool mid; @@ -1434,7 +1515,7 @@ ScrStr *scr_str_trim_end(ScrStr *s) { * lone surrogate (empty-separator split of an astral char, a pad fill * truncated mid-pair). */ static const struct { size_t rc; size_t len; size_t cap; char data[4]; } - scr_lit_fffd = {SIZE_MAX, 3, SCR_STR_CAP_WORD(3, SCR_KEY_HASH32_SHORT(3, 0xEFBFBD)), "\xEF\xBF\xBD"}; + scr_lit_fffd = {SIZE_MAX, 3, SCR_STR_CAP_WORD(3, SCR_KEY_HASH32_SHORT(3, 0xEFBFBD), 0), "\xEF\xBF\xBD"}; /* Split owns a fresh array and appends only present string values. Fill an * available dense slot directly; ordinary push handles growth and the sparse @@ -1583,15 +1664,14 @@ ScrArr *scr_str_split(ScrStr *s, ScrStr *sep) { static ScrStr *scr_pad_impl(ScrStr *s, double maxLength, ScrStr *fill, bool at_start) { double target = scr_to_integer_or_infinity(maxLength); - ScrSidx *e = scr_sidx(s); - size_t len16 = scr_sidx_len(s, e); + size_t len16 = scr_sidx_len(s, scr_sidx_for(s)); if (!(target > (double)len16) || fill->len == 0) return scr_str_retain(s); /* Reject pad sizes malloc could not satisfy before the doubleβ†’size_t * conversion can overflow (each unit is at most 3 bytes here: BMP chars * and the U+FFFD stand-in; astral chars are 4 bytes for 2 units). */ if (target > (double)((SIZE_MAX - sizeof(ScrStr) - 1) / 4)) scr_oom(); size_t pad16 = (size_t)target - len16; - size_t fill16 = scr_sidx_len(fill, scr_sidx(fill)); + size_t fill16 = scr_sidx_len(fill, scr_sidx_for(fill)); size_t reps = pad16 / fill16, rem16 = pad16 % fill16; /* The truncated prefix of fill: whole chars while they fit in rem16 * units; a final astral char that doesn't fit contributes U+FFFD. */ @@ -1936,7 +2016,7 @@ double scr_string_to_number(ScrStr *s) { ScrStr *scr_f64_to_scrstr(double x) { char buf[32]; size_t len = scr_f64_to_str(x, buf); - return scr_str_new(buf, len); + return scr_str_new_ascii(buf, len); /* number text is always ASCII */ } /* ── encodeURIComponent / encodeURI ─────────────────────────────────── @@ -2011,9 +2091,9 @@ ScrStr *scr_str_to_well_formed(ScrStr *s) { return scr_str_retain(s); } /* Immortal interned booleans (same layout trick the emitter uses for * string literals). */ static const struct { size_t rc; size_t len; size_t cap; char data[5]; } - scr_lit_true = {SIZE_MAX, 4, 4, "true"}; + scr_lit_true = {SIZE_MAX, 4, SCR_STR_CAP_WORD(4, 0, 1), "true"}; static const struct { size_t rc; size_t len; size_t cap; char data[6]; } - scr_lit_false = {SIZE_MAX, 5, 5, "false"}; + scr_lit_false = {SIZE_MAX, 5, SCR_STR_CAP_WORD(5, 0, 1), "false"}; ScrStr *scr_bool_to_scrstr(bool b) { return b ? (ScrStr *)&scr_lit_true : (ScrStr *)&scr_lit_false; @@ -2032,8 +2112,9 @@ ScrStr *scr_str_cp_at(ScrStr *s, double i) { double idx = scr_to_integer_or_infinity(i); if (!(idx >= 0)) return scr_str_empty(); if (idx >= (double)s->len) return scr_str_empty(); /* len16 <= len */ + if (s->ascii) return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]]; ScrSidx *e = scr_sidx(s); - if (e->u16len == s->len) { /* all ASCII */ + if (s->ascii || e->u16len == s->len) { /* all ASCII */ return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]]; } bool mid; diff --git a/tests/corpus/4476-string-ascii-header-fact.ts b/tests/corpus/4476-string-ascii-header-fact.ts new file mode 100644 index 000000000..e469bf97e --- /dev/null +++ b/tests/corpus/4476-string-ascii-header-fact.ts @@ -0,0 +1,106 @@ +// UTF-16 reads (length, charCodeAt, charAt, slice, substring, indexOf, +// lastIndexOf, startsWith/endsWith at a position, padStart) over strings of +// every provenance: literals, concatenations, in-place appends, slices, +// numbers, repeat, join and split pieces. Native strings record a +// proven-ASCII fact in their header; appending a non-ASCII character to a +// uniquely owned ASCII string must drop it, and every index form +// (fractional, -0, negative, NaN, infinite, out of range) keeps Node's +// semantics on both representations, including surrogate halves. (Reads +// never split a surrogate pair into a string: scriptc substitutes U+FFFD.) + +function codes(s: string): string { + const out: number[] = []; + for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i)); + return out.join(","); +} + +function probe(label: string, s: string): void { + const idx = [0, -0, 0.5, 1.9999, -0.5, -1, NaN, Infinity, -Infinity, s.length - 1, s.length, 2 ** 32]; + const units: string[] = []; + for (const i of idx) units.push(String(s.charCodeAt(i))); + const chars: string[] = []; + for (const i of idx) chars.push(JSON.stringify(s.charAt(i))); + console.log(label, s.length, units.join(" "), chars.join(" ")); + console.log( + " slice", + JSON.stringify(s.slice(1, -1)), + JSON.stringify(s.slice(-3)), + JSON.stringify(s.substring(5, 2)), + JSON.stringify(s.slice(2, 2)), + ); + console.log( + " search", + s.indexOf("b"), + s.indexOf("b", 3), + s.lastIndexOf("b"), + s.indexOf(""), + s.indexOf("", 99), + s.startsWith("b", 1), + s.endsWith("b", 2), + ); + console.log(" pad", JSON.stringify(s.padStart(s.length + 3, "-")), JSON.stringify(s.padEnd(s.length + 2, "\u00e9"))); +} + +// Literals: ASCII, BMP and astral. +probe("lit-ascii", "abcabc"); +probe("lit-bmp", "ab\u00e9cb\u00e9"); +probe("lit-astral", "ab\u{1F600}cbd"); + +// Concatenations of every pairing. +const n = 7; +const left = "ab" + String(n); +const right = "\u00e9b" + String(n); +probe("cat-aa", left + "cb"); +probe("cat-an", left + right); +probe("cat-na", right + left); +probe("cat-astral", left + "\u{1F600}b"); + +// Uniquely owned strings appended in place after an ASCII read. +let grow = ""; +for (let i = 0; i < 6; i++) grow += String.fromCharCode(97 + i); +console.log("grow", grow.length, grow.charCodeAt(3), codes(grow)); +grow += "\u00e9"; +console.log("grow+bmp", grow.length, grow.charCodeAt(6), grow.charCodeAt(7), codes(grow)); +grow += "\u{1F600}b"; +console.log("grow+astral", grow.length, grow.charCodeAt(7), grow.charCodeAt(8), grow.charCodeAt(9), codes(grow)); +probe("grow", grow); + +let chain = "x"; +for (let i = 0; i < 40; i++) { + chain += i % 13 === 12 ? "\u03a9" : String(i % 10); + if (chain.charCodeAt(chain.length - 1) > 127) console.log("non-ascii at", chain.length - 1, chain.charCodeAt(chain.length - 1)); +} +console.log("chain", chain.length, chain.indexOf("\u03a9"), chain.lastIndexOf("\u03a9"), chain.slice(10, 16)); + +// Template literals mixing numbers and strings. +const id = 42; +const key = `${id},${id + 1}|${right}`; +probe("template", key); + +// Slices: ASCII slices of non-ASCII strings and the reverse. +const mixed = "\u00e9abcb\u00e9xyz\u{1F600}!"; +const inner = mixed.slice(1, 5); +probe("slice-ascii-of-mixed", inner); +probe("slice-mixed", mixed.slice(0, 7)); +probe("substring-astral", mixed.substring(7, 13)); +console.log("astral halves", mixed.charCodeAt(9), mixed.charCodeAt(10), mixed.slice(9, 11) === "\u{1F600}"); + +// Numbers, repeat, join, split pieces. +probe("number", String(-1234.5e-7) + "b"); +probe("repeat", "ab".repeat(4)); +probe("repeat-bmp", "a\u00e9".repeat(3)); +probe("join", ["ab", "cb", "d"].join("")); +probe("join-bmp", ["ab", "\u00e9", "cb"].join("-")); +const pieces = "abcdefgh,b\u00e9cdefgh,abcdefgb".split(","); +for (const p of pieces) console.log("piece", p.length, p.charCodeAt(1), p.charCodeAt(p.length - 1), codes(p)); + +// Interleaved reads of several receivers (index cache residency). +const a1 = "alpha\u00e9" + String(n); +const a2 = "beta" + String(n); +const a3 = "\u{1F600}gamma"; +let mix = 0; +for (let i = 0; i < 8; i++) { + mix = mix * 31 + a1.charCodeAt(i % a1.length) + a2.charCodeAt(i % a2.length) * 3 + a3.charCodeAt(i % a3.length) * 7; + mix %= 1000003; +} +console.log("interleaved", mix); diff --git a/tests/corpus/4478-string-large-index-reads.ts b/tests/corpus/4478-string-large-index-reads.ts new file mode 100644 index 000000000..67c330eb0 --- /dev/null +++ b/tests/corpus/4478-string-large-index-reads.ts @@ -0,0 +1,75 @@ +// Sequential and random UTF-16 reads over large strings (well past 64 KiB): +// an all-ASCII text, a text with sparse non-ASCII and astral characters, a +// line-start scan in the style of a compiler scanner, alternating scans of +// two large receivers, and a uniquely owned large ASCII string that turns +// non-ASCII through an in-place append. Every result must match Node. + +function lineStarts(text: string): number[] { + const result: number[] = []; + let lineStart = 0; + let pos = 0; + while (pos < text.length) { + const ch = text.charCodeAt(pos); + pos++; + if (ch === 13) { + if (pos < text.length && text.charCodeAt(pos) === 10) pos++; + } else if (ch !== 10 && ch !== 0x2028 && ch !== 0x2029) { + continue; + } + result.push(lineStart); + lineStart = pos; + } + result.push(lineStart); + return result; +} + +function summary(label: string, text: string): void { + const starts = lineStarts(text); + let sum = 0; + for (let i = 0; i < text.length; i += 997) sum = (sum * 33 + text.charCodeAt(i)) % 1000000007; + let back = 0; + for (let i = text.length - 1; i >= 0; i -= 1009) back = (back * 31 + text.charCodeAt(i)) % 1000000007; + console.log(label, text.length, starts.length, starts[starts.length - 1], starts[1000] ?? -1, sum, back); +} + +const lines: string[] = []; +for (let i = 0; i < 6000; i++) lines.push(`const value${i} = compute(${i}, "text") + ${i * 3};`); +const ascii = lines.join("\n"); +summary("ascii", ascii); + +const mixedLines: string[] = []; +for (let i = 0; i < 6000; i++) { + const note = i % 97 === 0 ? " // caf\u00e9 \u{1F600}" : i % 501 === 0 ? "\u2028" : ""; + mixedLines.push(`let item${i} = [${i}, ${i + 1}];${note}`); +} +const mixed = mixedLines.join(i_crlf()); +summary("mixed", mixed); + +function i_crlf(): string { + return "\r\n"; +} + +// Alternate between the two large receivers and a short one. +let alt = 0; +const short = "s\u00e9q"; +for (let i = 0; i < 20000; i += 7) { + alt = (alt * 7 + ascii.charCodeAt(i) + mixed.charCodeAt(i) * 3 + short.charCodeAt(i % 3)) % 1000000007; +} +console.log("alternate", alt); + +// Search and slice far into both. +const far = mixed.indexOf("item5999"); +console.log("far", far, mixed.slice(far, far + 12), mixed.lastIndexOf("\u{1F600}"), mixed.lastIndexOf("caf\u00e9")); +const astralAt = mixed.indexOf("\u{1F600}"); +console.log("astral", astralAt, mixed.charCodeAt(astralAt), mixed.charCodeAt(astralAt + 1), mixed.charAt(astralAt - 1)); +console.log("ascii far", ascii.indexOf("value5999"), ascii.slice(-20), ascii.substring(100000, 100010).length); + +// A large uniquely owned ASCII string read, then appended in place. +let big = ""; +for (let i = 0; i < 9000; i++) big += "line" + String(i) + "\n"; +console.log("big", big.length, big.charCodeAt(70000), big.charCodeAt(big.length - 1)); +big += "\u00e9nd"; +console.log("big+bmp", big.length, big.charCodeAt(big.length - 3), big.charCodeAt(big.length - 1), big.slice(-6)); +big += "\u{1F600}"; +console.log("big+astral", big.length, big.charCodeAt(big.length - 2), big.charCodeAt(big.length - 1), big.indexOf("\u00e9")); +summary("big", big); From 83ab6ce2a8acc88280377aab406383c94bd03976 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 8 Oct 2026 23:32:26 -0700 Subject: [PATCH 2/3] Encode ASCII runs eight bytes at a time in Buffer#write buf.write(string, 'utf16le' | 'latin1' | 'ascii') decoded one UTF-8 code point per step. tsc-ts converts every source file to UTF-16 this way (toCodeUnits), so the per-character loop showed up in the profile. Eight-byte ASCII runs whose units all fit the remaining budget are now widened (utf16le) or copied directly; any other position, and a tail that would cross the budget, takes the unchanged per-code-point path, so truncation at a unit or surrogate boundary is exactly as before. --- .../test/ts7/baselines/order-parity.json | 6 ++ packages/runtime/src/scr_bytes.c | 21 +++++++ tests/corpus/4477-buffer-write-ascii-runs.ts | 55 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/corpus/4477-buffer-write-ascii-runs.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index dd4d8373e..6fa10a6cb 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -15467,6 +15467,12 @@ ], "diags": [] }, + "/tests/corpus/4477-buffer-write-ascii-runs.ts": { + "order": [ + "/tests/corpus/4477-buffer-write-ascii-runs.ts" + ], + "diags": [] + }, "/tests/corpus/4478-string-large-index-reads.ts": { "order": [ "/tests/corpus/4478-string-large-index-reads.ts" diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 3a87bd7c2..59a7fe152 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -2736,7 +2736,28 @@ double scr_bytes_write_str(ScrBytes *b, const ScrStr *s, const ScrStr *enc, bool utf16 = scr_enc_is(enc, "utf16le"); if (utf16 || scr_enc_is(enc, "latin1") || scr_enc_is(enc, "ascii")) { size_t i = 0, written = 0, width = utf16 ? 2 : 1; + const uint64_t high = UINT64_C(0x8080808080808080); while (i < s->len && budget - written >= width) { + /* ASCII runs encode byte-for-unit: widen (or copy) eight bytes per + * step while the budget holds them all, else one code point below. */ + if (s->len - i >= 8 && budget - written >= 8 * width) { + uint64_t w; + memcpy(&w, s->data + i, 8); + if (!(w & high)) { + uint8_t *d = b->data + o + written; + if (utf16) { + for (int k = 0; k < 8; k++) { + d[2 * k] = (uint8_t)s->data[i + k]; + d[2 * k + 1] = 0; + } + } else { + memcpy(d, s->data + i, 8); + } + i += 8; + written += 8 * width; + continue; + } + } uint32_t cp = scr_bytes_next_cp((const uint8_t *)s->data, &i); uint32_t first = cp, second = 0; if (cp > 0xffff) { diff --git a/tests/corpus/4477-buffer-write-ascii-runs.ts b/tests/corpus/4477-buffer-write-ascii-runs.ts new file mode 100644 index 000000000..865338db6 --- /dev/null +++ b/tests/corpus/4477-buffer-write-ascii-runs.ts @@ -0,0 +1,55 @@ +// Buffer#write with utf16le, latin1 and ascii encodings: long ASCII runs, +// runs broken by BMP and astral characters, writes truncated by an offset +// and length budget (including odd budgets that split a UTF-16 unit or a +// surrogate pair), and the Uint16Array view a scanner builds from text. + +function hex(buf: Buffer): string { + return buf.toString("hex"); +} + +const texts = [ + "", + "abc", + "abcdefgh", + "abcdefghijklmnopq", + "const x = 1;\nconst y = 2;\n", + "abcdefg\u00e9hijklmnopqrstu", + "\u00e9abcdefghijklmnop", + "abcdefghijklmn\u{1F600}opqrstuvwxyz0123", + "\u4e2d\u6587abcdefghijklmnopqrstuvwxyz", +]; + +function show(enc: string, text: string, n: number, buf: Buffer): void { + console.log(enc, text.length, n, hex(buf.subarray(0, n))); +} + +for (const text of texts) { + const size = text.length * 2 + 4; + const u16 = Buffer.alloc(size); + show("utf16le", text, u16.write(text, "utf16le"), u16); + const l1 = Buffer.alloc(size); + show("latin1", text, l1.write(text, "latin1"), l1); + const as = Buffer.alloc(size); + show("ascii", text, as.write(text, "ascii"), as); + for (const budget of [1, 3, 7, 15, 16, 17, 33]) { + const a = Buffer.alloc(40); + const b = Buffer.alloc(40); + const c = Buffer.alloc(40); + const wa = a.write(text, 2, budget, "utf16le"); + const wb = b.write(text, 2, budget, "latin1"); + const wc = c.write(text, 2, budget, "ascii"); + console.log(" budget", budget, wa, hex(a.subarray(0, 2 + wa)), wb, hex(b.subarray(0, 2 + wb)), wc, hex(c.subarray(0, 2 + wc))); + } +} + +function toCodeUnits(text: string): Uint16Array { + const chars = new Uint16Array(text.length + 4); + Buffer.from(chars.buffer).write(text, "utf16le"); + return chars; +} + +const source = "function f(a) {\n return a + 1; // caf\u00e9 \u{1F600}\n}\n".repeat(40); +const units = toCodeUnits(source); +let ok = units.length === source.length + 4; +for (let i = 0; i < source.length; i++) if (units[i] !== source.charCodeAt(i)) ok = false; +console.log("code units", units.length, ok, units[source.length], units[source.length + 3]); From af2521f688a5096c431dc356060b0dcf3d96fb96 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 9 Oct 2026 00:11:31 -0700 Subject: [PATCH 3/3] Copy short strings inline and drop index entries of proven-ASCII strings String construction copied every span with a libc memcpy call, and scr_str_new scanned the copied bytes for ASCII in a second pass. Spans of at most 16 bytes (identifiers, map keys, number text) now move as two overlapping fixed-size words, and scr_str_new tests those same words for ASCII, so the fact is free for short strings; longer spans keep the bulk copy and a chunked scan that stops at the first non-ASCII byte. A scan that proves a heap string ASCII now also drops its index-cache entry: a proven-ASCII string never owns one, because every UTF-16 mapper answers it by identity and re-checks the header bit after extending an index. Releasing such a string therefore skips the cache purge, and a large text proven ASCII once (a JSON document's length, a source file) no longer keeps every later string release walking the cache. Immortal strings cannot record the bit and keep their entries as before. --- packages/runtime/src/scr_runtime.h | 27 +++++++ packages/runtime/src/scr_string.c | 78 +++++++++++++++---- tests/corpus/4476-string-ascii-header-fact.ts | 11 +++ tests/corpus/4478-string-large-index-reads.ts | 10 +++ 4 files changed, 110 insertions(+), 16 deletions(-) diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 58b98dcf8..4af9a4f99 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -711,6 +711,33 @@ static inline void scr_str_note_ascii(ScrStr *s) { if (s->rc != SIZE_MAX) s->ascii = 1; } +/* memcpy for the short, variable-length copies of string construction + * (identifiers, keys, number text): fixed-size overlapping moves instead of + * a libc call. Longer spans keep the platform's bulk copy. */ +static inline void scr_copy_bytes(char *dst, const char *src, size_t n) { + if (n <= 16) { + if (n >= 8) { + uint64_t a, b; + memcpy(&a, src, 8); + memcpy(&b, src + n - 8, 8); + memcpy(dst, &a, 8); + memcpy(dst + n - 8, &b, 8); + } else if (n >= 4) { + uint32_t a, b; + memcpy(&a, src, 4); + memcpy(&b, src + n - 4, 4); + memcpy(dst, &a, 4); + memcpy(dst + n - 4, &b, 4); + } else if (n) { + dst[0] = src[0]; + dst[n / 2] = src[n / 2]; + dst[n - 1] = src[n - 1]; + } + return; + } + memcpy(dst, src, n); +} + /* Whether a byte span is all ASCII: 32 bytes per step, stopping at the * first chunk with a high bit. */ static inline bool scr_bytes_all_ascii(const char *p, size_t n) { diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 09a6e5f10..7c56d245c 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -291,13 +291,47 @@ static ScrStr *scr_str_alloc(size_t len, size_t cap) { return s; } +/* scr_copy_bytes that also reports whether the bytes are all ASCII. Short + * spans test the words they already moved; longer ones scan after the bulk + * copy, stopping at the first non-ASCII chunk. */ +static bool scr_copy_bytes_ascii(char *dst, const char *src, size_t n) { + const uint64_t high = UINT64_C(0x8080808080808080); + if (n <= 16) { + if (n >= 8) { + uint64_t a, b; + memcpy(&a, src, 8); + memcpy(&b, src + n - 8, 8); + memcpy(dst, &a, 8); + memcpy(dst + n - 8, &b, 8); + return ((a | b) & high) == 0; + } + if (n >= 4) { + uint32_t a, b; + memcpy(&a, src, 4); + memcpy(&b, src + n - 4, 4); + memcpy(dst, &a, 4); + memcpy(dst + n - 4, &b, 4); + return ((a | b) & UINT32_C(0x80808080)) == 0; + } + if (n == 0) return true; + unsigned char c0 = (unsigned char)src[0], c1 = (unsigned char)src[n / 2], + c2 = (unsigned char)src[n - 1]; + dst[0] = (char)c0; + dst[n / 2] = (char)c1; + dst[n - 1] = (char)c2; + return ((c0 | c1 | c2) & 0x80) == 0; + } + memcpy(dst, src, n); + return scr_bytes_all_ascii(src, n); +} + ScrStr *scr_str_new(const char *bytes, size_t len) { ScrStr *s = scr_str_alloc(len, len); - memcpy(s->data, bytes, len); + /* Proving ASCII while copying lets every later UTF-16 index, length and + * slice skip the index cache. */ + bool ascii = scr_copy_bytes_ascii(s->data, bytes, len); s->data[len] = '\0'; - /* The copy already streams the bytes; proving ASCII here lets every later - * UTF-16 index, length and slice skip the index cache. */ - if (scr_bytes_all_ascii(bytes, len)) s->ascii = 1; + s->ascii = ascii; return s; } @@ -305,7 +339,7 @@ ScrStr *scr_str_new(const char *bytes, size_t len) { * string): no scan. */ static ScrStr *scr_str_new_ascii(const char *bytes, size_t len) { ScrStr *s = scr_str_alloc(len, len); - memcpy(s->data, bytes, len); + scr_copy_bytes(s->data, bytes, len); s->data[len] = '\0'; s->ascii = 1; return s; @@ -364,7 +398,9 @@ void scr_str_release(ScrStr *s) { if (!s || s->rc == SIZE_MAX) return; /* NULL: an uninitialized `let` local */ if (--s->rc == 0) { scr_short_forget(s); - scr_sidx_purge(s); /* the address may be recycled by the next malloc */ + /* The address may be recycled by the next malloc. A proven-ASCII string + * owns no index entry (scr_sidx_finish). */ + if (!s->ascii) scr_sidx_purge(s); #ifdef SCR_RC_AUDIT scr_live_strings--; #endif @@ -395,7 +431,7 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) { size_t oldlen = a->len; scr_short_forget(a); scr_str_hash_forget(a); - memcpy(a->data + a->len, b->data, b->len); + scr_copy_bytes(a->data + a->len, b->data, b->len); a->len = newlen; a->data[newlen] = '\0'; a->ascii = ascii; @@ -427,8 +463,8 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) { if (newcap > SCR_STR_MAX_CAP && newlen <= SCR_STR_MAX_CAP) newcap = SCR_STR_MAX_CAP; ScrStr *s = scr_str_take_spare(newlen); if (!s) s = scr_str_alloc(newlen, newcap); - memcpy(s->data, a->data, a->len); - memcpy(s->data + a->len, b->data, b->len); + scr_copy_bytes(s->data, a->data, a->len); + scr_copy_bytes(s->data + a->len, b->data, b->len); s->data[newlen] = '\0'; s->ascii = ascii; return s; @@ -483,11 +519,11 @@ ScrStr *scr_str_concat_mixed(ScrStr *head, ScrStr *const *parts, const double *n if (newcap > SCR_STR_MAX_CAP) newcap = SCR_STR_MAX_CAP; out = scr_str_take_spare(newlen); if (!out) out = scr_str_alloc(newlen, newcap); - if (base) memcpy(out->data, head->data, base); + if (base) scr_copy_bytes(out->data, head->data, base); at = base; } for (size_t i = 0; i < count; i++) { - memcpy(out->data + at, parts[i] ? parts[i]->data : text[i], lens[i]); + scr_copy_bytes(out->data + at, parts[i] ? parts[i]->data : text[i], lens[i]); at += lens[i]; } out->len = newlen; @@ -525,7 +561,7 @@ ScrStr *scr_str_concat_parts(ScrStr *const *parts, size_t count) { bool ascii = true; for (size_t i = 0; i < count; i++) { const ScrStr *part = parts[i]; - memcpy(out->data + offset, part->data, part->len); + scr_copy_bytes(out->data + offset, part->data, part->len); offset += part->len; ascii &= part->ascii; } @@ -845,11 +881,21 @@ static void scr_sidx_rebuild_points(const ScrStr *s, ScrSidx *e) { e->points_complete = true; } +/* Completing a scan that proves a heap string ASCII records the fact in its + * header and drops the entry: a proven-ASCII string never owns one (every + * mapper answers it by identity first, and a release of it skips the + * purge). Callers re-check the header bit before touching the entry again. + * Immortal strings cannot record the bit and keep the entry. */ static void scr_sidx_finish(const ScrStr *s, ScrSidx *e) { if (e->indexed_cb != s->len) return; e->u16len = e->indexed_cu; if (e->u16len == s->len) { /* proven all ASCII: identity needs no index */ - scr_str_note_ascii((ScrStr *)s); + if (s->rc != SIZE_MAX) { + ((ScrStr *)s)->ascii = 1; + bool sparse = (uintptr_t)e - (uintptr_t)scr_sidx_sparse_tab < sizeof scr_sidx_sparse_tab; + scr_sidx_clear(e, sparse ? &scr_sidx_sparse_live : NULL); + return; + } free(e->points); e->points = NULL; e->npoints = 0; @@ -897,7 +943,7 @@ static size_t scr_sidx_len(const ScrStr *s, ScrSidx *e) { if (e->u16len != SCR_U16_UNKNOWN) return e->u16len; while (e->indexed_cb < s->len) scr_sidx_extend_one(s, e); scr_sidx_finish(s, e); - return e->u16len; + return s->ascii ? s->len : e->u16len; } /* Step the cursor back one char (cb must be > 0 and on a boundary). */ @@ -1040,7 +1086,7 @@ static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, scr_sidx_extend_to_u16(s, e, u16); /* Extending a far-end lookup can just have proved identity. Do not * materialize an index that the identity fast path will never consult. */ - if (e->u16len == s->len) { + if (s->ascii || e->u16len == s->len) { *mid = false; return u16 < s->len ? u16 : s->len; } @@ -1108,7 +1154,7 @@ static size_t scr_byte_to_u16_c(const ScrStr *s, ScrSidx *e, } /* As above, a completed all-ASCII scan is its own index. In particular, * do not rebuild points that scr_sidx_finish() deliberately discarded. */ - if (e->u16len == s->len) return byte_off; + if (s->ascii || e->u16len == s->len) return byte_off; scr_sidx_materialize_identity_prefix(s, e); ScrSidxPoint near = scr_sidx_near_byte(s, e, byte_off); size_t cu = near.cu, cb = near.cb; diff --git a/tests/corpus/4476-string-ascii-header-fact.ts b/tests/corpus/4476-string-ascii-header-fact.ts index e469bf97e..3f47451d5 100644 --- a/tests/corpus/4476-string-ascii-header-fact.ts +++ b/tests/corpus/4476-string-ascii-header-fact.ts @@ -94,6 +94,17 @@ probe("join-bmp", ["ab", "\u00e9", "cb"].join("-")); const pieces = "abcdefgh,b\u00e9cdefgh,abcdefgb".split(","); for (const p of pieces) console.log("piece", p.length, p.charCodeAt(1), p.charCodeAt(p.length - 1), codes(p)); +// Builder-made strings (join, JSON.stringify) are proven ASCII by their +// first read, then appended in place with non-ASCII text and read again. +let built = ["ab", "cd", "ef", String(n)].join(""); +console.log("built", built.length, built.charCodeAt(4), built.indexOf("7"), built.slice(2, 5)); +built += "\u00e9" + built; +console.log("built+bmp", built.length, built.charCodeAt(7), built.charCodeAt(8), built.lastIndexOf("e"), built.slice(5, 9)); +let json = JSON.stringify({ k: [1, 2, 3], s: "x" }); +console.log("json", json.length, json.charCodeAt(2), json.indexOf("s")); +json += "\u{1F600}"; +console.log("json+astral", json.length, json.charCodeAt(json.length - 2), json.charCodeAt(json.length - 1), json.slice(-4)); + // Interleaved reads of several receivers (index cache residency). const a1 = "alpha\u00e9" + String(n); const a2 = "beta" + String(n); diff --git a/tests/corpus/4478-string-large-index-reads.ts b/tests/corpus/4478-string-large-index-reads.ts index 67c330eb0..0a85415f9 100644 --- a/tests/corpus/4478-string-large-index-reads.ts +++ b/tests/corpus/4478-string-large-index-reads.ts @@ -73,3 +73,13 @@ console.log("big+bmp", big.length, big.charCodeAt(big.length - 3), big.charCodeA big += "\u{1F600}"; console.log("big+astral", big.length, big.charCodeAt(big.length - 2), big.charCodeAt(big.length - 1), big.indexOf("\u00e9")); summary("big", big); + +// A large builder-made string proven ASCII by its length, then appended in +// place: the proof must not survive the append. +const rows: number[][] = []; +for (let i = 0; i < 5000; i++) rows.push([i, i * 2, i % 7]); +let doc = JSON.stringify(rows); +console.log("doc", doc.length, doc.charCodeAt(doc.length - 1), doc.indexOf("[4999")); +doc += "\u00e9\u{1F600}"; +console.log("doc+", doc.length, doc.charCodeAt(doc.length - 3), doc.charCodeAt(doc.length - 2), doc.charCodeAt(doc.length - 1), doc.slice(-8)); +summary("doc", doc);