Merge pull request #794 from vercel-labs/perf/tscts/07-ascii-strings

tsc-ts -6%, runtime suite -6%: cache proven-ASCII strings and inline length/charCodeAt
This commit is contained in:
Malte Ubl
2026-10-09 12:15:08 -05:00
committed by GitHub
16 changed files with 703 additions and 74 deletions
+14 -2
View File
@@ -289,8 +289,9 @@ import { LlvmUnsupportedError } from "./unsupported.js";
import { LlWalkers } from "./walkers.js";
import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js";
import { emitNullablePresent, emitUnionPeek, emitUnionTag } from "./union-repr.js";
import { literalCapWord } from "./string-key-hash.js";
import { literalCapWord, literalCapWord32 } from "./string-key-hash.js";
import { NUMBER_MAP_ENTRY_DECL } from "./map-number-lookup.js";
import { STRING_READ_HELPERS } from "./string-reads.js";
import {
emitConcatInputs,
emitMixedConcat,
@@ -1770,8 +1771,11 @@ export class LlEmitter {
// the runtime ABI’s static table, retain/release skip rc == SIZE_MAX.
// 64-bit (little-endian) targets split the capacity word: the high
// half carries the precomputed Map key hash (SCR_STR_HASH_CACHE).
// Bit 31 marks all-ASCII literals (SCR_STR_ASCII_BIT) on every target.
const capWord =
this.sizeType === "i64" ? literalCapWord(Buffer.from(text, "utf8")) : String(lit.len);
this.sizeType === "i64"
? literalCapWord(Buffer.from(text, "utf8"))
: literalCapWord32(Buffer.from(text, "utf8"));
out.push(
`@${lit.sym} = internal global { ${this.sizeType}, ${this.sizeType}, ${this.sizeType}, [${lit.len + 1} x i8] } ` +
`{ ${this.sizeType} -1, ${this.sizeType} ${lit.len}, ${this.sizeType} ${capWord}, [${lit.len + 1} x i8] c"${llStrBytes(text)}" }`,
@@ -2643,6 +2647,14 @@ export class LlEmitter {
``,
);
}
// String length and charCodeAt read proven-ASCII strings inline: bit 31
// of the capacity word (SCR_STR_ASCII_BIT, the low half's sign bit on
// little-endian targets) makes UTF-16 units bytes. Unproven strings and
// anything but an in-range index take the runtime's UTF-16 mapping.
for (const helper of STRING_READ_HELPERS) {
if (!this.decls.has(helper.decl)) continue;
defs.push(...helper.define(this.sizeType, FN_ATTRS), ``);
}
// Ordering also decides inline when the first bytes differ: byte order
// is the plain comparison's order, and the UTF-16 order whenever both
// bytes are ASCII. Callers only test the sign of the result.
@@ -30,6 +30,7 @@ import {
} from "./map-number-lookup.js";
import { emitBorrowedInput } from "./borrowed-inputs.js";
import { emitStringSliceRead } from "./string-slices.js";
import { STRING_CHAR_CODE_AT_DECL, STRING_UTF16_LEN_DECL } from "./string-reads.js";
import { emitDenseArrayNumber, emitDenseArrayPush } from "./dense-array-access.js";
import { emitUnionWiden } from "./expr-records.js";
@@ -189,16 +190,19 @@ export function emitStrIntrinsic(
};
const method = e.method;
switch (method) {
case "length":
return call("scr_str_utf16_len", "double (ptr)", `ptr ${r.name}`, "double", false);
case "charCodeAt":
return call(
"scr_str_char_code_at",
"double (ptr, double)",
`ptr ${r.name}, double ${args[0]!.name}`,
"double",
false,
);
case "length": {
// sc_str_utf16_len (string-reads.ts) answers proven-ASCII strings inline.
host.declare(STRING_UTF16_LEN_DECL);
const t = B.tmp();
B.line(`${t} = call double @sc_str_utf16_len(ptr ${r.name})`);
return { name: t, type: e.type };
}
case "charCodeAt": {
host.declare(STRING_CHAR_CODE_AT_DECL);
const t = B.tmp();
B.line(`${t} = call double @sc_str_char_code_at(ptr ${r.name}, double ${args[0]!.name})`);
return { name: t, type: e.type };
}
case "charAt":
return call(
"scr_str_char_at",
@@ -80,8 +80,8 @@ test.each([32, 64] as const)(
const zero = /^define internal [^\n]*@sc_[bf]+_length_0\([^]*?^}/m.exec(llvm)![0];
const one = /^define internal [^\n]*@sc_[bf]+_length_1\([^]*?^}/m.exec(llvm)![0];
expect(zero).toContain(`getelementptr inbounds i${bits}`);
expect(zero).not.toContain("@scr_str_utf16_len");
expect(one).toContain("@scr_str_utf16_len");
expect(zero).not.toContain("_str_utf16_len");
expect(one).toContain("@sc_str_utf16_len");
},
);
@@ -1,5 +1,5 @@
import { describe, expect, test } from "vitest";
import { literalCapWord, stringKeyHash32 } from "./string-key-hash.js";
import { literalCapWord, literalCapWord32, stringKeyHash32 } from "./string-key-hash.js";
// Shared with packages/runtime/test/test_map.c (hash_vectors): the runtime
// computes these for heap strings, the compiler bakes them into literals.
@@ -26,12 +26,24 @@ describe("string key hash", () => {
expect(stringKeyHash32(Buffer.from("ab"))).not.toBe(stringKeyHash32(Buffer.from("abc")));
});
test("literal capacity word keeps cap = len in the low half", () => {
test("literal capacity word keeps cap = len in the low 31 bits", () => {
for (const [text, hash] of VECTORS) {
const bytes = Buffer.from(text, "utf8");
const word = BigInt.asUintN(64, BigInt(literalCapWord(bytes)));
expect(word & 0xffffffffn).toBe(BigInt(bytes.length));
expect(word & 0x7fffffffn).toBe(BigInt(bytes.length));
expect(word >> 32n).toBe(BigInt(hash));
}
});
test("literal capacity words carry the proven-ASCII bit exactly", () => {
for (const [text] of VECTORS) {
const bytes = Buffer.from(text, "utf8");
const ascii = bytes.every((byte) => byte < 0x80);
const word = BigInt.asUintN(64, BigInt(literalCapWord(bytes)));
expect((word & 0x80000000n) !== 0n).toBe(ascii);
const word32 = Number(literalCapWord32(bytes)) >>> 0;
expect(word32 & 0x7fffffff).toBe(bytes.length);
expect((word32 & 0x80000000) !== 0).toBe(ascii);
}
});
});
@@ -57,10 +57,28 @@ export function stringKeyHash32(bytes: Uint8Array): number {
return folded === 0 ? 1 : folded;
}
/** Bit 31 of every capacity word: the string is proven all ASCII, so the
* runtime and inline string reads map UTF-16 units to bytes by identity
* (SCR_STR_ASCII_BIT in scr_runtime.h). */
export const STRING_ASCII_BIT = 0x80000000;
function isAscii(bytes: Uint8Array): boolean {
for (const byte of bytes) if (byte >= 0x80) return false;
return true;
}
/** The capacity word of an immortal literal on 64-bit targets: cap = len in
* the low half, the precomputed key hash in the high half, printed as the
* signed i64 LLVM expects. */
* bits 0-30, the proven-ASCII bit, the precomputed key hash in the high
* half, printed as the signed i64 LLVM expects. */
export function literalCapWord(bytes: Uint8Array): string {
const word = BigInt(bytes.length) | (BigInt(stringKeyHash32(bytes)) << 32n);
const ascii = isAscii(bytes) ? BigInt(STRING_ASCII_BIT) : 0n;
const word = BigInt(bytes.length) | ascii | (BigInt(stringKeyHash32(bytes)) << 32n);
return BigInt.asIntN(64, word).toString();
}
/** The capacity word of an immortal literal on 32-bit targets: cap = len
* plus the proven-ASCII bit, printed as the signed i32 LLVM expects. */
export function literalCapWord32(bytes: Uint8Array): string {
const word = bytes.length + (isAscii(bytes) ? STRING_ASCII_BIT : 0);
return String(word | 0);
}
@@ -277,9 +277,11 @@ const methodCases: {
args: IrExpr[];
result: IrType;
target: string;
/** Sites enter through the inlined sc_str_* wrapper (string-reads.ts). */
inline?: boolean;
}[] = [
{ method: "length", args: [], result: F64, target: "utf16_len" },
{ method: "charCodeAt", args: [num(1)], result: F64, target: "char_code_at" },
{ method: "length", args: [], result: F64, target: "utf16_len", inline: true },
{ method: "charCodeAt", args: [num(1)], result: F64, target: "char_code_at", inline: true },
{ method: "charAt", args: [num(1)], result: STRING, target: "char_at" },
{ method: "indexOf", args: [ref("argument"), num(1)], result: F64, target: "index_of" },
{ method: "includes", args: [ref("argument")], result: BOOL, target: "includes" },
@@ -314,7 +316,7 @@ const methodCases: {
test.each(methodCases)(
"$method passes borrowed string inputs through its runtime ABI",
({ method, args, result, target }) => {
({ method, args, result, target, inline }) => {
const f = fn(
"method",
["receiver", "argument"],
@@ -322,7 +324,7 @@ test.each(methodCases)(
);
for (const bits of [32, 64] as const) {
const ir = body(mod(f), "sc_bf_method", bits);
expect(ir).toContain(`@scr_str_${target}(`);
expect(ir).toContain(`@${inline ? "sc" : "scr"}_str_${target}(`);
expect(ir).not.toContain("@scr_str_retain_v");
// Throwing methods release their owned result on the unwind path.
if (method !== "normalize" && method !== "repeat")
@@ -381,7 +383,7 @@ test("boxed string lengths borrow initialized payloads and keep checked TDZ read
expect(ir).toMatch(/br i1 %t\d+, label %lazy\.release\d+/);
expect(ir).toContain("@scr_box_release");
expect(ir).not.toContain("@scr_box_get_ref");
expect(ir).toContain("@scr_str_utf16_len");
expect(ir).toContain("@sc_str_utf16_len");
inspect.locals[0]!.tdz = true;
const checked = body(mod(inspect), "sc_f_inspect");
expect(checked).toContain("@scr_box_get_ref");
@@ -0,0 +1,84 @@
/* Inline UTF-16 reads of proven-ASCII strings.
*
* Strings are UTF-8, so `length` and `charCodeAt` need a UTF-16 mapping in
* general. Bit 31 of a string's capacity word (SCR_STR_ASCII_BIT in
* scr_runtime.h) proves every byte is ASCII, where units are bytes. Sites
* call the internal sc_str_* helpers below, which LLVM inlines: the proven
* case is a header load and, for charCodeAt, one byte load; every other
* string or index defers to the runtime function, which keeps the complete
* semantics (surrogate halves, out-of-range NaN, fractional, negative and
* NaN indices). On little-endian targets the bit is the sign bit of the
* capacity word's low 32 bits on both i64 and i32 layouts.
*/
export const STRING_UTF16_LEN_DECL = "declare double @scr_str_utf16_len(ptr)";
export const STRING_CHAR_CODE_AT_DECL = "declare double @scr_str_char_code_at(ptr, double)";
/** Load the proven-ASCII flag of `%s` into `%ascii`. */
function asciiTest(): string[] {
return [
` %cw.p = getelementptr inbounds %ScrStr, ptr %s, i32 0, i32 2`,
` %cw = load i32, ptr %cw.p`,
` %ascii = icmp slt i32 %cw, 0`,
];
}
function loadLength(sz: string): string[] {
return [
` %len.p = getelementptr inbounds %ScrStr, ptr %s, i32 0, i32 1`,
` %len = load ${sz}, ptr %len.p`,
];
}
export interface StringReadHelper {
decl: string;
define(sz: string, attrs: string): string[];
}
export const STRING_READ_HELPERS: readonly StringReadHelper[] = [
{
decl: STRING_UTF16_LEN_DECL,
define: (sz, attrs) => [
`define internal double @sc_str_utf16_len(ptr %s) ${attrs} {`,
`entry:`,
...asciiTest(),
` br i1 %ascii, label %fast, label %slow`,
`fast:`,
...loadLength(sz),
` %n = uitofp ${sz} %len to double`,
` ret double %n`,
`slow:`,
` %r = call double @scr_str_utf16_len(ptr %s)`,
` ret double %r`,
`}`,
],
},
{
decl: STRING_CHAR_CODE_AT_DECL,
define: (sz, attrs) => [
`define internal double @sc_str_char_code_at(ptr %s, double %i) ${attrs} {`,
`entry:`,
...asciiTest(),
` br i1 %ascii, label %bounds, label %slow`,
`bounds:`,
...loadLength(sz),
` %lenf = uitofp ${sz} %len to double`,
// NaN fails both ordered comparisons; -0 passes and truncates to 0.
` %lo = fcmp oge double %i, 0.0`,
` %hi = fcmp olt double %i, %lenf`,
` %in = and i1 %lo, %hi`,
` br i1 %in, label %read, label %slow`,
`read:`,
` %k = fptoui double %i to ${sz}`,
` %data = getelementptr inbounds %ScrStr, ptr %s, i32 1`,
` %p = getelementptr inbounds i8, ptr %data, ${sz} %k`,
` %byte = load i8, ptr %p`,
` %unit = uitofp i8 %byte to double`,
` ret double %unit`,
`slow:`,
` %r = call double @scr_str_char_code_at(ptr %s, double %i)`,
` ret double %r`,
`}`,
],
},
];
@@ -81,7 +81,7 @@ test("string dispatch uses target-width byte lengths and checks equality inside
const body = emit(STRING, ["", "a", "é", "😀", "tail", "é"].map(string), bits);
expect(body).toContain(`load i${bits}, ptr`);
expect(body).toContain(`switch i${bits}`);
expect(body).not.toContain("scr_str_utf16_len");
expect(body).not.toContain("_str_utf16_len");
expect(body.match(/call zeroext i1 @sc_str_eq/g)).toHaveLength(5);
const dispatch = body.split("\n").find((line) => line.includes(`switch i${bits}`))!;
expect(dispatch).toContain(`i${bits} 2, label`);
@@ -15460,6 +15460,24 @@
"<repo>/tests/corpus/4453-int32-closed-locals.ts"
],
"diags": []
},
"<repo>/tests/corpus/4476-string-ascii-header-fact.ts": {
"order": [
"<repo>/tests/corpus/4476-string-ascii-header-fact.ts"
],
"diags": []
},
"<repo>/tests/corpus/4477-buffer-write-ascii-runs.ts": {
"order": [
"<repo>/tests/corpus/4477-buffer-write-ascii-runs.ts"
],
"diags": []
},
"<repo>/tests/corpus/4478-string-large-index-reads.ts": {
"order": [
"<repo>/tests/corpus/4478-string-large-index-reads.ts"
],
"diags": []
}
}
}
+21
View File
@@ -2736,7 +2736,28 @@ double scr_bytes_write_str(ScrBytes *b, const ScrStr *s, const ScrStr *enc,
bool utf16 = scr_enc_is(enc, "utf16le");
if (utf16 || scr_enc_is(enc, "latin1") || scr_enc_is(enc, "ascii")) {
size_t i = 0, written = 0, width = utf16 ? 2 : 1;
const uint64_t high = UINT64_C(0x8080808080808080);
while (i < s->len && budget - written >= width) {
/* ASCII runs encode byte-for-unit: widen (or copy) eight bytes per
* step while the budget holds them all, else one code point below. */
if (s->len - i >= 8 && budget - written >= 8 * width) {
uint64_t w;
memcpy(&w, s->data + i, 8);
if (!(w & high)) {
uint8_t *d = b->data + o + written;
if (utf16) {
for (int k = 0; k < 8; k++) {
d[2 * k] = (uint8_t)s->data[i + k];
d[2 * k + 1] = 0;
}
} else {
memcpy(d, s->data + i, 8);
}
i += 8;
written += 8 * width;
continue;
}
}
uint32_t cp = scr_bytes_next_cp((const uint8_t *)s->data, &i);
uint32_t first = cp, second = 0;
if (cp > 0xffff) {
+1
View File
@@ -416,6 +416,7 @@ typedef struct ScrRegexInput {
} ScrRegexInput;
static bool scr_text_ascii(const ScrStr *s) {
if (scr_str_known_ascii(s)) return true;
size_t i = 0;
for (; s->len - i >= sizeof(uint64_t); i += sizeof(uint64_t)) {
uint64_t word;
+83 -10
View File
@@ -649,40 +649,113 @@ typedef struct ScrVt {
* owned heap strings are ever mutated, and every such path clears the hash
* (scr_str_hash_forget); immortal strings are never written, so they always
* hash on demand.
*
* Bit 31 of the capacity word (on every target) is the proven-ASCII fact:
* when set, every byte is below 0x80, so UTF-16 indices equal byte offsets
* and .length == len. A clear bit means "not known", never "non-ASCII":
* constructors set it only where it is cheap, and string reads that prove
* ASCII set it later. Capacity therefore stops at 2^31 - 1 bytes, which
* still holds every string V8 can represent (2^29 - 24 UTF-16 units, at
* most three UTF-8 bytes each). Generated code reads the bit for inline
* charCodeAt/length (llvm/strings.ts); emitted literals and runtime
* statics set it in their capacity word. scr_str_hash_forget clears it
* with the hash: every in-place mutation already forgets cached facts.
*/
#if SIZE_MAX > UINT32_MAX && defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define SCR_STR_HASH_CACHE 1
#define SCR_STR_MAX_CAP ((size_t)UINT32_MAX)
#else
#define SCR_STR_HASH_CACHE 0
#define SCR_STR_MAX_CAP (SIZE_MAX - sizeof(size_t) * 3 - 1)
#endif
#define SCR_STR_MAX_CAP ((size_t)0x7fffffff)
typedef struct ScrStr {
size_t rc;
size_t len;
#if SCR_STR_HASH_CACHE
uint32_t cap;
uint32_t cap : 31;
uint32_t ascii : 1;
uint32_t hash;
#else
size_t cap;
size_t cap : 31;
size_t ascii : 1;
#endif
char data[];
} ScrStr;
/* The capacity word of a static string: cap == len, plus the precomputed
* 32-bit key hash where the header caches one (0 leaves it to be computed). */
/* The capacity word of a static string: cap == len, the proven-ASCII bit,
* plus the precomputed 32-bit key hash where the header caches one (0
* leaves it to be computed). */
#define SCR_STR_ASCII_BIT ((size_t)1 << 31)
#if SCR_STR_HASH_CACHE
#define SCR_STR_CAP_WORD(len, hash32) ((size_t)(len) | ((size_t)(uint32_t)(hash32) << 32))
#define SCR_STR_CAP_WORD(len, hash32, ascii) \
((size_t)(len) | ((ascii) ? SCR_STR_ASCII_BIT : 0) | \
((size_t)(uint32_t)(hash32) << 32))
#else
#define SCR_STR_CAP_WORD(len, hash32) ((size_t)(len))
#define SCR_STR_CAP_WORD(len, hash32, ascii) ((size_t)(len) | ((ascii) ? SCR_STR_ASCII_BIT : 0))
#endif
/* Forget every cached content fact (key hash, proven ASCII) before a
* uniquely owned string's bytes change. */
static inline void scr_str_hash_forget(ScrStr *s) {
#if SCR_STR_HASH_CACHE
s->hash = 0;
#else
(void)s;
#endif
s->ascii = 0;
}
/* True only when every byte is proven below 0x80 (see above). */
static inline bool scr_str_known_ascii(const ScrStr *s) { return s->ascii; }
/* Record a proven-ASCII fact. Immortal strings may live in read-only
* memory and are shared across threads, so they are never written. */
static inline void scr_str_note_ascii(ScrStr *s) {
if (s->rc != SIZE_MAX) s->ascii = 1;
}
/* memcpy for the short, variable-length copies of string construction
* (identifiers, keys, number text): fixed-size overlapping moves instead of
* a libc call. Longer spans keep the platform's bulk copy. */
static inline void scr_copy_bytes(char *dst, const char *src, size_t n) {
if (n <= 16) {
if (n >= 8) {
uint64_t a, b;
memcpy(&a, src, 8);
memcpy(&b, src + n - 8, 8);
memcpy(dst, &a, 8);
memcpy(dst + n - 8, &b, 8);
} else if (n >= 4) {
uint32_t a, b;
memcpy(&a, src, 4);
memcpy(&b, src + n - 4, 4);
memcpy(dst, &a, 4);
memcpy(dst + n - 4, &b, 4);
} else if (n) {
dst[0] = src[0];
dst[n / 2] = src[n / 2];
dst[n - 1] = src[n - 1];
}
return;
}
memcpy(dst, src, n);
}
/* Whether a byte span is all ASCII: 32 bytes per step, stopping at the
* first chunk with a high bit. */
static inline bool scr_bytes_all_ascii(const char *p, size_t n) {
const uint64_t high = UINT64_C(0x8080808080808080);
size_t i = 0;
for (; n - i >= 32; i += 32) {
uint64_t w[4];
memcpy(w, p + i, sizeof w);
if ((w[0] | w[1] | w[2] | w[3]) & high) return false;
}
uint64_t acc = 0;
for (; n - i >= 8; i += 8) {
uint64_t w;
memcpy(&w, p + i, 8);
acc |= w;
}
for (; i < n; i++) acc |= (unsigned char)p[i];
return (acc & high) == 0;
}
ScrStr *scr_str_new(const char *bytes, size_t len); /* returns +1 */
+165 -38
View File
@@ -200,7 +200,9 @@ static ScrSidx *scr_sidx_claim(ScrSidx *table, unsigned *clock, unsigned *live)
* receivers still shed their point buffer after proving identity mapping.
* Both tiers remain fixed-size and allocation-free until a non-ASCII sparse
* receiver actually needs checkpoints. */
static ScrSidx *scr_sidx(const ScrStr *s) {
static SCR_TL ScrSidx *scr_sidx_last; /* most recent hit; a slot, never freed */
static ScrSidx *scr_sidx_lookup(const ScrStr *s) {
if (s->len >= SCR_SIDX_MIN_BYTES) {
for (int i = 0; i < SCR_SIDX_N; i++) {
if (scr_sidx_sparse_tab[i].s == s) return &scr_sidx_sparse_tab[i];
@@ -234,6 +236,27 @@ static ScrSidx *scr_sidx(const ScrStr *s) {
return e;
}
/* Sequential reads of one receiver (a scan over a source text) hit the
* same entry call after call. The memo is a pointer to a static slot, so a
* cleared, moved or reassigned entry simply fails the owner test. */
static ScrSidx *scr_sidx(const ScrStr *s) {
ScrSidx *last = scr_sidx_last;
/* An in-place append can carry a cursor-tier receiver across the sparse
* threshold; only the lookup performs that transfer, so the memo must
* also match the receiver's current tier. */
bool sparse = (uintptr_t)last - (uintptr_t)scr_sidx_sparse_tab < sizeof scr_sidx_sparse_tab;
if (last && last->s == s && sparse == (s->len >= SCR_SIDX_MIN_BYTES)) return last;
return scr_sidx_last = scr_sidx_lookup(s);
}
/* The index entry for a receiver that is not proven ASCII, or NULL when it
* is: proven-ASCII strings map units to bytes by identity, and every mapper
* below checks the header bit before touching the entry. The bit only ever
* turns on during a read, so "ascii or entry" holds for the whole call. */
static ScrSidx *scr_sidx_for(const ScrStr *s) {
return s->ascii ? NULL : scr_sidx(s);
}
/* In-place concat changes only the suffix. Keep every exact prefix anchor
* (including the former end, which is now an ordinary boundary), but remove
* the sole fact that described the old complete string. A threshold-crossing
@@ -268,10 +291,57 @@ static ScrStr *scr_str_alloc(size_t len, size_t cap) {
return s;
}
/* scr_copy_bytes that also reports whether the bytes are all ASCII. Short
* spans test the words they already moved; longer ones scan after the bulk
* copy, stopping at the first non-ASCII chunk. */
static bool scr_copy_bytes_ascii(char *dst, const char *src, size_t n) {
const uint64_t high = UINT64_C(0x8080808080808080);
if (n <= 16) {
if (n >= 8) {
uint64_t a, b;
memcpy(&a, src, 8);
memcpy(&b, src + n - 8, 8);
memcpy(dst, &a, 8);
memcpy(dst + n - 8, &b, 8);
return ((a | b) & high) == 0;
}
if (n >= 4) {
uint32_t a, b;
memcpy(&a, src, 4);
memcpy(&b, src + n - 4, 4);
memcpy(dst, &a, 4);
memcpy(dst + n - 4, &b, 4);
return ((a | b) & UINT32_C(0x80808080)) == 0;
}
if (n == 0) return true;
unsigned char c0 = (unsigned char)src[0], c1 = (unsigned char)src[n / 2],
c2 = (unsigned char)src[n - 1];
dst[0] = (char)c0;
dst[n / 2] = (char)c1;
dst[n - 1] = (char)c2;
return ((c0 | c1 | c2) & 0x80) == 0;
}
memcpy(dst, src, n);
return scr_bytes_all_ascii(src, n);
}
ScrStr *scr_str_new(const char *bytes, size_t len) {
ScrStr *s = scr_str_alloc(len, len);
memcpy(s->data, bytes, len);
/* Proving ASCII while copying lets every later UTF-16 index, length and
* slice skip the index cache. */
bool ascii = scr_copy_bytes_ascii(s->data, bytes, len);
s->data[len] = '\0';
s->ascii = ascii;
return s;
}
/* scr_str_new for bytes already known to be ASCII (slices of an ASCII
* string): no scan. */
static ScrStr *scr_str_new_ascii(const char *bytes, size_t len) {
ScrStr *s = scr_str_alloc(len, len);
scr_copy_bytes(s->data, bytes, len);
s->data[len] = '\0';
s->ascii = 1;
return s;
}
@@ -328,7 +398,9 @@ void scr_str_release(ScrStr *s) {
if (!s || s->rc == SIZE_MAX) return; /* NULL: an uninitialized `let` local */
if (--s->rc == 0) {
scr_short_forget(s);
scr_sidx_purge(s); /* the address may be recycled by the next malloc */
/* The address may be recycled by the next malloc. A proven-ASCII string
* owns no index entry (scr_sidx_finish). */
if (!s->ascii) scr_sidx_purge(s);
#ifdef SCR_RC_AUDIT
scr_live_strings--;
#endif
@@ -354,20 +426,25 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) {
* concat chains (`a + b + c`, template literals), where each intermediate
* result reaches the next concat as a sole-reference temp. Any string
* with rc > 1 might be aliased and is copied, never mutated. */
bool ascii = a->ascii && b->ascii;
if (a->rc == 1 && a != b && a->cap >= newlen) {
size_t oldlen = a->len;
scr_short_forget(a);
scr_str_hash_forget(a);
memcpy(a->data + a->len, b->data, b->len);
scr_copy_bytes(a->data + a->len, b->data, b->len);
a->len = newlen;
a->data[newlen] = '\0';
a->ascii = ascii;
/* A cached UTF-16 length for a is stale now; checkpoints and the exact
* old prefix remain valid. Its old terminal point is no longer an END
* fact (u16len is invalidated below), but remains an excellent ordinary
* checkpoint for accesses around the append boundary. The next mapper
* lazily continues from oldlen rather than scanning the unchanged prefix
* again. */
scr_sidx_concat_append(a, oldlen);
* again. An ASCII string that stays ASCII needs none of this: every
* mapper answers it by identity before consulting an entry, and the
* append that first adds non-ASCII bytes invalidates against its own
* old length, which bounds every older fact as well. */
if (!ascii) scr_sidx_concat_append(a, oldlen);
a->rc = 2; /* +1 for the returned reference, beside the caller's borrow */
return a;
}
@@ -386,9 +463,10 @@ ScrStr *scr_str_concat(ScrStr *a, ScrStr *b) {
if (newcap > SCR_STR_MAX_CAP && newlen <= SCR_STR_MAX_CAP) newcap = SCR_STR_MAX_CAP;
ScrStr *s = scr_str_take_spare(newlen);
if (!s) s = scr_str_alloc(newlen, newcap);
memcpy(s->data, a->data, a->len);
memcpy(s->data + a->len, b->data, b->len);
scr_copy_bytes(s->data, a->data, a->len);
scr_copy_bytes(s->data + a->len, b->data, b->len);
s->data[newlen] = '\0';
s->ascii = ascii;
return s;
}
@@ -406,10 +484,12 @@ ScrStr *scr_str_concat_mixed(ScrStr *head, ScrStr *const *parts, const double *n
const size_t limit = SCR_STR_MAX_CAP;
size_t extra = 0;
bool aliased = false;
bool ascii = !head || head->ascii; /* formatted numbers are ASCII */
for (size_t i = 0; i < count; i++) {
if (parts[i]) {
lens[i] = parts[i]->len;
aliased |= parts[i] == head;
ascii &= parts[i]->ascii;
} else {
lens[i] = scr_f64_to_str(nums[i], text[i]);
}
@@ -439,17 +519,18 @@ ScrStr *scr_str_concat_mixed(ScrStr *head, ScrStr *const *parts, const double *n
if (newcap > SCR_STR_MAX_CAP) newcap = SCR_STR_MAX_CAP;
out = scr_str_take_spare(newlen);
if (!out) out = scr_str_alloc(newlen, newcap);
if (base) memcpy(out->data, head->data, base);
if (base) scr_copy_bytes(out->data, head->data, base);
at = base;
}
for (size_t i = 0; i < count; i++) {
memcpy(out->data + at, parts[i] ? parts[i]->data : text[i], lens[i]);
scr_copy_bytes(out->data + at, parts[i] ? parts[i]->data : text[i], lens[i]);
at += lens[i];
}
out->len = newlen;
out->data[newlen] = '\0';
out->ascii = ascii;
if (out == head) {
scr_sidx_concat_append(head, base);
if (!ascii) scr_sidx_concat_append(head, base); /* see scr_str_concat */
head->rc = 2; /* +1 for the returned reference, beside the caller's borrow */
}
return out;
@@ -477,12 +558,15 @@ ScrStr *scr_str_concat_parts(ScrStr *const *parts, size_t count) {
if (nonempty == 1) return scr_str_retain(only);
ScrStr *out = scr_str_alloc_raw(len, len);
size_t offset = 0;
bool ascii = true;
for (size_t i = 0; i < count; i++) {
const ScrStr *part = parts[i];
memcpy(out->data + offset, part->data, part->len);
scr_copy_bytes(out->data + offset, part->data, part->len);
offset += part->len;
ascii &= part->ascii;
}
out->data[len] = '\0';
out->ascii = ascii;
return out;
}
@@ -529,7 +613,7 @@ int scr_str_cmp_u16(ScrStr *a, ScrStr *b) {
*/
typedef struct { size_t rc; size_t len; size_t cap; char data[2]; } ScrChar1;
#define SCR_A_WORD(c) (((uint64_t)(c) << 16) | ((uint64_t)(c) << 8) | (uint64_t)(c))
#define SCR_A(c) {SIZE_MAX, 1, SCR_STR_CAP_WORD(1, SCR_KEY_HASH32_SHORT(1, SCR_A_WORD(c))), {(char)(c), 0}}
#define SCR_A(c) {SIZE_MAX, 1, SCR_STR_CAP_WORD(1, SCR_KEY_HASH32_SHORT(1, SCR_A_WORD(c)), 1), {(char)(c), 0}}
#define SCR_A8(c) \
SCR_A(c), SCR_A(c + 1), SCR_A(c + 2), SCR_A(c + 3), \
SCR_A(c + 4), SCR_A(c + 5), SCR_A(c + 6), SCR_A(c + 7)
@@ -540,7 +624,7 @@ static const ScrChar1 scr_ascii1[128] = {
SCR_A8(96), SCR_A8(104), SCR_A8(112), SCR_A8(120),
};
static const struct { size_t rc; size_t len; size_t cap; char data[1]; }
scr_lit_empty = {SIZE_MAX, 0, SCR_STR_CAP_WORD(0, SCR_KEY_HASH32_SHORT(0, 0)), ""};
scr_lit_empty = {SIZE_MAX, 0, SCR_STR_CAP_WORD(0, SCR_KEY_HASH32_SHORT(0, 0), 1), ""};
static ScrStr *scr_str_empty(void) { return (ScrStr *)&scr_lit_empty; }
@@ -562,6 +646,13 @@ static ScrStr *scr_str_from_span(const char *bytes, size_t len) {
return scr_str_new(bytes, len);
}
/* scr_str_from_span for a span of a proven-ASCII string: the result is
* ASCII without another scan. */
static ScrStr *scr_str_from_span_ascii(const char *bytes, size_t len) {
if (len <= 4) return scr_str_from_span(bytes, len);
return scr_str_new_ascii(bytes, len);
}
/* A scalar fromCharCode needs neither the variadic argument array nor an
* encoding buffer. Reuse the same empty/ASCII/tiny-span storage policy as
* character indexing. A lone surrogate keeps the existing U+FFFD policy. */
@@ -790,10 +881,21 @@ static void scr_sidx_rebuild_points(const ScrStr *s, ScrSidx *e) {
e->points_complete = true;
}
/* Completing a scan that proves a heap string ASCII records the fact in its
* header and drops the entry: a proven-ASCII string never owns one (every
* mapper answers it by identity first, and a release of it skips the
* purge). Callers re-check the header bit before touching the entry again.
* Immortal strings cannot record the bit and keep the entry. */
static void scr_sidx_finish(const ScrStr *s, ScrSidx *e) {
if (e->indexed_cb != s->len) return;
e->u16len = e->indexed_cu;
if (e->u16len == s->len) { /* proven all ASCII: identity needs no index */
if (s->rc != SIZE_MAX) {
((ScrStr *)s)->ascii = 1;
bool sparse = (uintptr_t)e - (uintptr_t)scr_sidx_sparse_tab < sizeof scr_sidx_sparse_tab;
scr_sidx_clear(e, sparse ? &scr_sidx_sparse_live : NULL);
return;
}
free(e->points);
e->points = NULL;
e->npoints = 0;
@@ -837,10 +939,11 @@ static void scr_sidx_materialize_identity_prefix(const ScrStr *s, ScrSidx *e) {
* historical single word-wise scan. `u16len == byte len` proves ASCII and
* restores identity mapping with zero retained checkpoint memory. */
static size_t scr_sidx_len(const ScrStr *s, ScrSidx *e) {
if (s->ascii) return s->len;
if (e->u16len != SCR_U16_UNKNOWN) return e->u16len;
while (e->indexed_cb < s->len) scr_sidx_extend_one(s, e);
scr_sidx_finish(s, e);
return e->u16len;
return s->ascii ? s->len : e->u16len;
}
/* Step the cursor back one char (cb must be > 0 and on a boundary). */
@@ -970,7 +1073,7 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) {
* Same contract as a from-scratch scan. */
static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16,
bool *mid) {
if (e->u16len == s->len) { /* all ASCII: identity mapping */
if (s->ascii || e->u16len == s->len) { /* all ASCII: identity mapping */
*mid = false;
return u16 < s->len ? u16 : s->len;
}
@@ -983,7 +1086,7 @@ static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16,
scr_sidx_extend_to_u16(s, e, u16);
/* Extending a far-end lookup can just have proved identity. Do not
* materialize an index that the identity fast path will never consult. */
if (e->u16len == s->len) {
if (s->ascii || e->u16len == s->len) {
*mid = false;
return u16 < s->len ? u16 : s->len;
}
@@ -1037,7 +1140,7 @@ static size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16,
* the closest sparse anchor or hot cursor. */
static size_t scr_byte_to_u16_c(const ScrStr *s, ScrSidx *e,
size_t byte_off) {
if (e->u16len == s->len) return byte_off; /* all ASCII */
if (s->ascii || e->u16len == s->len) return byte_off; /* all ASCII */
if (byte_off >= e->ascii_cb && byte_off < e->ascii_end) {
e->cu = e->ascii_cu + (byte_off - e->ascii_cb);
e->cb = byte_off;
@@ -1051,7 +1154,7 @@ static size_t scr_byte_to_u16_c(const ScrStr *s, ScrSidx *e,
}
/* As above, a completed all-ASCII scan is its own index. In particular,
* do not rebuild points that scr_sidx_finish() deliberately discarded. */
if (e->u16len == s->len) return byte_off;
if (s->ascii || e->u16len == s->len) return byte_off;
scr_sidx_materialize_identity_prefix(s, e);
ScrSidxPoint near = scr_sidx_near_byte(s, e, byte_off);
size_t cu = near.cu, cb = near.cb;
@@ -1098,7 +1201,7 @@ static const char *scr_byte_find(const char *hay, size_t hay_len,
/* Search backwards from a clamped UTF-16 position. A low-surrogate position
* maps to its scalar's first byte, so that scalar remains searchable. */
double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) {
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
size_t len16 = scr_sidx_len(s, e);
size_t start16 = isnan(position) || position >= (double)len16 ? len16
: position <= 0 ? 0 : (size_t)trunc(position);
@@ -1121,16 +1224,32 @@ double scr_str_last_index_of(ScrStr *s, ScrStr *needle) {
}
double scr_str_utf16_len(ScrStr *s) {
if (s->ascii) return (double)s->len;
return (double)scr_sidx_len(s, scr_sidx(s));
}
double scr_str_char_code_at(ScrStr *s, double i) {
/* A sequential scan of a non-ASCII receiver stays inside the bounded
* ASCII span cached at its last resolved position. Its unit offset is an
* integer, so trunc(i) - ascii_cu == trunc(i - ascii_cu) for i at or past
* it; NaN and anything before the span fail the comparison. */
ScrSidx *last = scr_sidx_last;
if (last && last->s == s && !s->ascii) {
double rel = i - (double)last->ascii_cu;
if (rel >= 0 && rel < (double)(last->ascii_end - last->ascii_cb)) {
size_t k = (size_t)rel;
last->cu = last->ascii_cu + k;
last->cb = last->ascii_cb + k;
return (double)(unsigned char)s->data[last->cb];
}
}
double idx = scr_to_integer_or_infinity(i);
if (!(idx >= 0)) return NAN; /* negative or -Infinity */
/* UTF-16 length <= byte length always, so this also fences the cast. */
if (idx >= (double)s->len) return NAN;
if (s->ascii) return (double)(unsigned char)s->data[(size_t)idx];
ScrSidx *e = scr_sidx(s);
if (e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx];
if (s->ascii || e->u16len == s->len) return (double)(unsigned char)s->data[(size_t)idx];
bool mid;
size_t off = scr_u16_to_byte_c(s, e, (size_t)idx, &mid);
if (off >= s->len) return NAN; /* idx >= length */
@@ -1144,7 +1263,7 @@ double scr_str_char_code_at(ScrStr *s, double i) {
double scr_str_index_of(ScrStr *s, ScrStr *needle, double fromIndex) {
double pos = scr_to_integer_or_infinity(fromIndex);
if (needle->len > s->len) return -1.0;
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
/* Per spec, the empty needle is found at the clamped fromIndex itself —
* even when that index is between the halves of an astral pair. */
if (needle->len == 0) {
@@ -1173,7 +1292,7 @@ static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e,
/* Normalize once, then share slice's extraction without another cache
* lookup, length query or floating-point boundary conversion. */
ScrStr *scr_str_substring(ScrStr *s, double start, double end) {
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
double len16 = (double)scr_sidx_len(s, e);
double a = scr_to_integer_or_infinity(start);
double b = scr_to_integer_or_infinity(end);
@@ -1208,7 +1327,7 @@ bool scr_str_starts_with_from(ScrStr *s, ScrStr *needle, double position) {
double pos = scr_to_integer_or_infinity(position);
if (pos <= 0) return scr_str_starts_with(s, needle);
if (pos >= (double)s->len || needle->len > s->len) return false;
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
bool mid;
size_t start_byte = scr_u16_to_byte_c(s, e, (size_t)pos, &mid);
return !mid && needle->len <= s->len - start_byte &&
@@ -1221,8 +1340,8 @@ bool scr_str_ends_with_from(ScrStr *s, ScrStr *needle, double end_position) {
if (end_position >= (double)s->len) return scr_str_ends_with(s, needle);
/* A second receiver lookup can evict the first receiver's entry. Resolve
* the needle before borrowing the haystack entry for both conversions. */
size_t needle16 = scr_sidx_len(needle, scr_sidx(needle));
ScrSidx *e = scr_sidx(s);
size_t needle16 = scr_sidx_len(needle, scr_sidx_for(needle));
ScrSidx *e = scr_sidx_for(s);
size_t len16 = scr_sidx_len(s, e);
size_t end16 = scr_str_clamp_u16_position(end_position, len16);
if (needle16 > end16) return false;
@@ -1245,7 +1364,7 @@ static size_t scr_slice_boundary(double v, size_t len16) {
}
ScrStr *scr_str_slice(ScrStr *s, double start, double end) {
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
size_t len16 = scr_sidx_len(s, e);
size_t from = scr_slice_boundary(scr_to_integer_or_infinity(start), len16);
size_t to = scr_slice_boundary(scr_to_integer_or_infinity(end), len16);
@@ -1254,7 +1373,7 @@ ScrStr *scr_str_slice(ScrStr *s, double start, double end) {
void scr_str_slice_range(ScrStr *s, double start, double end, bool substring,
ScrStringSlice *out) {
ScrSidx *e = scr_sidx(s);
ScrSidx *e = scr_sidx_for(s);
size_t length = scr_sidx_len(s, e);
double a = scr_to_integer_or_infinity(start), b = scr_to_integer_or_infinity(end);
size_t from, to;
@@ -1288,6 +1407,11 @@ double scr_str_slice_char_code_at(ScrStr *s, const ScrStringSlice *range, double
static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e,
size_t from, size_t to) {
if (from >= to) return scr_str_empty();
if (s->ascii) {
/* Identity mapping; callers clamp both bounds to the length. */
if (from == 0 && to == s->len) return scr_str_retain(s);
return scr_str_from_span_ascii(s->data + from, to - from);
}
if (from == 0 && to == e->u16len) return scr_str_retain(s);
bool from_mid, to_mid;
@@ -1303,7 +1427,8 @@ static ScrStr *scr_str_slice_units(ScrStr *s, ScrSidx *e,
/* Unit boundaries already prove the exact output length. Preserve it
* instead of decoding a potentially megabyte-sized copy on .length.
* Index checkpoints remain lazy and owned by the result's cache slot. */
if (content_len >= SCR_SIDX_MIN_BYTES) scr_sidx(result)->u16len = to - from;
if (content_len >= SCR_SIDX_MIN_BYTES && !result->ascii)
scr_sidx(result)->u16len = to - from;
return result;
}
@@ -1350,6 +1475,7 @@ ScrStr *scr_str_repeat(ScrStr *s, double count) {
filled += chunk;
}
r->data[total] = '\0';
r->ascii = s->ascii;
return r;
}
@@ -1388,8 +1514,9 @@ ScrStr *scr_str_char_at(ScrStr *s, double i) {
double idx = scr_to_integer_or_infinity(i);
if (!(idx >= 0)) return scr_str_empty();
if (idx >= (double)s->len) return scr_str_empty(); /* len16 <= len */
if (s->ascii) return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]];
ScrSidx *e = scr_sidx(s);
if (e->u16len == s->len) { /* all ASCII */
if (s->ascii || e->u16len == s->len) { /* all ASCII */
return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]];
}
bool mid;
@@ -1434,7 +1561,7 @@ ScrStr *scr_str_trim_end(ScrStr *s) {
* lone surrogate (empty-separator split of an astral char, a pad fill
* truncated mid-pair). */
static const struct { size_t rc; size_t len; size_t cap; char data[4]; }
scr_lit_fffd = {SIZE_MAX, 3, SCR_STR_CAP_WORD(3, SCR_KEY_HASH32_SHORT(3, 0xEFBFBD)), "\xEF\xBF\xBD"};
scr_lit_fffd = {SIZE_MAX, 3, SCR_STR_CAP_WORD(3, SCR_KEY_HASH32_SHORT(3, 0xEFBFBD), 0), "\xEF\xBF\xBD"};
/* Split owns a fresh array and appends only present string values. Fill an
* available dense slot directly; ordinary push handles growth and the sparse
@@ -1583,15 +1710,14 @@ ScrArr *scr_str_split(ScrStr *s, ScrStr *sep) {
static ScrStr *scr_pad_impl(ScrStr *s, double maxLength, ScrStr *fill,
bool at_start) {
double target = scr_to_integer_or_infinity(maxLength);
ScrSidx *e = scr_sidx(s);
size_t len16 = scr_sidx_len(s, e);
size_t len16 = scr_sidx_len(s, scr_sidx_for(s));
if (!(target > (double)len16) || fill->len == 0) return scr_str_retain(s);
/* Reject pad sizes malloc could not satisfy before the double→size_t
* conversion can overflow (each unit is at most 3 bytes here: BMP chars
* and the U+FFFD stand-in; astral chars are 4 bytes for 2 units). */
if (target > (double)((SIZE_MAX - sizeof(ScrStr) - 1) / 4)) scr_oom();
size_t pad16 = (size_t)target - len16;
size_t fill16 = scr_sidx_len(fill, scr_sidx(fill));
size_t fill16 = scr_sidx_len(fill, scr_sidx_for(fill));
size_t reps = pad16 / fill16, rem16 = pad16 % fill16;
/* The truncated prefix of fill: whole chars while they fit in rem16
* units; a final astral char that doesn't fit contributes U+FFFD. */
@@ -1936,7 +2062,7 @@ double scr_string_to_number(ScrStr *s) {
ScrStr *scr_f64_to_scrstr(double x) {
char buf[32];
size_t len = scr_f64_to_str(x, buf);
return scr_str_new(buf, len);
return scr_str_new_ascii(buf, len); /* number text is always ASCII */
}
/* ── encodeURIComponent / encodeURI ───────────────────────────────────
@@ -2011,9 +2137,9 @@ ScrStr *scr_str_to_well_formed(ScrStr *s) { return scr_str_retain(s); }
/* Immortal interned booleans (same layout trick the emitter uses for
* string literals). */
static const struct { size_t rc; size_t len; size_t cap; char data[5]; }
scr_lit_true = {SIZE_MAX, 4, 4, "true"};
scr_lit_true = {SIZE_MAX, 4, SCR_STR_CAP_WORD(4, 0, 1), "true"};
static const struct { size_t rc; size_t len; size_t cap; char data[6]; }
scr_lit_false = {SIZE_MAX, 5, 5, "false"};
scr_lit_false = {SIZE_MAX, 5, SCR_STR_CAP_WORD(5, 0, 1), "false"};
ScrStr *scr_bool_to_scrstr(bool b) {
return b ? (ScrStr *)&scr_lit_true : (ScrStr *)&scr_lit_false;
@@ -2032,8 +2158,9 @@ ScrStr *scr_str_cp_at(ScrStr *s, double i) {
double idx = scr_to_integer_or_infinity(i);
if (!(idx >= 0)) return scr_str_empty();
if (idx >= (double)s->len) return scr_str_empty(); /* len16 <= len */
if (s->ascii) return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]];
ScrSidx *e = scr_sidx(s);
if (e->u16len == s->len) { /* all ASCII */
if (s->ascii || e->u16len == s->len) { /* all ASCII */
return (ScrStr *)&scr_ascii1[(unsigned char)s->data[(size_t)idx]];
}
bool mid;
@@ -0,0 +1,117 @@
// UTF-16 reads (length, charCodeAt, charAt, slice, substring, indexOf,
// lastIndexOf, startsWith/endsWith at a position, padStart) over strings of
// every provenance: literals, concatenations, in-place appends, slices,
// numbers, repeat, join and split pieces. Native strings record a
// proven-ASCII fact in their header; appending a non-ASCII character to a
// uniquely owned ASCII string must drop it, and every index form
// (fractional, -0, negative, NaN, infinite, out of range) keeps Node's
// semantics on both representations, including surrogate halves. (Reads
// never split a surrogate pair into a string: scriptc substitutes U+FFFD.)
function codes(s: string): string {
const out: number[] = [];
for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i));
return out.join(",");
}
function probe(label: string, s: string): void {
const idx = [0, -0, 0.5, 1.9999, -0.5, -1, NaN, Infinity, -Infinity, s.length - 1, s.length, 2 ** 32];
const units: string[] = [];
for (const i of idx) units.push(String(s.charCodeAt(i)));
const chars: string[] = [];
for (const i of idx) chars.push(JSON.stringify(s.charAt(i)));
console.log(label, s.length, units.join(" "), chars.join(" "));
console.log(
" slice",
JSON.stringify(s.slice(1, -1)),
JSON.stringify(s.slice(-3)),
JSON.stringify(s.substring(5, 2)),
JSON.stringify(s.slice(2, 2)),
);
console.log(
" search",
s.indexOf("b"),
s.indexOf("b", 3),
s.lastIndexOf("b"),
s.indexOf(""),
s.indexOf("", 99),
s.startsWith("b", 1),
s.endsWith("b", 2),
);
console.log(" pad", JSON.stringify(s.padStart(s.length + 3, "-")), JSON.stringify(s.padEnd(s.length + 2, "\u00e9")));
}
// Literals: ASCII, BMP and astral.
probe("lit-ascii", "abcabc");
probe("lit-bmp", "ab\u00e9cb\u00e9");
probe("lit-astral", "ab\u{1F600}cbd");
// Concatenations of every pairing.
const n = 7;
const left = "ab" + String(n);
const right = "\u00e9b" + String(n);
probe("cat-aa", left + "cb");
probe("cat-an", left + right);
probe("cat-na", right + left);
probe("cat-astral", left + "\u{1F600}b");
// Uniquely owned strings appended in place after an ASCII read.
let grow = "";
for (let i = 0; i < 6; i++) grow += String.fromCharCode(97 + i);
console.log("grow", grow.length, grow.charCodeAt(3), codes(grow));
grow += "\u00e9";
console.log("grow+bmp", grow.length, grow.charCodeAt(6), grow.charCodeAt(7), codes(grow));
grow += "\u{1F600}b";
console.log("grow+astral", grow.length, grow.charCodeAt(7), grow.charCodeAt(8), grow.charCodeAt(9), codes(grow));
probe("grow", grow);
let chain = "x";
for (let i = 0; i < 40; i++) {
chain += i % 13 === 12 ? "\u03a9" : String(i % 10);
if (chain.charCodeAt(chain.length - 1) > 127) console.log("non-ascii at", chain.length - 1, chain.charCodeAt(chain.length - 1));
}
console.log("chain", chain.length, chain.indexOf("\u03a9"), chain.lastIndexOf("\u03a9"), chain.slice(10, 16));
// Template literals mixing numbers and strings.
const id = 42;
const key = `${id},${id + 1}|${right}`;
probe("template", key);
// Slices: ASCII slices of non-ASCII strings and the reverse.
const mixed = "\u00e9abcb\u00e9xyz\u{1F600}!";
const inner = mixed.slice(1, 5);
probe("slice-ascii-of-mixed", inner);
probe("slice-mixed", mixed.slice(0, 7));
probe("substring-astral", mixed.substring(7, 13));
console.log("astral halves", mixed.charCodeAt(9), mixed.charCodeAt(10), mixed.slice(9, 11) === "\u{1F600}");
// Numbers, repeat, join, split pieces.
probe("number", String(-1234.5e-7) + "b");
probe("repeat", "ab".repeat(4));
probe("repeat-bmp", "a\u00e9".repeat(3));
probe("join", ["ab", "cb", "d"].join(""));
probe("join-bmp", ["ab", "\u00e9", "cb"].join("-"));
const pieces = "abcdefgh,b\u00e9cdefgh,abcdefgb".split(",");
for (const p of pieces) console.log("piece", p.length, p.charCodeAt(1), p.charCodeAt(p.length - 1), codes(p));
// Builder-made strings (join, JSON.stringify) are proven ASCII by their
// first read, then appended in place with non-ASCII text and read again.
let built = ["ab", "cd", "ef", String(n)].join("");
console.log("built", built.length, built.charCodeAt(4), built.indexOf("7"), built.slice(2, 5));
built += "\u00e9" + built;
console.log("built+bmp", built.length, built.charCodeAt(7), built.charCodeAt(8), built.lastIndexOf("e"), built.slice(5, 9));
let json = JSON.stringify({ k: [1, 2, 3], s: "x" });
console.log("json", json.length, json.charCodeAt(2), json.indexOf("s"));
json += "\u{1F600}";
console.log("json+astral", json.length, json.charCodeAt(json.length - 2), json.charCodeAt(json.length - 1), json.slice(-4));
// Interleaved reads of several receivers (index cache residency).
const a1 = "alpha\u00e9" + String(n);
const a2 = "beta" + String(n);
const a3 = "\u{1F600}gamma";
let mix = 0;
for (let i = 0; i < 8; i++) {
mix = mix * 31 + a1.charCodeAt(i % a1.length) + a2.charCodeAt(i % a2.length) * 3 + a3.charCodeAt(i % a3.length) * 7;
mix %= 1000003;
}
console.log("interleaved", mix);
@@ -0,0 +1,55 @@
// Buffer#write with utf16le, latin1 and ascii encodings: long ASCII runs,
// runs broken by BMP and astral characters, writes truncated by an offset
// and length budget (including odd budgets that split a UTF-16 unit or a
// surrogate pair), and the Uint16Array view a scanner builds from text.
function hex(buf: Buffer): string {
return buf.toString("hex");
}
const texts = [
"",
"abc",
"abcdefgh",
"abcdefghijklmnopq",
"const x = 1;\nconst y = 2;\n",
"abcdefg\u00e9hijklmnopqrstu",
"\u00e9abcdefghijklmnop",
"abcdefghijklmn\u{1F600}opqrstuvwxyz0123",
"\u4e2d\u6587abcdefghijklmnopqrstuvwxyz",
];
function show(enc: string, text: string, n: number, buf: Buffer): void {
console.log(enc, text.length, n, hex(buf.subarray(0, n)));
}
for (const text of texts) {
const size = text.length * 2 + 4;
const u16 = Buffer.alloc(size);
show("utf16le", text, u16.write(text, "utf16le"), u16);
const l1 = Buffer.alloc(size);
show("latin1", text, l1.write(text, "latin1"), l1);
const as = Buffer.alloc(size);
show("ascii", text, as.write(text, "ascii"), as);
for (const budget of [1, 3, 7, 15, 16, 17, 33]) {
const a = Buffer.alloc(40);
const b = Buffer.alloc(40);
const c = Buffer.alloc(40);
const wa = a.write(text, 2, budget, "utf16le");
const wb = b.write(text, 2, budget, "latin1");
const wc = c.write(text, 2, budget, "ascii");
console.log(" budget", budget, wa, hex(a.subarray(0, 2 + wa)), wb, hex(b.subarray(0, 2 + wb)), wc, hex(c.subarray(0, 2 + wc)));
}
}
function toCodeUnits(text: string): Uint16Array {
const chars = new Uint16Array(text.length + 4);
Buffer.from(chars.buffer).write(text, "utf16le");
return chars;
}
const source = "function f(a) {\n return a + 1; // caf\u00e9 \u{1F600}\n}\n".repeat(40);
const units = toCodeUnits(source);
let ok = units.length === source.length + 4;
for (let i = 0; i < source.length; i++) if (units[i] !== source.charCodeAt(i)) ok = false;
console.log("code units", units.length, ok, units[source.length], units[source.length + 3]);
@@ -0,0 +1,85 @@
// Sequential and random UTF-16 reads over large strings (well past 64 KiB):
// an all-ASCII text, a text with sparse non-ASCII and astral characters, a
// line-start scan in the style of a compiler scanner, alternating scans of
// two large receivers, and a uniquely owned large ASCII string that turns
// non-ASCII through an in-place append. Every result must match Node.
function lineStarts(text: string): number[] {
const result: number[] = [];
let lineStart = 0;
let pos = 0;
while (pos < text.length) {
const ch = text.charCodeAt(pos);
pos++;
if (ch === 13) {
if (pos < text.length && text.charCodeAt(pos) === 10) pos++;
} else if (ch !== 10 && ch !== 0x2028 && ch !== 0x2029) {
continue;
}
result.push(lineStart);
lineStart = pos;
}
result.push(lineStart);
return result;
}
function summary(label: string, text: string): void {
const starts = lineStarts(text);
let sum = 0;
for (let i = 0; i < text.length; i += 997) sum = (sum * 33 + text.charCodeAt(i)) % 1000000007;
let back = 0;
for (let i = text.length - 1; i >= 0; i -= 1009) back = (back * 31 + text.charCodeAt(i)) % 1000000007;
console.log(label, text.length, starts.length, starts[starts.length - 1], starts[1000] ?? -1, sum, back);
}
const lines: string[] = [];
for (let i = 0; i < 6000; i++) lines.push(`const value${i} = compute(${i}, "text") + ${i * 3};`);
const ascii = lines.join("\n");
summary("ascii", ascii);
const mixedLines: string[] = [];
for (let i = 0; i < 6000; i++) {
const note = i % 97 === 0 ? " // caf\u00e9 \u{1F600}" : i % 501 === 0 ? "\u2028" : "";
mixedLines.push(`let item${i} = [${i}, ${i + 1}];${note}`);
}
const mixed = mixedLines.join(i_crlf());
summary("mixed", mixed);
function i_crlf(): string {
return "\r\n";
}
// Alternate between the two large receivers and a short one.
let alt = 0;
const short = "s\u00e9q";
for (let i = 0; i < 20000; i += 7) {
alt = (alt * 7 + ascii.charCodeAt(i) + mixed.charCodeAt(i) * 3 + short.charCodeAt(i % 3)) % 1000000007;
}
console.log("alternate", alt);
// Search and slice far into both.
const far = mixed.indexOf("item5999");
console.log("far", far, mixed.slice(far, far + 12), mixed.lastIndexOf("\u{1F600}"), mixed.lastIndexOf("caf\u00e9"));
const astralAt = mixed.indexOf("\u{1F600}");
console.log("astral", astralAt, mixed.charCodeAt(astralAt), mixed.charCodeAt(astralAt + 1), mixed.charAt(astralAt - 1));
console.log("ascii far", ascii.indexOf("value5999"), ascii.slice(-20), ascii.substring(100000, 100010).length);
// A large uniquely owned ASCII string read, then appended in place.
let big = "";
for (let i = 0; i < 9000; i++) big += "line" + String(i) + "\n";
console.log("big", big.length, big.charCodeAt(70000), big.charCodeAt(big.length - 1));
big += "\u00e9nd";
console.log("big+bmp", big.length, big.charCodeAt(big.length - 3), big.charCodeAt(big.length - 1), big.slice(-6));
big += "\u{1F600}";
console.log("big+astral", big.length, big.charCodeAt(big.length - 2), big.charCodeAt(big.length - 1), big.indexOf("\u00e9"));
summary("big", big);
// A large builder-made string proven ASCII by its length, then appended in
// place: the proof must not survive the append.
const rows: number[][] = [];
for (let i = 0; i < 5000; i++) rows.push([i, i * 2, i % 7]);
let doc = JSON.stringify(rows);
console.log("doc", doc.length, doc.charCodeAt(doc.length - 1), doc.indexOf("[4999"));
doc += "\u00e9\u{1F600}";
console.log("doc+", doc.length, doc.charCodeAt(doc.length - 3), doc.charCodeAt(doc.length - 2), doc.charCodeAt(doc.length - 1), doc.slice(-8));
summary("doc", doc);