mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-01 07:33:12 +08:00
Chris Tate
a88fdacbe2
for-of, spread, Array.from, yield* and [Symbol.iterator]() over a string walked a UTF-16 index. Each step mapped the index to a byte offset, built the character, and measured its UTF-16 length to advance. They now walk a byte cursor over the WTF-8 data: - The cpAtByte intrinsic reads the character that starts at the cursor. ASCII bytes take the one-character table inline; other characters call scr_str_cp_at_byte. byteLength advances the cursor by the character's byte count. A lone surrogate is its own three-byte character, so it stays a one-unit character, as in Node. - Spread and Array.from of a string call scr_str_code_points, which counts the characters, allocates the array once and fills it. - The one-character strings of U+0080..U+07FF are made immortal on first use and shared, as the ASCII ones are static. The cpAt intrinsic, which no lowering emits any more, is removed.
60 lines
2.0 KiB
TypeScript
60 lines
2.0 KiB
TypeScript
// Code point iteration over mixed text: a word tokenizer over for-of, a
|
|
// character histogram, reversal through spread, and Array.from over
|
|
// words. The text mixes ASCII, Latin-1 letters, CJK and emoji.
|
|
let seed = 11;
|
|
function random(): number {
|
|
seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
|
|
return seed / 4294967296;
|
|
}
|
|
const pieces = [
|
|
"the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog",
|
|
"café", "naïve", "über", "señor", "straße", "déjà", "vu",
|
|
"日本", "語", "テキスト", "中文", "한국어",
|
|
"😀", "🚀", "👍", "🎉",
|
|
];
|
|
const punctuation = [" ", " ", " ", ", ", ". ", "! ", "\n"];
|
|
const scale = Number(process.argv[2] ?? "1");
|
|
const lines: string[] = [];
|
|
for (let i = 0; i < Math.floor(4000 * scale); i++) {
|
|
let line = "";
|
|
const words = 6 + Math.floor(random() * 10);
|
|
for (let w = 0; w < words; w++) {
|
|
line += pieces[Math.floor(random() * pieces.length)]!;
|
|
line += punctuation[Math.floor(random() * punctuation.length)]!;
|
|
}
|
|
lines.push(line);
|
|
}
|
|
|
|
function isWordChar(c: string): boolean {
|
|
const code = c.codePointAt(0)!;
|
|
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || code >= 0xc0;
|
|
}
|
|
|
|
let tokens = 0;
|
|
let tokenUnits = 0;
|
|
let reversedLength = 0;
|
|
let fromLength = 0;
|
|
const histogram = new Map<string, number>();
|
|
for (let round = 0; round < 30; round++) {
|
|
for (const line of lines) {
|
|
let word = "";
|
|
for (const c of line) {
|
|
if (isWordChar(c)) {
|
|
word += c;
|
|
continue;
|
|
}
|
|
if (word !== "") {
|
|
tokens++;
|
|
tokenUnits += word.length;
|
|
if (round === 0) histogram.set(word, (histogram.get(word) ?? 0) + 1);
|
|
fromLength += Array.from(word).length;
|
|
word = "";
|
|
}
|
|
}
|
|
reversedLength += [...line].reverse().join("").length;
|
|
}
|
|
}
|
|
const top = [...histogram].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 5);
|
|
console.log(top.map(([word, count]) => `${word}:${count}`).join(" "));
|
|
console.log("tokens", tokens, "units", tokenUnits, "reversed", reversedLength, "from", fromLength);
|