From 975270cef5a59ffe33821c8abe2be257dc141b24 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 03:02:53 -0500 Subject: [PATCH] Decode base64 eight characters at a time Buffer.from(text, "base64" | "base64url") decoded each four-character group with four lookups and a sign test, then stored its three bytes one at a time. The decoder now first takes complete groups two at a time: each character's value comes from a table that holds it already shifted into place within its group's 24 bits, a single bit test covers all eight characters, and each group's three bytes go out in one four-byte store whose last byte the next store overwrites. A group with any other character (padding, whitespace, an invalid character) takes the previous path, and the two-group step resumes after it, so line-wrapped text keeps most of the gain. The destination already holds two spare bytes, which cover the extra store byte. --- .../corpus/8006-buffer-base64-decode.ts.json | 6 ++ packages/runtime/src/scr_bytes.c | 66 ++++++++++++++++--- tests/corpus/8006-buffer-base64-decode.ts | 66 +++++++++++++++++++ .../8006-buffer-base64-decode.ts.json | 1 + 4 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8006-buffer-base64-decode.ts.json create mode 100644 tests/corpus/8006-buffer-base64-decode.ts create mode 100644 tests/harness/corpus-costs/8006-buffer-base64-decode.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8006-buffer-base64-decode.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8006-buffer-base64-decode.ts.json new file mode 100644 index 000000000..135fe4af6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8006-buffer-base64-decode.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8006-buffer-base64-decode.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 6b8594c08..31716604e 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -2672,20 +2672,66 @@ static ScrBytes *scr_bytes_conversion_storage(size_t capacity) { return scr_bytes_conversion_storage_in(NULL, capacity); } +/* scr_b64_values, each value shifted into its place in a four-character + * group's 24 bits; a character outside both alphabets sets bit 24. */ +#define SCR_B64_VALUE(c) \ + ((c) >= 'A' && (c) <= 'Z' ? (c) - 'A' \ + : (c) >= 'a' && (c) <= 'z' ? (c) - 'a' + 26 \ + : (c) >= '0' && (c) <= '9' ? (c) - '0' + 52 \ + : (c) == '+' || (c) == '-' ? 62 \ + : (c) == '/' || (c) == '_' ? 63 \ + : -1) +#define SCR_B64_AT(c, shift) \ + (SCR_B64_VALUE(c) < 0 ? UINT32_C(0x1000000) : (uint32_t)SCR_B64_VALUE(c) << (shift)) +#define SCR_B64_4(c, s) SCR_B64_AT(c, s), SCR_B64_AT(c + 1, s), SCR_B64_AT(c + 2, s), SCR_B64_AT(c + 3, s) +#define SCR_B64_16(c, s) SCR_B64_4(c, s), SCR_B64_4(c + 4, s), SCR_B64_4(c + 8, s), SCR_B64_4(c + 12, s) +#define SCR_B64_64(c, s) SCR_B64_16(c, s), SCR_B64_16(c + 16, s), SCR_B64_16(c + 32, s), SCR_B64_16(c + 48, s) +#define SCR_B64_256(s) SCR_B64_64(0, s), SCR_B64_64(64, s), SCR_B64_64(128, s), SCR_B64_64(192, s) +static const uint32_t scr_b64_shifted[4][256] = { + {SCR_B64_256(18)}, {SCR_B64_256(12)}, {SCR_B64_256(6)}, {SCR_B64_256(0)}}; + +/* One group's 24 bits as three bytes, in a four-byte store whose last byte + * the next store overwrites (the destination holds two spare bytes). */ +static inline void scr_b64_put_group(uint8_t *out, uint32_t bits) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + uint32_t word = __builtin_bswap32(bits << 8); +#else + uint32_t word = bits << 8; +#endif + memcpy(out, &word, 4); +} + static ScrBytes *scr_bytes_decode_b64(ScrBytesSink *sink, const uint8_t *in, size_t n) { ScrBytes *b = scr_bytes_conversion_storage_in(sink, n / 4 * 3 + 2); + /* A local copy: a byte store could otherwise change b->data. */ + uint8_t *out = b->data; + const uint32_t(*t)[256] = scr_b64_shifted; size_t i = 0, o = 0; unsigned acc = 0, have = 0; while (i < n) { + /* Complete groups, two per step: each character's shifted value comes + * from a table and one bit test covers all eight. The second store + * ends at o + 6, within the destination while eight characters remain. */ + if (have == 0) { + while (n - i >= 8) { + uint32_t first = t[0][in[i]] | t[1][in[i + 1]] | t[2][in[i + 2]] | t[3][in[i + 3]]; + uint32_t second = t[0][in[i + 4]] | t[1][in[i + 5]] | t[2][in[i + 6]] | t[3][in[i + 7]]; + if ((first | second) & UINT32_C(0x1000000)) break; + scr_b64_put_group(out + o, first); + scr_b64_put_group(out + o + 3, second); + o += 6; + i += 8; + } + } /* Common complete groups need neither per-character branches nor state. */ if (have == 0 && n - i >= 4) { int a = scr_b64_values[in[i]], c = scr_b64_values[in[i + 1]]; int d = scr_b64_values[in[i + 2]], e = scr_b64_values[in[i + 3]]; if ((a | c | d | e) >= 0) { unsigned v = ((unsigned)a << 18) | ((unsigned)c << 12) | ((unsigned)d << 6) | (unsigned)e; - b->data[o++] = (uint8_t)(v >> 16); - b->data[o++] = (uint8_t)(v >> 8); - b->data[o++] = (uint8_t)v; + out[o++] = (uint8_t)(v >> 16); + out[o++] = (uint8_t)(v >> 8); + out[o++] = (uint8_t)v; i += 4; continue; } @@ -2696,17 +2742,17 @@ static ScrBytes *scr_bytes_decode_b64(ScrBytesSink *sink, const uint8_t *in, siz if (v < 0) continue; acc = (acc << 6) | (unsigned)v; if (++have == 4) { - b->data[o++] = (uint8_t)(acc >> 16); - b->data[o++] = (uint8_t)(acc >> 8); - b->data[o++] = (uint8_t)acc; + out[o++] = (uint8_t)(acc >> 16); + out[o++] = (uint8_t)(acc >> 8); + out[o++] = (uint8_t)acc; acc = 0; have = 0; } } - if (have == 2) b->data[o++] = (uint8_t)(acc >> 4); + if (have == 2) out[o++] = (uint8_t)(acc >> 4); else if (have == 3) { - b->data[o++] = (uint8_t)(acc >> 10); - b->data[o++] = (uint8_t)(acc >> 2); + out[o++] = (uint8_t)(acc >> 10); + out[o++] = (uint8_t)(acc >> 2); } b->len = o; return b; @@ -2764,7 +2810,7 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const /* Node interprets encoded text as the low byte of each UTF-16 unit. * ASCII can borrow storage; non-ASCII must preserve that truncation. */ ScrBytes *narrowed = NULL; - if (scr_bytes_ascii_prefix(in, n) != n) { + if (!scr_str_known_ascii(s) && scr_bytes_ascii_prefix(in, n) != n) { ScrStr *latin1 = scr_str_new("latin1", 6); narrowed = scr_bytes_encode_str(NULL, s, latin1); scr_str_release(latin1); diff --git a/tests/corpus/8006-buffer-base64-decode.ts b/tests/corpus/8006-buffer-base64-decode.ts new file mode 100644 index 000000000..3d7b20303 --- /dev/null +++ b/tests/corpus/8006-buffer-base64-decode.ts @@ -0,0 +1,66 @@ +// Buffer.from(text, "base64" | "base64url") over complete eight-character +// blocks and the text around them: every length from 0 to 40, an invalid +// character at each position of a block, padding and text after it, both +// alphabets, line breaks every 76 characters, sizes around the pool +// threshold, and characters past U+00FF. + +let seed = 7; +function next(): number { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + return seed >>> 8; +} +function bytes(size: number): Buffer { + const buf = Buffer.alloc(size); + for (let i = 0; i < size; i++) buf[i] = next() & 255; + return buf; +} + +const rows: string[] = []; +for (let size = 0; size <= 30; size++) { + const buf = bytes(size); + const text = buf.toString("base64"); + const url = buf.toString("base64url"); + rows.push( + `${size}:${text.length}:${Buffer.from(text, "base64").equals(buf)}:${Buffer.from(url, "base64url").equals(buf)}:${Buffer.from(url, "base64").equals(buf)}`, + ); +} +console.log(rows.join(" ")); + +const block = Buffer.from("0123456789abcdefghijklmnopqrstuv").toString("base64"); +for (let at = 0; at < 16; at++) { + for (const bad of [" ", "\n", "*", "=", "é", "中"]) { + const text = block.slice(0, at) + bad + block.slice(at); + console.log(at, JSON.stringify(bad), Buffer.from(text, "base64").toString("hex")); + } +} + +for (const text of [ + "QUJDREVGR0g=", + "QUJDREVGR0", + "QUJDREVGR0g=QUJD", + "QUJD=REVGR0hJ", + "QUJDREVGR0hJSktM", + "QUJDREVGR0hJSktMTQ", + "QUJDREVGR0hJSktMTQ==", + "--__--__", + "++//++//", + "+-/_+-/_+-/_", + "Q U J D R E V G R 0 h J", + "QUJD\tREVG\r\nR0hJ", + "=QUJDREVGR0hJ", + "", +]) { + console.log(JSON.stringify(text), Buffer.from(text, "base64").toString("hex"), Buffer.from(text, "base64url").length); +} + +for (const size of [4095, 4096, 4097, 6144, 6145, 70000]) { + const buf = bytes(size); + const text = buf.toString("base64"); + const wrapped = text.replace(/.{76}/g, "$&\n"); + const decoded = Buffer.from(text, "base64"); + const unwrapped = Buffer.from(wrapped, "base64"); + console.log(size, decoded.length, decoded.equals(buf), unwrapped.equals(buf), Buffer.byteLength(text, "base64")); +} + +const wide = "QUJDREVGR0hJİSktM"; +console.log(Buffer.from(wide, "base64").toString("hex"), Buffer.from("ŁłQUJD", "base64").toString("hex")); diff --git a/tests/harness/corpus-costs/8006-buffer-base64-decode.ts.json b/tests/harness/corpus-costs/8006-buffer-base64-decode.ts.json new file mode 100644 index 000000000..88f85d0aa --- /dev/null +++ b/tests/harness/corpus-costs/8006-buffer-base64-decode.ts.json @@ -0,0 +1 @@ +{ "plain": 3995, "san": 2236 }