Convert Buffers to and from latin1 and hex text a word at a time

- toString("latin1"): counts the high bytes eight at a time, copies
  words of ASCII bytes, and writes other bytes from a table of their
  UTF-8 pairs without branches. ASCII-only results are marked ASCII, and
  other results record their UTF-16 length (one unit per byte), so
  `.length` does not walk the string.
- Buffer.from(text, "latin1" | "ascii" | "binary"): copies text known to
  be ASCII, copies words of ASCII bytes, and decodes eight bytes of one-
  and two-byte sequences in one word, storing each output byte without
  branches. Longer sequences take the previous path.
- Buffer.from(text, "hex"): reads digit values from a table.
- toString("hex") and toString("base64"/"base64url") mark their results
  ASCII, so a later decode, length or index read skips the scan for
  non-ASCII bytes.

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8005-buffer-encoding-paths.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8005-buffer-encoding-paths.ts"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_bytes.c

@@ -1577,6 +1577,7 @@ static ScrStr *scr_bytes_encode_b64(const uint8_t *in, size_t n, bool url) {
     if (!url) out[o++] = '=';
   }
   out[outlen] = '\0';
+  scr_str_note_ascii(s);
   return s;
 }
 
@@ -2497,6 +2498,106 @@ static ScrStr *scr_bytes_decode_utf16le(const uint8_t *in, size_t n) {
   return s;
 }
 
+/* The UTF-8 bytes of U+0000..U+00FF: the byte itself below 0x80 (the
+ * second entry is overwritten by the next write), else a two-byte
+ * sequence. */
+static const uint8_t scr_latin1_utf8[256][2] = {
+    {0x00, 0x00}, {0x01, 0x00}, {0x02, 0x00}, {0x03, 0x00}, {0x04, 0x00}, {0x05, 0x00}, {0x06, 0x00}, {0x07, 0x00},
+    {0x08, 0x00}, {0x09, 0x00}, {0x0a, 0x00}, {0x0b, 0x00}, {0x0c, 0x00}, {0x0d, 0x00}, {0x0e, 0x00}, {0x0f, 0x00},
+    {0x10, 0x00}, {0x11, 0x00}, {0x12, 0x00}, {0x13, 0x00}, {0x14, 0x00}, {0x15, 0x00}, {0x16, 0x00}, {0x17, 0x00},
+    {0x18, 0x00}, {0x19, 0x00}, {0x1a, 0x00}, {0x1b, 0x00}, {0x1c, 0x00}, {0x1d, 0x00}, {0x1e, 0x00}, {0x1f, 0x00},
+    {0x20, 0x00}, {0x21, 0x00}, {0x22, 0x00}, {0x23, 0x00}, {0x24, 0x00}, {0x25, 0x00}, {0x26, 0x00}, {0x27, 0x00},
+    {0x28, 0x00}, {0x29, 0x00}, {0x2a, 0x00}, {0x2b, 0x00}, {0x2c, 0x00}, {0x2d, 0x00}, {0x2e, 0x00}, {0x2f, 0x00},
+    {0x30, 0x00}, {0x31, 0x00}, {0x32, 0x00}, {0x33, 0x00}, {0x34, 0x00}, {0x35, 0x00}, {0x36, 0x00}, {0x37, 0x00},
+    {0x38, 0x00}, {0x39, 0x00}, {0x3a, 0x00}, {0x3b, 0x00}, {0x3c, 0x00}, {0x3d, 0x00}, {0x3e, 0x00}, {0x3f, 0x00},
+    {0x40, 0x00}, {0x41, 0x00}, {0x42, 0x00}, {0x43, 0x00}, {0x44, 0x00}, {0x45, 0x00}, {0x46, 0x00}, {0x47, 0x00},
+    {0x48, 0x00}, {0x49, 0x00}, {0x4a, 0x00}, {0x4b, 0x00}, {0x4c, 0x00}, {0x4d, 0x00}, {0x4e, 0x00}, {0x4f, 0x00},
+    {0x50, 0x00}, {0x51, 0x00}, {0x52, 0x00}, {0x53, 0x00}, {0x54, 0x00}, {0x55, 0x00}, {0x56, 0x00}, {0x57, 0x00},
+    {0x58, 0x00}, {0x59, 0x00}, {0x5a, 0x00}, {0x5b, 0x00}, {0x5c, 0x00}, {0x5d, 0x00}, {0x5e, 0x00}, {0x5f, 0x00},
+    {0x60, 0x00}, {0x61, 0x00}, {0x62, 0x00}, {0x63, 0x00}, {0x64, 0x00}, {0x65, 0x00}, {0x66, 0x00}, {0x67, 0x00},
+    {0x68, 0x00}, {0x69, 0x00}, {0x6a, 0x00}, {0x6b, 0x00}, {0x6c, 0x00}, {0x6d, 0x00}, {0x6e, 0x00}, {0x6f, 0x00},
+    {0x70, 0x00}, {0x71, 0x00}, {0x72, 0x00}, {0x73, 0x00}, {0x74, 0x00}, {0x75, 0x00}, {0x76, 0x00}, {0x77, 0x00},
+    {0x78, 0x00}, {0x79, 0x00}, {0x7a, 0x00}, {0x7b, 0x00}, {0x7c, 0x00}, {0x7d, 0x00}, {0x7e, 0x00}, {0x7f, 0x00},
+    {0xc2, 0x80}, {0xc2, 0x81}, {0xc2, 0x82}, {0xc2, 0x83}, {0xc2, 0x84}, {0xc2, 0x85}, {0xc2, 0x86}, {0xc2, 0x87},
+    {0xc2, 0x88}, {0xc2, 0x89}, {0xc2, 0x8a}, {0xc2, 0x8b}, {0xc2, 0x8c}, {0xc2, 0x8d}, {0xc2, 0x8e}, {0xc2, 0x8f},
+    {0xc2, 0x90}, {0xc2, 0x91}, {0xc2, 0x92}, {0xc2, 0x93}, {0xc2, 0x94}, {0xc2, 0x95}, {0xc2, 0x96}, {0xc2, 0x97},
+    {0xc2, 0x98}, {0xc2, 0x99}, {0xc2, 0x9a}, {0xc2, 0x9b}, {0xc2, 0x9c}, {0xc2, 0x9d}, {0xc2, 0x9e}, {0xc2, 0x9f},
+    {0xc2, 0xa0}, {0xc2, 0xa1}, {0xc2, 0xa2}, {0xc2, 0xa3}, {0xc2, 0xa4}, {0xc2, 0xa5}, {0xc2, 0xa6}, {0xc2, 0xa7},
+    {0xc2, 0xa8}, {0xc2, 0xa9}, {0xc2, 0xaa}, {0xc2, 0xab}, {0xc2, 0xac}, {0xc2, 0xad}, {0xc2, 0xae}, {0xc2, 0xaf},
+    {0xc2, 0xb0}, {0xc2, 0xb1}, {0xc2, 0xb2}, {0xc2, 0xb3}, {0xc2, 0xb4}, {0xc2, 0xb5}, {0xc2, 0xb6}, {0xc2, 0xb7},
+    {0xc2, 0xb8}, {0xc2, 0xb9}, {0xc2, 0xba}, {0xc2, 0xbb}, {0xc2, 0xbc}, {0xc2, 0xbd}, {0xc2, 0xbe}, {0xc2, 0xbf},
+    {0xc3, 0x80}, {0xc3, 0x81}, {0xc3, 0x82}, {0xc3, 0x83}, {0xc3, 0x84}, {0xc3, 0x85}, {0xc3, 0x86}, {0xc3, 0x87},
+    {0xc3, 0x88}, {0xc3, 0x89}, {0xc3, 0x8a}, {0xc3, 0x8b}, {0xc3, 0x8c}, {0xc3, 0x8d}, {0xc3, 0x8e}, {0xc3, 0x8f},
+    {0xc3, 0x90}, {0xc3, 0x91}, {0xc3, 0x92}, {0xc3, 0x93}, {0xc3, 0x94}, {0xc3, 0x95}, {0xc3, 0x96}, {0xc3, 0x97},
+    {0xc3, 0x98}, {0xc3, 0x99}, {0xc3, 0x9a}, {0xc3, 0x9b}, {0xc3, 0x9c}, {0xc3, 0x9d}, {0xc3, 0x9e}, {0xc3, 0x9f},
+    {0xc3, 0xa0}, {0xc3, 0xa1}, {0xc3, 0xa2}, {0xc3, 0xa3}, {0xc3, 0xa4}, {0xc3, 0xa5}, {0xc3, 0xa6}, {0xc3, 0xa7},
+    {0xc3, 0xa8}, {0xc3, 0xa9}, {0xc3, 0xaa}, {0xc3, 0xab}, {0xc3, 0xac}, {0xc3, 0xad}, {0xc3, 0xae}, {0xc3, 0xaf},
+    {0xc3, 0xb0}, {0xc3, 0xb1}, {0xc3, 0xb2}, {0xc3, 0xb3}, {0xc3, 0xb4}, {0xc3, 0xb5}, {0xc3, 0xb6}, {0xc3, 0xb7},
+    {0xc3, 0xb8}, {0xc3, 0xb9}, {0xc3, 0xba}, {0xc3, 0xbb}, {0xc3, 0xbc}, {0xc3, 0xbd}, {0xc3, 0xbe}, {0xc3, 0xbf},
+};
+
+/* Each byte is U+00XX ("binary" normalizes here upstream): a byte below
+ * 0x80 is one UTF-8 byte and any other byte is two. Eight bytes without a
+ * high bit copy as one word; other bytes take their two table bytes and
+ * advance by their length, without branches (the terminator slot holds
+ * the last spare byte). */
+/* The number of bytes from 0x80: eight per step, each step adding its
+ * high bits into byte lanes that are summed every 255 steps. */
+static size_t scr_bytes_count_high(const uint8_t *in, size_t n) {
+  const uint64_t ones = UINT64_C(0x0101010101010101);
+  const uint64_t pairs = UINT64_C(0x00ff00ff00ff00ff);
+  size_t high = 0, i = 0;
+  while (n - i >= 8) {
+    size_t steps = (n - i) / 8 < 255 ? (n - i) / 8 : 255;
+    uint64_t lanes = 0;
+    for (size_t end = i + steps * 8; i < end; i += 8) {
+      uint64_t w;
+      memcpy(&w, in + i, 8);
+      lanes += (w >> 7) & ones;
+    }
+    lanes = (lanes & pairs) + ((lanes >> 8) & pairs);
+    high += (size_t)((lanes * UINT64_C(0x0001000100010001)) >> 48);
+  }
+  for (; i < n; i++) high += in[i] >> 7;
+  return high;
+}
+
+static ScrStr *scr_bytes_latin1_text(const uint8_t *in, size_t n) {
+  size_t high = scr_bytes_count_high(in, n);
+  if (n > SIZE_MAX - high) scr_bytes_oom();
+  ScrStr *s = scr_str_alloc_raw(n + high, n + high);
+  uint8_t *out = (uint8_t *)s->data;
+  if (high == 0) {
+    memcpy(out, in, n);
+    out[n] = '\0';
+    scr_str_note_ascii(s);
+    return s;
+  }
+  const uint64_t top = UINT64_C(0x8080808080808080);
+  size_t i = 0, o = 0;
+  while (n - i >= 8) {
+    uint64_t w;
+    memcpy(&w, in + i, 8);
+    if (!(w & top)) {
+      memcpy(out + o, &w, 8);
+      i += 8;
+      o += 8;
+      continue;
+    }
+    for (size_t end = i + 8; i < end; i++) {
+      memcpy(out + o, scr_latin1_utf8[in[i]], 2);
+      o += 1 + (in[i] >> 7);
+    }
+  }
+  for (; i < n; i++) {
+    memcpy(out + o, scr_latin1_utf8[in[i]], 2);
+    o += 1 + (in[i] >> 7);
+  }
+  out[o] = '\0';
+  scr_str_note_utf16_len(s, n);
+  return s;
+}
+
 ScrStr *scr_bytes_to_str(const ScrBytes *b, const ScrStr *enc) {
   SCR_BYTES_SNAPSHOT(b);
   const uint8_t *in = b->data;
@@ -2512,21 +2613,12 @@ ScrStr *scr_bytes_to_str(const ScrBytes *b, const ScrStr *enc) {
       out[i * 2 + 1] = scr_hex_digits[in[i] & 0xf];
     }
     out[n * 2] = '\0';
+    scr_str_note_ascii(s);
     return s;
   }
   if (scr_enc_is(enc, "base64")) return scr_bytes_encode_b64(in, n, false);
   if (scr_enc_is(enc, "base64url")) return scr_bytes_encode_b64(in, n, true);
-  if (scr_enc_is(enc, "latin1")) {
-    /* Each byte is U+00XX ("binary" normalizes here upstream). */
-    size_t high = 0;
-    for (size_t i = 0; i < n; i++) high += in[i] >> 7;
-    if (n > SIZE_MAX - high) scr_bytes_oom();
-    ScrStr *s = scr_str_alloc_raw(n + high, n + high);
-    size_t o = 0;
-    for (size_t i = 0; i < n; i++) o = scr_bytes_put_cp(s->data, o, in[i]);
-    s->data[o] = '\0';
-    return s;
-  }
+  if (scr_enc_is(enc, "latin1")) return scr_bytes_latin1_text(in, n);
   if (scr_enc_is(enc, "ascii")) {
     /* Node's ascii decode masks the high bit: byte & 0x7f. */
     ScrStr *s = scr_str_alloc_raw(n, n);
@@ -2621,12 +2713,25 @@ ScrStr *scr_bytes_to_str_checked_range(const ScrBytes *b, const ScrStr *enc,
   return out;
 }
 
-static int scr_hex_val(uint8_t c) {
-  if (c >= '0' && c <= '9') return c - '0';
-  if (c >= 'a' && c <= 'f') return c - 'a' + 10;
-  if (c >= 'A' && c <= 'F') return c - 'A' + 10;
-  return -1;
-}
+/* Hex digit values, -1 for any other byte. */
+static const int8_t scr_hex_values[256] = {
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+   0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1, -1, -1, -1, -1,
+  -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+};
 
 /* Both alphabets are accepted by both Buffer base64 encoding names. */
 static const int8_t scr_b64_values[256] = {
@@ -2720,17 +2825,71 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const
   size_t n = s->len;
   if (scr_enc_is(enc, "latin1") || scr_enc_is(enc, "ascii")) {
     /* Node writes charCodeAt(i) & 0xFF for BOTH spellings — an astral
-     * code point contributes its two surrogates' low bytes. */
+     * code point contributes its two surrogates' low bytes. ASCII text
+     * copies. Elsewhere eight ASCII bytes copy as one word. In eight bytes
+     * holding two or more high bytes and no sequence longer than two
+     * bytes, every continuation byte's output byte (the code point's low
+     * byte, from the continuation and the lead before it) is computed in
+     * one word; each byte is then stored without branches, a continuation
+     * replacing the byte its lead wrote. */
     ScrBytes *b = scr_bytes_conversion_storage_in(sink, n);
+    if (scr_str_known_ascii(s)) {
+      memcpy(b->data, in, n);
+      b->len = n;
+      return b;
+    }
+    /* A local copy: a byte store could otherwise change b->data. */
+    uint8_t *dst = b->data;
+    const uint64_t top = UINT64_C(0x8080808080808080);
     size_t o = 0, i = 0;
     while (i < n) {
+      if (n - i >= 8) {
+        uint64_t w;
+        memcpy(&w, in + i, 8);
+        uint64_t high = w & top;
+        if (!high) {
+          memcpy(dst + o, &w, 8);
+          i += 8;
+          o += 8;
+          continue;
+        }
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+        /* Byte k of the input is bits 8k..8k+7 of w. */
+        if (i > 0 && (high & (high - 1)) && !(w & (w << 1) & (w << 2) & top)) {
+          uint64_t prev = (w << 8) | in[i - 1];
+          uint64_t cont = w & ~(w << 1) & top;
+          uint64_t low = ((prev & UINT64_C(0x0303030303030303)) << 6) |
+                         (w & UINT64_C(0x3f3f3f3f3f3f3f3f));
+          uint64_t mask = (cont >> 7) * 0xff;
+          uint64_t out = (w & ~mask) | (low & mask);
+          for (unsigned k = 0; k < 8; k++) {
+            o -= (size_t)(cont >> (8 * k + 7)) & 1;
+            dst[o++] = (uint8_t)(out >> (8 * k));
+          }
+          i += 8;
+          continue;
+        }
+#endif
+      }
+      unsigned c = in[i];
+      if (c < 0x80) {
+        dst[o++] = (uint8_t)c;
+        i++;
+        continue;
+      }
+      if (c >> 6 == 2) {
+        /* The rest of a two-byte sequence whose lead ended a word. */
+        dst[o - 1] = (uint8_t)(((in[i - 1] & 3u) << 6) | (c & 0x3f));
+        i++;
+        continue;
+      }
       uint32_t cp = scr_bytes_next_cp(in, &i);
       if (cp > 0xffff) {
         cp -= 0x10000;
-        b->data[o++] = (uint8_t)(0xd800 + (cp >> 10));
-        b->data[o++] = (uint8_t)(0xdc00 + (cp & 0x3ff));
+        dst[o++] = (uint8_t)(0xd800 + (cp >> 10));
+        dst[o++] = (uint8_t)(0xdc00 + (cp & 0x3ff));
       } else {
-        b->data[o++] = (uint8_t)cp;
+        dst[o++] = (uint8_t)cp;
       }
     }
     b->len = o; /* never grows past n: multi-byte cps shrink */
@@ -2762,7 +2921,7 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const
     /* Node interprets encoded text as the low byte of each UTF-16 unit.
      * ASCII can borrow storage; non-ASCII must preserve that truncation. */
     ScrBytes *narrowed = NULL;
-    if (scr_bytes_ascii_prefix(in, n) != n) {
+    if (!scr_str_known_ascii(s) && scr_bytes_ascii_prefix(in, n) != n) {
       ScrStr *latin1 = scr_str_new("latin1", 6);
       narrowed = scr_bytes_encode_str(NULL, s, latin1);
       scr_str_release(latin1);
@@ -2773,11 +2932,12 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const
     if (hex) {
       /* Stop at the first invalid pair, discarding an odd tail. */
       b = scr_bytes_conversion_storage_in(sink, n / 2);
-      size_t o = 0;
-      for (size_t i = 0; i + 1 < n; i += 2) {
-        int hi = scr_hex_val(in[i]), lo = scr_hex_val(in[i + 1]);
-        if (hi < 0 || lo < 0) break;
-        b->data[o++] = (uint8_t)((hi << 4) | lo);
+      uint8_t *dst = b->data;
+      size_t pairs = n / 2, o = 0;
+      for (; o < pairs; o++) {
+        int hi = scr_hex_values[in[2 * o]], lo = scr_hex_values[in[2 * o + 1]];
+        if ((hi | lo) < 0) break;
+        dst[o] = (uint8_t)((hi << 4) | lo);
       }
       b->len = o;
     } else {

packages/runtime/src/scr_string.c

@@ -1991,6 +1991,19 @@ double scr_str_utf16_len(ScrStr *s) {
   return (double)(int64_t)units;
 }
 
+/* Records the UTF-16 length of a string whose producer counted it (latin1
+ * text has one unit per byte), so scr_str_utf16_len does not walk it. Only
+ * strings that scr_str_utf16_len would otherwise record are kept. */
+void scr_str_note_utf16_len(ScrStr *s, size_t units) {
+  if (s->ascii || s->len <= 64 || s->rc == SIZE_MAX) return;
+  if (!scr_u16len_tab) {
+    scr_u16len_tab = calloc(SCR_U16LEN_N, sizeof *scr_u16len_tab);
+    if (!scr_u16len_tab) return;
+    scr_sidx_register_cleanup();
+  }
+  *scr_u16len_slot(s) = (ScrU16Len){s, units};
+}
+
 double scr_str_char_code_at(ScrStr *s, double i) {
   /* A sequential scan of a non-ASCII receiver stays inside the bounded
    * ASCII span cached at its last resolved position. Its unit offset is an

packages/runtime/src/scr_string.h

@@ -83,6 +83,9 @@ static inline void scr_str_hash_forget(ScrStr *s) {
 /* True only when every byte is proven below 0x80 (see above). */
 static inline bool scr_str_known_ascii(const ScrStr *s) { return s->ascii; }
 
+/* Record the UTF-16 length a string's producer counted (scr_string.c). */
+void scr_str_note_utf16_len(ScrStr *s, size_t units);
+
 /* Record a proven-ASCII fact. Immortal strings may live in read-only
  * memory and are shared across threads, so they are never written. */
 static inline void scr_str_note_ascii(ScrStr *s) {

tests/corpus/8005-buffer-encoding-paths.ts

@@ -0,0 +1,105 @@
+// Buffer latin1, ascii, hex and base64 conversions in both directions:
+// ASCII, binary and mixed contents at sizes around eight-byte words and the
+// pool threshold, strings with code points past U+00FF, invalid hex and
+// base64 text, and the lengths and characters of the strings produced.
+
+let seed = 12345;
+function next(): number {
+  seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+  return seed >>> 8;
+}
+
+function fill(size: number, kind: string): Buffer {
+  const buf = Buffer.alloc(size);
+  for (let i = 0; i < size; i++) {
+    const r = next();
+    if (kind === "ascii") buf[i] = 32 + (r % 95);
+    else if (kind === "binary") buf[i] = r & 255;
+    else if (kind === "high") buf[i] = 128 + (r % 128);
+    else buf[i] = r % 23 === 0 ? 160 + (r % 96) : 97 + (r % 26);
+  }
+  return buf;
+}
+
+function digest(text: string): number {
+  let h = 0;
+  for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0;
+  return h;
+}
+
+const sizes = [0, 1, 7, 8, 9, 15, 16, 17, 63, 64, 65, 4095, 4096, 4097, 70001];
+for (const kind of ["ascii", "binary", "high", "sparse"]) {
+  const rows: string[] = [];
+  for (const size of sizes) {
+    const buf = fill(size, kind);
+    const latin1 = buf.toString("latin1");
+    const back = Buffer.from(latin1, "latin1");
+    const ascii = Buffer.from(latin1, "ascii");
+    const binary = Buffer.from(buf.toString("binary"), "binary");
+    const hex = buf.toString("hex");
+    const b64 = buf.toString("base64");
+    const url = buf.toString("base64url");
+    rows.push(
+      [
+        size,
+        latin1.length,
+        digest(latin1),
+        back.equals(buf),
+        ascii.equals(buf),
+        binary.equals(buf),
+        Buffer.from(hex, "hex").equals(buf),
+        Buffer.from(b64, "base64").equals(buf),
+        Buffer.from(url, "base64url").equals(buf),
+        hex.length,
+        b64.length,
+        Buffer.byteLength(latin1, "latin1"),
+        size > 2 ? latin1.charCodeAt(size - 2) : -1,
+        size > 4 ? latin1.slice(1, 4).length : -1,
+      ].join(","),
+    );
+  }
+  console.log(kind, rows.join(" "));
+}
+
+// Strings with code points past U+00FF keep the low byte of each UTF-16
+// unit, and an astral code point gives its two surrogates' low bytes.
+const wide = [
+  "abcéÿ",
+  "ĀŁ߿",
+  "xࠀ€￿y",
+  "\u{1F600}a\u{10FFFF}",
+  "é".repeat(9) + "€" + "è".repeat(9),
+  "aaaaaaaébbbbbbbb中cccccccc\u{1F600}",
+  "ÀÁÂÃÄÅÆÇࠀÈ",
+];
+for (const text of wide) {
+  console.log(
+    Buffer.from(text, "latin1").toString("hex"),
+    Buffer.from(text, "ascii").toString("hex"),
+    Buffer.byteLength(text, "latin1"),
+  );
+}
+const mixed = "é".repeat(5000) + "中" + "z".repeat(5000) + "ÿ".repeat(3000);
+const mixedBytes = Buffer.from(mixed, "latin1");
+console.log(mixedBytes.length, mixedBytes[0], mixedBytes[5000], mixedBytes[5001], mixedBytes[13000]);
+
+// Hex stops at the first invalid pair and drops an odd last digit.
+for (const text of ["0aFf", "0aF", "0a1g22", "zz00", "", "ABCDEF0123456789", "0aİ0b", "00a"]) {
+  console.log(JSON.stringify(text), Buffer.from(text, "hex").toString("hex"), Buffer.byteLength(text, "hex"));
+}
+const longHex = "0123456789abcdef".repeat(600) + "0x" + "ff".repeat(10);
+console.log(Buffer.from(longHex, "hex").length);
+
+// base64 skips characters outside both alphabets and stops at padding.
+for (const text of ["aGVsbG8=", "aGVsbG8", "aGV sbG8=", "aGV\nsbG8", "-_-_", "+/+/", "aGVsbG8=d29ybGQ=", "a", "ab", "abc", "aébc", "====", "YQ"]) {
+  console.log(JSON.stringify(text), Buffer.from(text, "base64").toString("hex"), Buffer.from(text, "base64url").toString("hex"));
+}
+
+// Strings produced by hex and base64 behave as ASCII text afterwards.
+const produced = fill(5000, "binary");
+const producedHex = produced.toString("hex");
+const joined = producedHex + "é" + produced.toString("base64");
+console.log(joined.length, joined.indexOf("é"), digest(joined.slice(9990, 10020)));
+const latinText = fill(9000, "high").toString("latin1");
+const appended = latinText + "€";
+console.log(latinText.length, appended.length, appended.charCodeAt(8999), appended.charCodeAt(9000), appended.slice(8998).length);

tests/harness/corpus-costs/8005-buffer-encoding-paths.ts.json

@@ -0,0 +1 @@
+{ "plain": 2558, "san": 4160 }