Decode string storage within its length in Buffer encodings

String storage can hold bytes that are not well-formed UTF-8: an HTTP
header value keeps the raw bytes a client sent. Encoding such a string
read past its end.

- scr_bytes_next_cp decoded any high byte as the lead of a sequence and
  read up to three bytes after it without a bound. It now takes the
  storage length, decodes only complete, well-formed sequences, and
  returns any other byte as U+0000..U+00FF, advancing by one. That is
  each such byte's value in Node, which decodes header values as latin1.
  This covers latin1, ascii, binary and utf16le encoding, hex and base64
  of non-ASCII text (which narrow through latin1), Buffer#write and
  TextEncoder#encodeInto.
- base64 decoding read one byte past its input when the eight-character
  groups ended exactly at the end.

The runtime's bytes unit test (built with ASan) encodes ill-formed
storage at every offset within a word and random high bytes, and
compares latin1, ascii and utf16le with a scalar reference. A corpus
program sends such header bytes to a server and encodes them as Node
does.

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8010-http-header-raw-bytes.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8010-http-header-raw-bytes.ts"
+ ],
+ "diags": []
+}

packages/runtime/src/scr_bytes.c

@@ -2240,29 +2240,26 @@ ScrStr *scr_text_decode_stream(ScrDyn *state, const ScrBytes *input,
   return result;
 }
 
-/* Decode the next code point from an ScrStr's storage — ALWAYS valid
- * UTF-8 (the string runtime never stores ill-formed sequences), so no
- * validation re-runs here. */
-static uint32_t scr_bytes_next_cp(const uint8_t *in, size_t *ip) {
-  uint8_t b = in[*ip];
-  if (b < 0x80) {
-    *ip += 1;
-    return b;
-  }
-  if ((b & 0xe0) == 0xc0) {
-    uint32_t cp = ((uint32_t)(b & 0x1f) << 6) | (in[*ip + 1] & 0x3f);
-    *ip += 2;
-    return cp;
-  }
-  if ((b & 0xf0) == 0xe0) {
-    uint32_t cp = ((uint32_t)(b & 0xf) << 12) | ((uint32_t)(in[*ip + 1] & 0x3f) << 6) |
-                  (in[*ip + 2] & 0x3f);
-    *ip += 3;
-    return cp;
-  }
-  uint32_t cp = ((uint32_t)(b & 0x7) << 18) | ((uint32_t)(in[*ip + 1] & 0x3f) << 12) |
-                ((uint32_t)(in[*ip + 2] & 0x3f) << 6) | (in[*ip + 3] & 0x3f);
-  *ip += 4;
+/* Decode the next code point from the n bytes of an ScrStr's storage.
+ * Storage is WTF-8, except where raw outside bytes become a string
+ * unchecked (HTTP header values). A byte that does not start a complete,
+ * well-formed sequence stands for U+0000..U+00FF, as each byte of such a
+ * header does in Node, and advances by one; nothing is read past n. */
+static uint32_t scr_bytes_next_cp(const uint8_t *in, size_t n, size_t *ip) {
+  size_t i = *ip;
+  uint8_t b = in[i];
+  *ip = i + 1;
+  if (b < 0x80) return b;
+  size_t len = b >= 0xc2 && b <= 0xdf ? 2 : b >= 0xe0 && b <= 0xef ? 3 : b >= 0xf0 && b <= 0xf4 ? 4 : 0;
+  if (len == 0 || n - i < len) return b;
+  uint8_t lo = b == 0xe0 ? 0xa0 : b == 0xf0 ? 0x90 : 0x80;
+  uint8_t hi = b == 0xf4 ? 0x8f : 0xbf;
+  if (in[i + 1] < lo || in[i + 1] > hi) return b;
+  for (size_t k = 2; k < len; k++)
+    if ((in[i + k] & 0xc0) != 0x80) return b;
+  uint32_t cp = b & (0x7f >> len);
+  for (size_t k = 1; k < len; k++) cp = (cp << 6) | (in[i + k] & 0x3f);
+  *ip = i + len;
   return cp;
 }
 
@@ -2287,7 +2284,7 @@ ScrDyn *scr_text_encode_into(const ScrDyn *source, const ScrDyn *destination) {
     read += ascii;
     if (written == limit) break;
     size_t end = written;
-    uint32_t cp = scr_bytes_next_cp((const uint8_t *)text->data, &end);
+    uint32_t cp = scr_bytes_next_cp((const uint8_t *)text->data, text->len, &end);
     if (end > dest->len) break;
     read += cp > 0xffff ? 2 : 1;
     written = end;
@@ -2440,7 +2437,7 @@ ScrBytes *scr_buffer_transcode(const ScrDyn *source, const ScrDyn *from, const S
     }
     ScrStr *text = scr_str_from_utf8_lossy(input->data, len);
     for (size_t i = 0; i < text->len;) {
-      uint32_t cp = scr_bytes_next_cp((const uint8_t *)text->data, &i);
+      uint32_t cp = scr_bytes_next_cp((const uint8_t *)text->data, text->len, &i);
       written = scr_transcode_put(result->data, written, cp, to_encoding);
     }
     scr_str_release(text);
@@ -2774,6 +2771,7 @@ static ScrBytes *scr_bytes_decode_b64(ScrBytesSink *sink, const uint8_t *in, siz
         o += 6;
         i += 8;
       }
+      if (i == n) break; /* the groups ended the input */
     }
     /* Common complete groups need neither per-character branches nor state. */
     if (have == 0 && n - i >= 4) {
@@ -2824,7 +2822,7 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const
     size_t o = s->ascii ? n : scr_bytes_ascii_prefix(in, n), i = o;
     memcpy(b->data, in, o);
     while (i < n) {
-      uint32_t cp = scr_bytes_next_cp(in, &i);
+      uint32_t cp = scr_bytes_next_cp(in, n, &i);
       if (cp > 0xffff) {
         cp -= 0x10000;
         b->data[o++] = (uint8_t)(0xd800 + (cp >> 10));
@@ -2841,7 +2839,7 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const
     ScrBytes *b = scr_bytes_conversion_storage_in(sink, n * 2);
     size_t o = 0, i = 0;
     while (i < n) {
-      uint32_t cp = scr_bytes_next_cp(in, &i);
+      uint32_t cp = scr_bytes_next_cp(in, n, &i);
       if (cp > 0xffff) {
         cp -= 0x10000;
         uint32_t hi = 0xd800 + (cp >> 10), lo = 0xdc00 + (cp & 0x3ff);
@@ -3745,7 +3743,7 @@ double scr_bytes_write_str(ScrBytes *b, const ScrStr *s, const ScrStr *enc,
           continue;
         }
       }
-      uint32_t cp = scr_bytes_next_cp((const uint8_t *)s->data, &i);
+      uint32_t cp = scr_bytes_next_cp((const uint8_t *)s->data, s->len, &i);
       uint32_t first = cp, second = 0;
       if (cp > 0xffff) {
         cp -= 0x10000;

packages/runtime/test/test_bytes.c

@@ -216,6 +216,135 @@ static void test_encodings(void) {
   scr_str_release(b64);
 }
 
+/* The length of the well-formed WTF-8 sequence at in[i], or 0. */
+static size_t ref_seq_len(const uint8_t *in, size_t n, size_t i) {
+  uint8_t b = in[i];
+  if (b < 0x80) return 1;
+  size_t len;
+  uint8_t lo = 0x80, hi = 0xbf;
+  if (b >= 0xc2 && b <= 0xdf) len = 2;
+  else if (b >= 0xe0 && b <= 0xef) len = 3;
+  else if (b >= 0xf0 && b <= 0xf4) len = 4;
+  else return 0;
+  if (b == 0xe0) lo = 0xa0;
+  if (b == 0xf0) lo = 0x90;
+  if (b == 0xf4) hi = 0x8f;
+  if (n - i < len || in[i + 1] < lo || in[i + 1] > hi) return 0;
+  for (size_t k = 2; k < len; k++)
+    if ((in[i + k] & 0xc0) != 0x80) return 0;
+  return len;
+}
+
+/* The UTF-16 units of string storage, one at a time: a well-formed
+ * sequence gives its code point's units and any other byte stands for
+ * U+0000..U+00FF (a raw HTTP header byte). */
+static size_t ref_units(const uint8_t *in, size_t n, uint16_t *out) {
+  size_t k = 0;
+  for (size_t i = 0; i < n;) {
+    size_t len = ref_seq_len(in, n, i);
+    if (len == 0) {
+      out[k++] = in[i++];
+      continue;
+    }
+    uint32_t cp = len == 1 ? in[i] : (uint32_t)(in[i] & (0x7f >> len));
+    for (size_t j = 1; j < len; j++) cp = (cp << 6) | (in[i + j] & 0x3f);
+    i += len;
+    if (cp > 0xffff) {
+      out[k++] = (uint16_t)(0xd800 + ((cp - 0x10000) >> 10));
+      out[k++] = (uint16_t)(0xdc00 + ((cp - 0x10000) & 0x3ff));
+    } else {
+      out[k++] = (uint16_t)cp;
+    }
+  }
+  return k;
+}
+
+/* Encodes storage that is not well-formed (raw header bytes become string
+ * storage unchecked) and compares latin1, ascii and utf16le with the
+ * reference; hex and base64 decode the latin1 bytes. ASan checks that no
+ * path reads or writes outside either buffer. */
+/* encs: latin1, ascii, utf16le, hex, base64. */
+static void check_raw_encodings(ScrStr *const *encs, const uint8_t *in, size_t n, const char *what) {
+  ScrStr *latin1 = encs[0], *ascii = encs[1], *utf16le = encs[2], *hex = encs[3], *b64 = encs[4];
+  ScrStr *s = scr_str_new((const char *)in, n);
+  uint16_t units[64];
+  size_t k = ref_units(in, n, units);
+  ScrStr *narrow[] = {latin1, ascii};
+  for (size_t e = 0; e < 2; e++) {
+    ScrBytes *b = scr_bytes_from_str(s, narrow[e]);
+    bool ok = b->len == k;
+    for (size_t j = 0; ok && j < k; j++) ok = b->data[j] == (uint8_t)units[j];
+    check(ok, what);
+    scr_bytes_release(b);
+  }
+  ScrBytes *wide = scr_bytes_from_str(s, utf16le);
+  bool ok = wide->len == 2 * k;
+  for (size_t j = 0; ok && j < k; j++)
+    ok = wide->data[2 * j] == (uint8_t)units[j] && wide->data[2 * j + 1] == units[j] >> 8;
+  check(ok, what);
+  scr_bytes_release(wide);
+  scr_bytes_release(scr_bytes_from_str(s, hex));
+  scr_bytes_release(scr_bytes_from_str(s, b64));
+  scr_str_release(s);
+}
+
+static void test_raw_storage_encodings(void) {
+  /* Each pattern at every offset in an eight-byte word, alone and followed
+   * by ASCII, two-byte characters or more of the pattern. */
+  static const struct { const char *bytes; size_t n; } patterns[] = {
+      {"\x80", 1}, {"\x80\x81\xbf\x90", 4}, {"\xc3", 1}, {"\xc3\xa9", 2},
+      {"\xc3\xa9\xc3", 3}, {"\xc0\x80\xc1\xbf", 4}, {"\xe9\x41\xe9", 3},
+      {"\xf0\x9f\x41", 3}, {"\xe2\x82", 2}, {"\xe2\x82\xac", 3},
+      {"\xf0\x9f\x98\x80", 4}, {"\xed\xa0\x80", 3}, {"\xff\xfe\xf5", 3},
+      {"\xdf\xbf\x80\x80", 4}, {"\xc2\x80\xc2", 3}, {"x\xe9y\x80\x80z\xdf", 7},
+  };
+  static const struct { const char *bytes; size_t n; } tails[] = {
+      {"", 0}, {"bcdefghij", 9}, {"\xc3\xa9\xc3\xa9\xc3\xa9\xc3\xa9", 8}, {"\x80\xc3\xa9", 3},
+  };
+  ScrStr *encs[] = {S("latin1"), S("ascii"), S("utf16le"), S("hex"), S("base64")};
+  uint8_t buf[64];
+  char what[96];
+  for (size_t p = 0; p < sizeof patterns / sizeof patterns[0]; p++) {
+    for (size_t offset = 0; offset < 17; offset++) {
+      for (size_t t = 0; t < sizeof tails / sizeof tails[0]; t++) {
+        for (size_t lead = 0; lead < 2; lead++) {
+          /* lead: the offset is two-byte characters instead of ASCII. */
+          size_t n = 0;
+          for (size_t j = 0; j < offset; j++) {
+            if (lead && j + 1 < offset) {
+              buf[n++] = 0xc3;
+              buf[n++] = 0xa9;
+              j++;
+            } else {
+              buf[n++] = 'a';
+            }
+          }
+          memcpy(buf + n, patterns[p].bytes, patterns[p].n);
+          n += patterns[p].n;
+          memcpy(buf + n, tails[t].bytes, tails[t].n);
+          n += tails[t].n;
+          snprintf(what, sizeof what, "raw storage encodings: pattern %zu offset %zu tail %zu lead %zu",
+                   p, offset, t, lead);
+          check_raw_encodings(encs, buf, n, what);
+        }
+      }
+    }
+  }
+  /* Random bytes, mostly high, of every length up to 40. */
+  uint32_t seed = 12345;
+  for (int round = 0; round < 4000; round++) {
+    size_t n = (size_t)round % 41;
+    for (size_t j = 0; j < n; j++) {
+      seed = seed * 1103515245u + 12345u;
+      uint8_t r = (uint8_t)(seed >> 16);
+      buf[j] = (seed >> 27) & 1 ? (uint8_t)(0x80 | r) : (uint8_t)(r & 0x7f);
+    }
+    snprintf(what, sizeof what, "raw storage encodings: random round %d", round);
+    check_raw_encodings(encs, buf, n, what);
+  }
+  for (size_t e = 0; e < sizeof encs / sizeof encs[0]; e++) scr_str_release(encs[e]);
+}
+
 static void test_concat_and_u32be(void) {
   ScrArr *list = scr_arr_new(SCR_ELEM_BYTES, 0);
   ScrBytes *a = scr_bytes_new(SCR_BYTES_U8, 2);
@@ -445,6 +574,7 @@ int main(int argc, char **argv) {
   scr_bytes_test_bulk_storage();
   scr_bytes_test_buffer_pool();
   test_encodings();
+  test_raw_storage_encodings();
   test_concat_and_u32be();
   test_from_arr_and_random();
   test_zlib_roundtrip();

tests/corpus/8010-http-header-raw-bytes.ts

@@ -0,0 +1,72 @@
+// Header values hold the raw bytes a client sent, one character per byte
+// (Node decodes them as latin1). Bytes that are not well-formed UTF-8 (lone
+// continuation bytes, runs of them, truncated leads, overlong leads and
+// bytes no sequence uses) at every offset within an eight-byte word encode
+// to the same bytes as in Node with latin1, ascii, binary, hex, base64 and
+// utf16le. Strings built in the program take the same encodings with two-,
+// three- and four-byte characters at every offset.
+import { createServer } from "node:http";
+import { connect } from "node:net";
+
+const patterns: number[][] = [
+  [0x80],
+  [0x80, 0x81, 0xbf, 0x90],
+  [0xc3],
+  [0xc0, 0x80, 0xc1, 0xbf],
+  [0xe9, 0x41, 0xe9],
+  [0xf0, 0x9f, 0x41],
+  [0xe2, 0x82],
+  [0xff, 0xfe, 0xf5],
+  [0x78, 0xe9, 0x79, 0x80, 0x80, 0x7a, 0xdf],
+];
+const encodings = ["latin1", "ascii", "binary", "hex", "base64", "utf16le"] as const;
+
+function hex(bytes: Buffer): string {
+  return bytes.toString("hex");
+}
+
+function describe(label: string, value: string): void {
+  console.log(label, encodings.map((encoding) => `${encoding}=${hex(Buffer.from(value, encoding))}`).join(" "));
+}
+
+const names: string[] = [];
+const head: Buffer[] = [Buffer.from("GET / HTTP/1.1\r\nHost: x\r\nConnection: close\r\n")];
+for (let p = 0; p < patterns.length; p++) {
+  for (let offset = 0; offset < 10; offset++) {
+    for (const tail of ["", "bcdefghij"]) {
+      const name = `x-${p}-${offset}-${tail.length}`;
+      names.push(name);
+      head.push(Buffer.from(`${name}: ${"a".repeat(offset)}`));
+      head.push(Buffer.from(patterns[p]!));
+      head.push(Buffer.from(`${tail}\r\n`));
+    }
+  }
+}
+head.push(Buffer.from("\r\n"));
+
+const server = createServer((req, res) => {
+  for (const name of names) {
+    const value = req.headers[name];
+    if (typeof value === "string") describe(name, value);
+    else console.log(name, "missing");
+  }
+  res.end("ok");
+});
+
+server.listen(0, "127.0.0.1", () => {
+  const address = server.address();
+  const port = typeof address === "object" && address !== null ? address.port : 0;
+  const socket = connect({ port, host: "127.0.0.1" }, () => {
+    socket.write(Buffer.concat(head));
+  });
+  socket.on("data", () => {});
+  socket.on("close", () => {
+    server.close();
+    for (const text of ["é", "éü", "߀", "😀", "é😀ü", "߿\u0080"]) {
+      for (let offset = 0; offset < 17; offset++) {
+        const value = "a".repeat(offset) + text + "b".repeat(offset % 9) + text;
+        describe(`text ${offset} ${JSON.stringify(text)}`, value);
+      }
+    }
+  });
+});

tests/harness/corpus-costs/8010-http-header-raw-bytes.ts.json

@@ -0,0 +1 @@
+{ "plain": 3468, "san": 7463 }