From cd8a5543742424fa4a384ab186abfefca5e03d70 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 9 Aug 2026 21:22:56 -0500 Subject: [PATCH] fix(runtime): match Node malformed text decoding --- packages/runtime/src/scr_bytes.c | 54 +++++++++++++++++++++++--------- tests/corpus/1423-text-codec.ts | 12 +++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index cb17f4323..9c17b8256 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -835,6 +835,11 @@ static ScrStr *scr_td_utf16(const ScrBytes *b, bool be) { scr_td_put(&out, 0x10000 + ((cu - 0xd800) << 10) + (lo - 0xdc00)); continue; } + } else if (i < n) { + /* ICU treats a lead surrogate plus one final byte as one malformed + * UTF-16 subsequence. Consume that byte here so the odd-tail path + * below does not emit a second replacement. */ + i++; } scr_td_error(&out); continue; @@ -1108,11 +1113,18 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { enum ScrTdIsoState state = SCR_TD_ISO_ASCII; enum ScrTdIsoState output_state = SCR_TD_ISO_ASCII; uint8_t lead = 0; + int replay = -1; bool output_flag = false; size_t i = 0; bool at_eof = false; while (!at_eof) { - int item = i < b->len ? b->data[i++] : -1; + int item; + if (replay >= 0) { + item = replay; + replay = -1; + } else { + item = i < b->len ? b->data[i++] : -1; + } uint8_t byte = item < 0 ? 0 : (uint8_t)item; /* ICU treats CR/LF as line boundaries while a non-ASCII designation is * active: emit the separator and resume in ASCII. A pending JIS lead is @@ -1151,6 +1163,17 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { if (byte >= 0x21 && byte <= 0x7e) { output_flag = false; lead = byte; state = SCR_TD_ISO_TRAIL; break; } + /* ICU's fixed-width JIS converter folds two adjacent non-starter + * bytes into one malformed subsequence. A valid lead, ESC, and the + * SO/SI controls begin their own item and must remain queued; CR/LF + * following an invalid byte is payload here rather than the line + * reset handled above. */ + if (byte != 0x0e && byte != 0x0f && i < b->len) { + uint8_t next = b->data[i]; + bool starts_item = next == 0x0e || next == 0x0f || next == 0x1b || + (next >= 0x21 && next <= 0x7e); + if (!starts_item) i++; + } output_flag = false; scr_td_error(&out); break; case SCR_TD_ISO_TRAIL: @@ -1160,7 +1183,13 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { if (byte >= 0x21 && byte <= 0x7e) { uint32_t cp = scr_td_jis0208[(lead - 0x21) * 94 + byte - 0x21]; if (cp) scr_td_put(&out, cp); else scr_td_error(&out); - } else scr_td_error(&out); + } else { + scr_td_error(&out); + /* SO/SI are standalone illegal items in ICU's JIS state rather + * than trails consumed with the pending lead. Replay them so they + * each contribute their own replacement. */ + if (byte == 0x0e || byte == 0x0f) i--; + } break; case SCR_TD_ISO_ESCAPE_START: @@ -1187,7 +1216,7 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { if ((int)next >= 0) { state = output_state = next; bool repeated = output_flag; - output_flag = true; + output_flag = !repeated; if (repeated) scr_td_error(&out); break; } @@ -1201,6 +1230,9 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { (lead == 0x2e && (byte == 0x41 || byte == 0x46)); if (item >= 0 && lead == 0x26 && byte == 0x40) { state = output_state; + bool repeated = output_flag; + output_flag = !repeated; + if (repeated) scr_td_error(&out); break; } if (item < 0 || consume_error) { @@ -1208,20 +1240,12 @@ static ScrStr *scr_td_iso_2022_jp_decode(const ScrBytes *b) { if (item < 0) at_eof = true; break; } - if (item >= 0) i--; + i--; /* The first escape payload byte is restored before the current - * byte. Process it immediately in the prior output state. */ - uint8_t restore = lead; + * byte. Replay it through the full prior state machine: in JIS + * mode it can become the lead paired with the current byte. */ + replay = lead; output_flag = false; state = output_state; scr_td_error(&out); - if (restore == 0x1b) { state = SCR_TD_ISO_ESCAPE_START; } - else if (state == SCR_TD_ISO_ROMAN && restore == 0x5c) scr_td_put(&out, 0x00a5); - else if (state == SCR_TD_ISO_ROMAN && restore == 0x7e) scr_td_put(&out, 0x203e); - else if (state == SCR_TD_ISO_KATAKANA && restore >= 0x21 && restore <= 0x5f) - scr_td_put(&out, 0xff61 + restore - 0x21); - else if (state != SCR_TD_ISO_KATAKANA && restore < 0x80 && restore != 0x0e && restore != 0x0f) - scr_td_put(&out, restore); - else scr_td_error(&out); - if (item < 0) at_eof = true; break; } } diff --git a/tests/corpus/1423-text-codec.ts b/tests/corpus/1423-text-codec.ts index e16962fff..ecb67b3eb 100644 --- a/tests/corpus/1423-text-codec.ts +++ b/tests/corpus/1423-text-codec.ts @@ -89,6 +89,8 @@ console.log(JSON.stringify(new TextDecoder("x-user-defined").decode(new Uint8Arr console.log(new TextDecoder("utf-16le").decode(new Uint8Array([0xff, 0xfe, 0x61, 0x00, 0x3d, 0xd8, 0x00, 0xde]))); console.log(new TextDecoder("unicodefffe").decode(new Uint8Array([0xfe, 0xff, 0x00, 0x61, 0xd8, 0x3d, 0xde, 0x00]))); console.log(JSON.stringify(new TextDecoder("utf-16le").decode(new Uint8Array([0x00, 0xd8, 0x61, 0x00, 0xff])))); +console.log(JSON.stringify(new TextDecoder("utf-16le").decode(new Uint8Array([0x00, 0xd8, 0x12])))); +console.log(JSON.stringify(new TextDecoder("utf-16be").decode(new Uint8Array([0xd8, 0x00, 0x12])))); // The multibyte families: two- and four-byte GB, traditional Chinese, // Japanese stateful/stateless forms, and Korean. @@ -122,6 +124,16 @@ const recoverySamples = [ new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x41])), new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x4f])), new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x28, 0x48, 0x5c, 0x7e])), + // In JIS mode ICU groups adjacent invalid payload bytes, but SO/SI begin + // standalone errors; a grouped CR/LF does not perform the line reset. + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x00, 0x00])), + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x21, 0x0e])), + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x00, 0x0a, 0x41])), + // Malformed escapes restore their payload in the active JIS state; + // consecutive designations toggle ICU's replacement flag. + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x1b, 0x24, 0x21])), + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x1b, 0x24, 0x42, 0x1b, 0x24, 0x42])), + new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x26, 0x40, 0x1b, 0x24, 0x42, 0x46, 0x7c])), // ICU emits line separators in JIS/Katakana states and resumes in ASCII. new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x24, 0x42, 0x46, 0x7c, 0x0a, 0x41])), new TextDecoder("iso-2022-jp").decode(new Uint8Array([0x1b, 0x28, 0x49, 0x21, 0x0d, 0x41])),