Process non-ASCII URL hosts with UTS #46 as Node does

docs/content/limitations/builtin-apis/url.mdx

@@ -1,5 +1,4 @@
-- **URL:** Hosts of special URLs that hold non-ASCII text, or percent-escapes that decode to it, throw `ERR_INVALID_URL`, and `xn--` labels are kept without Punycode validation.
-  Absolute inputs and relative inputs with runtime string or URL bases, including optional bases, compile.
+- **URL:** Absolute inputs and relative inputs with runtime string or URL bases, including optional bases, compile.
   `URL.canParse`, `URL.parse`, `toString()`, and `toJSON()` compile; the factories use the same parser as construction, returning `false` or `null` for rejected inputs while preserving conversion errors.
   `href`, `protocol`, `username`, `password`, `host`, `hostname`, `port`, `pathname`, `search`, and `hash` support assignment in native code.
   Invalid component writes are ignored except for invalid `href` values, which throw; credentials are percent-encoded and ports accept valid leading digits.

packages/compiler/src/backend/llvm/emit-module.ts

@@ -344,6 +344,8 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
   // Fetch-referencing programs register native fetch before main.
   // Runtime selection compiles scr_fetch.c on the same predicate.
   const usesFetch = runtimeFeatures.fetch;
+  // URL programs give the host parser the IDNA unit's ToASCII.
+  const usesUrlIdna = runtimeFeatures.urlIdna;
   const snapshotsTlsCa = runtimeFeatures.tls || runtimeFeatures.tlsCa;
   // The process verdict has the same precedence as the C reference
   // emitter: node:test owns the final status when present; otherwise an
@@ -396,6 +398,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
   if (usesHttp2) emitter.declare(`declare void @scr_http2_dyn_install()`);
   if (usesHttp) emitter.declare(`declare void @scr_http_dyn_install()`);
   if (usesFetch) emitter.declare(`declare void @scr_fetch_install()`);
+  if (usesUrlIdna) emitter.declare(`declare void @scr_url_idna_install()`);
 
   if (usesNodeTest) {
     emitter.declare(`declare i32 @scr_test_exit_code()`);
@@ -1027,6 +1030,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
     ...(emitter.ffiHasForeignCallback ? [`  call void @scr_ffi_install()`] : []),
     ...(snapshotsTlsCa ? [`  call void @scr_tls_ca_install()`] : []),
     ...(usesFetch ? [`  call void @scr_fetch_install()`] : []),
+    ...(usesUrlIdna ? [`  call void @scr_url_idna_install()`] : []),
     ...(usesNet ? [`  call void @scr_net_install()`, `  call void @scr_net_dyn_install()`] : []),
     ...(usesNetLookup ? [`  call void @scr_net_lookup_install()`] : []),
     ...(usesHttp ? [`  call void @scr_http_dyn_install()`] : []),

packages/compiler/src/backend/vendor-archives.ts

@@ -14,6 +14,10 @@ import type { CcDriver } from "./native/driver.js";
 
 const execFileAsync = promisify(execFile);
 
+/** The section flags the libregexp/libunicode objects build with (the
+ * runtime pack's executable_section_elimination flags). */
+const LRE_SECTION_FLAGS = ["-ffunction-sections", "-fdata-sections"];
+
 import {
   QJS_COMMIT,
   MBEDTLS_VERSION,
@@ -191,13 +195,13 @@ export function createVendorArchives(context: VendorArchiveContext) {
       (sanitize ? "asan" : "plain") +
       (driver.argv.length === 1 && driver.argv[0] === "clang" ? "" : "-zigcc") +
       `-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
-    const cacheDir = join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-lre-${flavor}`);
+    const cacheDir = join(cacheRoot, `${QJS_COMMIT.slice(0, 12)}-lre-sections-${flavor}`);
     return LRE_SOURCES.map((f) => join(cacheDir, f.replace(/\.c$/, ".o")));
   }
 
   /** The libregexp objects for one flavor, compiled lazily on the first
    * regex-using build (~1s) and cached under
-   * <build-cache>/vendor/<commit>-lre-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
+   * <build-cache>/vendor/<commit>-lre-sections-<flavor>-<target>-<toolchain>/*.o — with the same atomic-rename
    * publish (parallel first builds race safely; losers discard their work).
    * Plain is -Os to keep regex-using binaries small; asan matches the final link so
    * the sanitized lane instruments the matcher too. Cross targets get their
@@ -228,6 +232,10 @@ export function createVendorArchives(context: VendorArchiveContext) {
       vendor: vendorEngineDir(),
       sources: LRE_SOURCES,
       objects: lreObjectPaths(sanitize, driver, buildIdentity, cacheRoot),
+      // Section elimination keeps only the libunicode tables a program
+      // reaches: the URL host parser's NFC links the normalization tables
+      // without the regex property ones.
+      flags: LRE_SECTION_FLAGS,
     });
   }
 
@@ -296,6 +304,7 @@ export function createVendorArchives(context: VendorArchiveContext) {
     vendor: string;
     sources: readonly string[];
     objects: string[];
+    flags?: readonly string[];
   }): Promise<string[]> {
     const { sanitize, driver, cacheRoot, flavor, name, vendor, sources, objects } = options;
     if ((await Promise.all(objects.map(validVendorArtifact))).every(Boolean)) return objects;
@@ -312,6 +321,7 @@ export function createVendorArchives(context: VendorArchiveContext) {
             "-std=c11",
             ...driver.targetArgs,
             ...(sanitize ? ["-O1", "-fsanitize=address"] : ["-Os"]),
+            ...(options.flags ?? []),
             "-I",
             vendor,
             "-c",

packages/compiler/src/ir/runtime-features.ts

@@ -715,6 +715,10 @@ interface RuntimeFeatureRule {
 /** Gives every rule in the table one type. */
 const rule = (r: RuntimeFeatureRule): RuntimeFeatureRule => r;
 
+/* The node:url and WHATWG URL entry points: every URL value comes from
+ * one, and parsing one can reach the UTS #46 host processing. */
+const URL_PREFIXES: readonly string[] = ["url."];
+
 /* The HTTP, HTTPS, HTTP/2 and TLS surfaces, which link the net unit. */
 const NET_PREFIXES: readonly string[] = ["net.", "http.", "tls.", "https.", "http2."];
 
@@ -1063,9 +1067,10 @@ const RUNTIME_FEATURE_RULES = {
   /** scr_url_legacy.c parses and stringifies queries. */
   qs: rule({ fns: [...QS_LIB_FNS, ...LEGACY_URL_LIB_FNS] }),
   /** scr_url_legacy.c lowercases and normalizes hostnames with the regex
-   * unit's libunicode, NODE_DEBUG matching in scr_util_callable.c and
-   * readline's terminal unit use the regex unit, and locale case mapping
-   * runs its default conversion. */
+   * unit's libunicode, the WHATWG URL host parser runs UTS #46 through
+   * scr_idna.c (which links with the regex unit), NODE_DEBUG matching in
+   * scr_util_callable.c and readline's terminal unit use the regex unit,
+   * and locale case mapping runs its default conversion. */
   regex: rule({
     fns: [
       ...REGEX_LIB_FNS,
@@ -1074,13 +1079,17 @@ const RUNTIME_FEATURE_RULES = {
       "rl.terminal",
       "str.localeCase",
     ],
+    prefixes: URL_PREFIXES,
     expr: (node, _tables) =>
       node.kind === "regexLit" ||
       node.kind === "regexIntrinsic" ||
       (node.kind === "strIntrinsic" && CASE_MAPPING_STR_METHODS.has(node.method)),
-    types: ["regex"],
+    types: ["regex", "url"],
   }),
   searchParams: rule({ fns: ["url.searchParams"], prefixes: ["sp."], types: ["searchParams"] }),
+  /** URL programs install the IDNA unit's ToASCII for the WHATWG host
+   * parser (scr_url_idna_install). */
+  urlIdna: rule({ prefixes: URL_PREFIXES, types: ["url"] }),
   /** stdin setEncoding / string data listeners / stdio 'resize' events. A
    * typed string 'data' listener takes setEncoding-decoded chunks; the
    * process stream object's setEncoding and 'resize' members are reached
@@ -1177,6 +1186,7 @@ function noRuntimeFeatures(): RuntimeFeatures {
     qs: false,
     regex: false,
     searchParams: false,
+    urlIdna: false,
     stdioExtended: false,
     stream: false,
     streamDyn: false,

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8105-url-idna-hosts.js.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8105-url-idna-hosts.js"
+ ],
+ "diags": []
+}

packages/runtime-pack-common/scripts/build.mjs

@@ -386,6 +386,10 @@ async function build() {
               ? [
                   await archive("libregexp", lreSources, quickjs, [
                     ...vendorTarget,
+                    // Section elimination keeps only the libunicode tables a
+                    // program reaches: the URL host parser's NFC links the
+                    // normalization tables without the regex property ones.
+                    ...matrix.executable_section_elimination.compile_flags,
                     "-std=c11",
                     "-Os",
                     "-I",

packages/runtime/src/scr_idna.c

@@ -1,8 +1,8 @@
 /* UTS #46 domain processing for node:url — the conversions Node applies to
- * url.parse() hostnames (ToASCII), url.domainToASCII()/domainToUnicode()
- * and url.format()'s unicode option: nontransitional mapping, NFC, label
- * validity (leading combining marks, ContextJ, and the RFC 5893 bidi rule),
- * and Punycode. It matches Node 24's implementation, including its
+ * WHATWG URL hosts and url.parse() hostnames (ToASCII),
+ * url.domainToASCII()/domainToUnicode() and url.format()'s unicode option:
+ * nontransitional mapping, NFC, label validity (leading combining marks,
+ * ContextJ, and the RFC 5893 bidi rule), and Punycode. It matches Node 24's implementation, including its
  * validity-data snapshot and the shortcuts its label checks take; the
  * differential corpus and test/url-legacy-cases.txt (generated from Node)
  * are the oracle.
@@ -511,3 +511,6 @@ ScrStr *scr_idna_to_unicode(const char *p, size_t n) {
   }
   return scr_jb_finish(&out);
 }
+
+/* The URL host parser's UTS #46 ToASCII (scr_url.c). */
+void scr_url_idna_install(void) { scr_url_host_to_ascii = &scr_idna_to_ascii; }

packages/runtime/src/scr_url.c

@@ -21,10 +21,8 @@
  *     case preserved, C0-encoded), a single "/" roots a normalized path,
  *     anything else is an OPAQUE path with the C0 control set encoded
  *     (data:text/plain,hi there)
- *
- * DOCUMENTED DIVERGENCE: a special host that holds non-ASCII text after
- * percent-decoding is rejected with "Invalid URL" (no UTS #46 processing),
- * and "xn--" labels are kept without Punycode validation.
+ *   - UTS #46 processing of non-ASCII hosts through the IDNA unit
+ *     (scr_url_idna_install)
  *
  * Failures THROW catchable TypeErrors through the exception cell with
  * Node's messages; callers are compiler-emitted pending checks. */
@@ -430,16 +428,21 @@ static int hex_val(char c) {
   return -1;
 }
 
+/* UTS #46 ToASCII (scr_idna.c), which programs that use URLs install at
+ * startup (scr_url_idna_install). NULL refuses hosts that hold non-ASCII
+ * text. */
+ScrStr *(*scr_url_host_to_ascii)(const char *p, size_t n) = NULL;
+
 /* The URL Standard's host parser for the constructor and the setters,
  * with canonical IPv6 serialization:
  * - a non-special (opaque) host refuses forbidden host code points and
  *   percent-encodes the C0 control set;
- * - a special host is percent-decoded, lowercased, refused when it holds a
- *   forbidden domain code point, and serialized as dotted decimal when it
- *   ends in a number (or refused when that number is invalid).
- * Divergence: a special host that decodes to non-ASCII needs UTS #46
- * processing, which this unit does not link, so it is refused; "xn--"
- * labels are kept without Punycode validation. */
+ * - a special host is percent-decoded and lowercased; when that leaves
+ *   non-ASCII text it goes through UTS #46 ToASCII (as Node's host parser
+ *   does, an ASCII host, "xn--" labels included, is only lowercased). The
+ *   result is refused when it holds a forbidden domain code point, and
+ *   serialized as dotted decimal when it ends in a number (or refused when
+ *   that number is invalid). */
 static bool parse_host(const char *hp, size_t host_len, bool special, bool is_file,
                        ScrStr **host) {
   bool ipv6 = host_len > 0 && hp[0] == '[';
@@ -467,6 +470,7 @@ static bool parse_host(const char *hp, size_t host_len, bool special, bool is_fi
     *host = ub_take(&hb);
     return true;
   }
+  bool non_ascii = false;
   for (size_t i = 0; i < host_len; i++) {
     unsigned char c = (unsigned char)hp[i];
     int hi, lo;
@@ -475,12 +479,27 @@ static bool parse_host(const char *hp, size_t host_len, bool special, bool is_fi
       c = (unsigned char)(hi * 16 + lo);
       i += 2;
     }
-    if (url_host_byte_class[c] & URL_FORBIDDEN_DOMAIN) {
+    if (c >= 0x80) non_ascii = true;
+    if (c >= 'A' && c <= 'Z') c = (unsigned char)(c - 'A' + 'a');
+    ub_push(&hb, (char)c);
+  }
+  if (non_ascii) {
+    /* UTS #46 ToASCII, or refused without the IDNA unit. */
+    ScrStr *ascii = scr_url_host_to_ascii ? scr_url_host_to_ascii(hb.data, hb.len) : NULL;
+    free(hb.data);
+    if (!ascii || ascii->len == 0) {
+      scr_str_release(ascii);
+      return false;
+    }
+    ub_init(&hb);
+    ub_append(&hb, ascii->data, ascii->len);
+    scr_str_release(ascii);
+  }
+  for (size_t i = 0; i < hb.len; i++) {
+    if (url_host_byte_class[(unsigned char)hb.data[i]] & URL_FORBIDDEN_DOMAIN) {
       free(hb.data);
       return false;
     }
-    if (c >= 'A' && c <= 'Z') c = (unsigned char)(c - 'A' + 'a');
-    ub_push(&hb, (char)c);
   }
   if (url_ends_in_number(hb.data, hb.len)) {
     UrlBuf ip;

packages/runtime/src/scr_url.h

@@ -46,6 +46,10 @@ void scr_url_set_checked(ScrUrl *u, ScrStr *field, const ScrDyn *value); /* borr
  * null and has no lowering for an object that is not a URL. Borrowed; +1,
  * or NULL with the error pending. */
 ScrDyn *scr_url_to_http_options(ScrUrl *u);
+/* UTS #46 ToASCII for the host parser: NULL until scr_url_idna_install
+ * (scr_idna.c, emitted at startup by programs that use URLs) sets it. */
+extern ScrStr *(*scr_url_host_to_ascii)(const char *p, size_t n);
+void scr_url_idna_install(void);
 ScrDyn *scr_url_to_http_options_dyn(const ScrDyn *value);
 /* node:url's legacy surface (scr_url_legacy.c — linked with the regex,
  * querystring and dyn-async units). url.parse() answers a checked-dynamic

tests/corpus/8105-url-idna-hosts.js

@@ -0,0 +1,58 @@
+// WHATWG URL hosts that hold non-ASCII text, directly or after
+// percent-decoding, go through UTS #46 ToASCII as the host parser in Node
+// does: mapped and normalized labels become Punycode, and disallowed code
+// points, invalid joiners and bidi violations make the URL invalid. Cases
+// from the WPT url test data, plus setters, pathToFileURL UNC hosts and
+// URL.canParse.
+import url from "node:url";
+
+const parses = [
+  ["http://é@é", null],
+  ["http://GOO\u200B\u2060\uFEFFgoo.com", "http://other.com/"],
+  ["http://www.foo。bar.com", "http://other.com/"],
+  ["http://Ｇｏ.com", "http://other.com/"],
+  ["http://你好你好", "http://other.com/"],
+  ["https://faß.ExAmPlE/", null],
+  ["http://０Ｘｃ０．０２５０．０１", "http://other.com/"],
+  ["ftp://%e2%98%83", null],
+  ["https://%e2%98%83", null],
+  ["file://a\u00ADb/p", null],
+  ["file://a%C2%ADb/p", null],
+  ["file://loC𝐀𝐋𝐇𝐨𝐬𝐭/usr/bin", null],
+  ["https://a%C2%ADb/", null],
+  ["http://münchen.de/straße", null],
+  ["https://日本語.jp/", null],
+  ["http://例え.テスト/", null],
+  ["http://xn--mnchen-3ya.de/", null],
+  ["http://a\u200Db.com/", null],
+  ["http://אa.com/", null],
+  ["http://ab�c/", null],
+  ["ws://ÉXAMPLE.org:8080/x", null],
+];
+function describe(value) {
+  return [value.href, value.host, value.hostname, value.origin].join(" | ");
+}
+for (const [input, base] of parses) {
+  let line;
+  try {
+    line = describe(base === null ? new URL(input) : new URL(input, base));
+  } catch (e) {
+    line = `throws ${e.code}`;
+  }
+  console.log(JSON.stringify(input), "=>", line, URL.canParse(input, base ?? undefined));
+}
+const writes = [
+  ["host", "https://x/", "ß"],
+  ["host", "https://example.com/", "a%C2%ADb"],
+  ["hostname", "https://example.com/", "a%C2%ADb"],
+  ["hostname", "https://example.com/", "BÜCHER.example"],
+];
+for (const [name, href, value] of writes) {
+  const target = new URL(href);
+  if (name === "host") target.host = value;
+  else target.hostname = value;
+  console.log(name, JSON.stringify(value), "=>", describe(target));
+}
+for (const path of ["\\\\münchen\\share\\f", "\\\\BÜCHER\\s\\f"]) {
+  console.log(JSON.stringify(path), url.pathToFileURL(path, { windows: true }).href);
+}