Guard the X509 dynamic-value fence against recursive types

refuseX509DynValues walks a converted value's type looking for an
X509Certificate record, and it memoized only records (by shape id) and
unions (by union id). A type that reaches itself through an array, set,
map or promise has no id to memoize by, so `type Rec = string | Rec[]`
in a program that also holds a certificate recursed until the compiler
ran out of stack — a crash, not a diagnostic.

The walk now keeps an identity-based visited set over those kinds: a type
met again while it is still being visited answers false, the least fixed
point of reachability, since nothing outside the cycle makes a
certificate reachable. Unions keep their memo, which now also covers a
union cycle that names one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/compiler/src/frontend/lowering/builtins/x509.ts

@@ -62,6 +62,22 @@ export function refuseX509DynValues(lowerer: Lowerer, functions: readonly IrFunc
   )
     return;
   const seen = new Map<string, boolean>();
+  // A type is recursive when it reaches itself without naming a shape or a
+  // union: `type Rec = string | Rec[]` passes through the array, set, map
+  // or promise case below. Those kinds have no id to memoize by, so a type
+  // met again while it is still being visited answers false: what a cycle
+  // adds to `holds` (a certificate exists somewhere in the type) is
+  // reachability, and nothing outside the cycle makes it reachable.
+  const visiting = new Set<IrType>();
+  const descend = (t: IrType, visit: () => boolean): boolean => {
+    if (visiting.has(t)) return false;
+    visiting.add(t);
+    try {
+      return visit();
+    } finally {
+      visiting.delete(t);
+    }
+  };
   const holds = (t: IrType): boolean => {
     switch (t.kind) {
       case "record": {
@@ -79,13 +95,24 @@ export function refuseX509DynValues(lowerer: Lowerer, functions: readonly IrFunc
       }
       case "array":
       case "set":
-        return holds(t.elem);
+        return descend(t, () => holds(t.elem));
       case "map":
-        return holds(t.key) || holds(t.value);
+        return descend(t, () => holds(t.key) || holds(t.value));
       case "promise":
-        return holds(t.inner);
-      case "union":
-        return lowerer.unions.get(t.unionId)?.arms.some(holds) === true;
+        return descend(t, () => holds(t.inner));
+      case "union": {
+        // Unions have an id, so a cycle that names one is answered from the
+        // memo above; the identity guard covers a cycle that does not.
+        const key = `union:${t.unionId}`;
+        const known = seen.get(key);
+        if (known !== undefined) return known;
+        return descend(t, () => {
+          seen.set(key, false);
+          const result = lowerer.unions.get(t.unionId)?.arms.some(holds) === true;
+          seen.set(key, result);
+          return result;
+        });
+      }
       default:
         return false;
     }

packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8420-x509-recursive-dyn-type.ts.json

@@ -0,0 +1,6 @@
+{
+ "order": [
+  "<repo>/tests/corpus/8420-x509-recursive-dyn-type.ts"
+ ],
+ "diags": []
+}

tests/corpus/8420-x509-recursive-dyn-type.ts

@@ -0,0 +1,42 @@
+// A recursive type that cycles through an array and a set, next to a
+// certificate: the fence that refuses an X509Certificate in an `unknown`
+// slot (x509.ts) walks the converted value's type, and a cycle through
+// anything but a record used to recurse until the stack ran out — a crash,
+// not a diagnostic. The types are named here the way a JSON-ish union is,
+// and the certificate is the one 7090 parses.
+import { X509Certificate } from "node:crypto";
+
+// A self-signed certificate without alternative names (as in 7090).
+const PLAIN = `-----BEGIN CERTIFICATE-----
+MIIC8jCCAdoCAQcwDQYJKoZIhvcNAQELBQAwPjEaMBgGA1UEAwwRcGxhaW4uZXhh
+bXBsZS5vcmcxIDAeBgkqhkiG9w0BCQEWEXBsYWluQGV4YW1wbGUub3JnMCAXDTI2
+MTAyMjA5NDY0MVoYDzIxMjYwOTI4MDk0NjQxWjA+MRowGAYDVQQDDBFwbGFpbi5l
+eGFtcGxlLm9yZzEgMB4GCSqGSIb3DQEJARYRcGxhaW5AZXhhbXBsZS5vcmcwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCq01NcNAy9Ng04mVuAFvmZUvsm
+rY7aOaMgtODmptulgZ2qw25E09RgGYCn1S+4IYDu9hH9X4s+28Ag7Khmj4nmeoXN
+of5095wche8DR2AAUElqyr/Nmgm2kTLdIJX4wl0eWXHIlmSPdOUNovA6JjGKH99b
+8W7a2TqY/TsSadpPSvvN2esr7Jr4KgW7MXPUOnwNbcBOVcQP9rtFrduAWz01q0HM
+02LboclnUnk/e58BXAj8vgdVojljDcc0bhbdz/Hh1/5gaaNILjJNOd4AgwZaCT02
+KojGz4Nb6MSgFmON2iMgAZjY1fXXkJ7wsfVPytjW73e7uV/RFuqSHQZenxupAgMB
+AAEwDQYJKoZIhvcNAQELBQADggEBAJX1DGuiSLoCKBDI3oCgAu7X86G01ovEgZJr
+8JrmGHW5q7ujjo53t46h8XPm3LJ8UV9/MZ+Yry43xB2txXYv3b3g/i+7JdTWLgUD
+INA5VWvVkEZfm0lBEpU1Agpy4lV5uUCKZTgP/3MY5hnJ0YhQIs4BOjbwueMRFshL
+yyHVb73MMbpWaZ7+T368seafa+ekeuY6gQly/HacuszjoYd6KKV3n8eDewenClCF
+HoOF1Y4I+CvmR+a6B3iBczJ+a4tceFIIWsZUIeqzvGHpM7G9YwlmbvGprfLgR8Nv
+igUib/D9tU3hdL6VWN5nMjdXV/8Le4nDnBjVBjYDkVbsnmT91AQ=
+-----END CERTIFICATE-----
+`;
+
+type Rec = string | Rec[];
+type RecSet = number | Set<RecSet>;
+
+const cert = new X509Certificate(PLAIN);
+const rec: Rec = ["a", ["b"]];
+const set: RecSet = new Set([1, 2]);
+
+function describe(value: unknown): string {
+  return typeof value;
+}
+
+console.log(describe(rec), rec.length, describe(set), set.size);
+console.log(cert.subject);

tests/harness/corpus-costs/8420-x509-recursive-dyn-type.ts.json

@@ -0,0 +1 @@
+{ "plain": 2100, "san": 1600 }