diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 052811da90..0b6748edf7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,11 +101,11 @@ jobs: if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`); NODE - # Collects the native compiler's profile on this commit - # (scripts/refresh-cli-profile.mjs), so every platform's CLI build optimizes - # with a profile of the code it compiles. The committed profile falls behind - # main within a day or two. If this job fails, the CLI builds use the - # committed profile. + # Collects the native compiler's profile and function layout on this commit + # (scripts/refresh-cli-profile.mjs, scripts/refresh-cli-layout.mjs), so every + # platform's CLI build optimizes and lays out the code it compiles. The + # committed ones fall behind main within a day or two. If this job fails, + # the CLI builds use the committed profile and layout. cli-profile: name: Collect the native compiler profile needs: check-release @@ -139,11 +139,17 @@ jobs: - name: Collect the profile run: >- node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 - --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" + --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" + - name: Collect the function layout + run: >- + node scripts/refresh-cli-layout.mjs --jobs 4 --instrumented "$RUNNER_TEMP/cli-profile-work/cli" + --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" - uses: actions/upload-artifact@v4 with: name: cli-profile - path: ${{ runner.temp }}/cli-profile/cli.profdata.gz + path: | + ${{ runner.temp }}/cli-profile/cli.profdata.gz + ${{ runner.temp }}/cli-profile/cli.layout.gz retention-days: 7 if-no-files-found: error @@ -336,8 +342,8 @@ jobs: with: name: cli-profile path: ${{ runner.temp }}/cli-profile - # Without the artifact, SCRIPTC_NATIVE_PROFILE_USE stays unset and - # build-native-cli.mts uses the committed profile. + # Without the artifact, SCRIPTC_NATIVE_PROFILE_USE and SCRIPTC_NATIVE_LAYOUT + # stay unset and build-native-cli.mts uses the committed profile and layout. - name: Select the native compiler profile if: matrix.cli != '' shell: bash @@ -347,6 +353,11 @@ jobs: if [ -n "$COLLECTED" ]; then echo "SCRIPTC_NATIVE_PROFILE_USE=$COLLECTED" >> "$GITHUB_ENV" echo "The native CLI builds with the profile collected on $GITHUB_SHA" + layout="$(dirname "$COLLECTED")/cli.layout.gz" + if [ -f "$layout" ]; then + echo "SCRIPTC_NATIVE_LAYOUT=$layout" >> "$GITHUB_ENV" + echo "The native CLI builds with the function layout collected on $GITHUB_SHA" + fi else echo "::warning::No profile was collected on this commit; the native CLI builds with the committed profile" fi diff --git a/RELEASING.md b/RELEASING.md index 42bfe75add..8eac23a8ed 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,6 +18,8 @@ The release job builds each pinned LLVM helper, precompiled runtime pack, and na Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged sparse profile. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. +Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`, reusing the job's instrumented compiler), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Refresh the committed layout the same way as the profile, with `node scripts/refresh-cli-layout.mjs`, and commit it on its own. + Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for `release.yml` and the `Release` environment. A missing configuration fails diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index 5afa6ff730..0a82775ca3 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -170,7 +170,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where %p expands to the process ID; the LLVM_PROFILE_FILE environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. Valid only for executable builds with --optimization=release or speed.
--profile-use <file.profdata>
-
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. Valid only for executable builds with --optimization=release or speed.
+
Optimize the executable with a profile from scriptc merge-profiles: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. scriptc build prints how the profile matched the program's functions on stderr; see Profile-guided optimization. Valid only for executable builds with --optimization=release or speed.
--npm-static <pkg[,pkg…]|auto>
Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing ..., as in fs-extra..., also names every package that package depends on. auto uses the default automatic selection. Packages that can't compile produce diagnostics. See npm Dependencies.
@@ -262,6 +262,25 @@ $ scriptc build app.ts --optimization=speed --profile-use app.profdata Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time. +`scriptc build --profile-use` prints one line on stderr with how the profile matched the program's functions: + +``` +scriptc: profile app.profdata: 9170 functions annotated, 0 mismatched, 28636 without a record +``` + +- **annotated:** functions optimized with the profile's counts. +- **mismatched:** functions whose control flow changed since the profile was collected. They are optimized without counts. +- **without a record:** functions the profile does not describe: new functions, and functions the profiled runs never reached. +- When the build reuses cached partitions whose counts were not recorded, the line ends with the number of those partitions. + +When more than 10% of the functions the profile describes (annotated and mismatched) are mismatched, the build also prints a warning: + +``` +scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) changed since it was collected and are optimized without their counts; collect the profile again +``` + +The build still succeeds. Collecting the profile again with the current program and scriptc version restores the counts. + ## Target CPU By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it: diff --git a/packages/cli/scripts/install-native.mjs b/packages/cli/scripts/install-native.mjs index 84eb846bcb..4db76cc24c 100644 --- a/packages/cli/scripts/install-native.mjs +++ b/packages/cli/scripts/install-native.mjs @@ -49,6 +49,7 @@ export function relocateToolchain(manifest, sourceDirectory, destinationDirector "declarations", "comptime", "wasi_node_runner", + "lib_diagnostics", ]) { if (typeof relocated[name] === "string") relocated[name] = pathFrom(relocated[name]); } diff --git a/packages/cli/test/native-install.test.ts b/packages/cli/test/native-install.test.ts index c75b542679..5e51a2447f 100644 --- a/packages/cli/test/native-install.test.ts +++ b/packages/cli/test/native-install.test.ts @@ -53,6 +53,7 @@ function fixture() { dsymutil: "dsymutil", comptime: "../lib/comptime", wasi_node_runner: "../lib/wasi/cli/wasi-runner.js", + lib_diagnostics: "../lib/lib-diagnostics.json", }; writeFileSync(binary + ".json", JSON.stringify(toolchain)); const helpers = [ @@ -93,6 +94,9 @@ test("installs a direct executable and relocatable references to platform assets expect(resolve(dirname(installed), manifest.wasi_node_runner)).toBe( join(platform, "wasi/cli/wasi-runner.js"), ); + expect(resolve(dirname(installed), manifest.lib_diagnostics)).toBe( + join(platform, "lib-diagnostics.json"), + ); expect(resolve(dirname(installed), manifest.runtime_packs[0].path)).toBe( join(platform, "runtime"), ); diff --git a/packages/cli/test/profile.test.ts b/packages/cli/test/profile.test.ts index 24c759e921..da40a15b62 100644 --- a/packages/cli/test/profile.test.ts +++ b/packages/cli/test/profile.test.ts @@ -63,8 +63,16 @@ test.skipIf(process.platform !== "linux")( expect(merge.stdout).toBe(`${merged}\n`); const optimized = join(dir, "optimized"); - await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + const fresh = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\n"); + // The build reports how the profile matched the program's functions. + expect(fresh.stderr).toMatch( + /^scriptc: profile .*: [1-9]\d* functions annotated, 0 mismatched, \d+ without a record\n$/, + ); + expect(fresh.stderr).toContain(`scriptc: profile ${merged}: `); + // A build from the cache reports the same match. + const again = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + expect(again.stderr).toBe(fresh.stderr); // A sparse profile leaves out the runtime functions the runs never // reached. @@ -77,10 +85,15 @@ test.skipIf(process.platform !== "linux")( await scriptc(["build", entry, "--profile-use", sparse, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\n"); - // A profile from an older version of the program still builds it. + // A profile from an older version of the program still builds it. The + // edited function no longer matches its counts, and the build warns. await writeFile(entry, PROGRAM + 'console.log("edited");\n'); - await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); + const stale = await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], env); expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\nedited\n"); + expect(stale.stderr).toMatch(/ [1-9]\d* mismatched, /); + expect(stale.stderr).toMatch( + /\nscriptc: warning: [1-9]\d* of the \d+ functions .* describes \([\d.]+%\) changed since it was collected and are optimized without their counts; collect the profile again\n$/, + ); } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index ee2208bd90..79305a8ba9 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1718,6 +1718,16 @@ declare module "node:fs" { ): void; export function rmdirSync(path: string | URL): void; export function readdirSync(path: string | URL): string[]; + /* The recursive form: every entry below the path, relative to it. */ + export function readdirSync( + path: string | URL, + options: { + recursive?: boolean; + withFileTypes?: false; + encoding?: "utf8" | "utf-8"; + [option: string]: unknown; + }, + ): string[]; /* The withFileTypes form: Dirent rows (name + parentPath + the type * probes — the honest subset of Node's Dirent surface). The type is * interned explicitly in type-mapper.ts (the record's hidden %dtype field diff --git a/packages/compiler/native/cli.layout.gz b/packages/compiler/native/cli.layout.gz new file mode 100644 index 0000000000..4ede69568a Binary files /dev/null and b/packages/compiler/native/cli.layout.gz differ diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index 24d330cc0e..bffdf933f8 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -56,7 +56,9 @@ void scr_collect_cycles_phase(void); * frontend's syntax trees and program state refer to each other, so * reference counting frees almost none of them when the frontend is * disposed, and the collector's schedule alone may not run a full pass - * before the build ends. */ + * before the build ends. The sweep then returns the freed pages to the + * system (scr_heap_trim), so code generation does not grow on top of the + * frontend's heap. */ void scriptc_native_collect_cycles(void) { scr_collect_cycles_phase(); } @@ -80,9 +82,30 @@ void scriptc_native_unlock(void) {} #include #include -int32_t scriptc_native_fork(void) { return (int32_t)fork(); } +#if defined(__linux__) +/* LLVM's profile runtime, linked into instrumented builds of the compiler + * (SCRIPTC_NATIVE_PROFILE_GENERATE); other builds leave these null. A worker + * starts with zeroed counters and writes its profile before _exit, which + * skips the runtime's atexit writer, so the profile counts the worker's work + * once. */ +extern void __llvm_profile_reset_counters(void) __attribute__((weak)); +extern int __llvm_profile_write_file(void) __attribute__((weak)); +#endif -void scriptc_native_exit(int32_t code) { _exit(code); } +int32_t scriptc_native_fork(void) { + pid_t pid = fork(); +#if defined(__linux__) + if (pid == 0 && __llvm_profile_reset_counters != NULL) __llvm_profile_reset_counters(); +#endif + return (int32_t)pid; +} + +void scriptc_native_exit(int32_t code) { +#if defined(__linux__) + if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file(); +#endif + _exit(code); +} /* The worker's exit status, 128 plus the signal for a worker a signal * ended, or -1 when waitpid fails. */ diff --git a/packages/compiler/native/ts7-process.test.c b/packages/compiler/native/ts7-process.test.c index b6d760abd1..85de63db16 100644 --- a/packages/compiler/native/ts7-process.test.c +++ b/packages/compiler/native/ts7-process.test.c @@ -1,6 +1,10 @@ #ifndef _WIN32 #define _POSIX_C_SOURCE 200809L #endif +/* macOS declares mkdtemp only with the Darwin extensions visible. */ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) +#define _DARWIN_C_SOURCE 1 +#endif #include "ts7-process.h" #include #include diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 03af418a5d..d1c39d8aca 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -40,7 +40,7 @@ }, "dependencies": { "@scriptc/runtime": "workspace:*", - "tsc-ts": "0.0.9", + "tsc-ts": "0.0.10", "typescript": "7.0.2", "typescript5": "npm:typescript@5.9.3" }, diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index 521f1d53ec..14ffdff28b 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -166,6 +166,7 @@ export async function compilePackedLibrary( options.programPublicSymbols, stem, target.platform, + { inProcessElf: true }, ); } else { program = await emit( diff --git a/packages/compiler/src/backend/llvm/bytes-headers.ts b/packages/compiler/src/backend/llvm/bytes-headers.ts new file mode 100644 index 0000000000..f5b95cf758 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-headers.ts @@ -0,0 +1,77 @@ +/* Typed-array header fields read once at a speculative loop's entry. + * + * A view's length, data pointer and shared storage are set at construction + * and never change, and a view held by a const local or a parameter that + * the loop does not rebind is the same view in every iteration. The entry + * reads those fields once, and every element access in the loop's copies + * (the ordinary copy, the fast copy and the copy a failed check resumes in) + * uses the values. No header read is then left inside the copies for LLVM's + * load PRE to move ahead of the fast copy's exits: such a move keeps the + * fast loop's first iteration in the loop, with the checks that one + * iteration of it makes redundant. */ +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { everyStmt } from "../../ir/traverse.js"; +import { emitBytesData, emitBytesHeaderLength, emitBytesSharedStorage } from "./expr-bytes.js"; +import type { LlEmitter } from "./emitter.js"; + +export interface BytesHeader { + length: string; + data: string; + /** The view's shared storage (worker programs only). */ + shared: string | null; +} + +export interface BytesHeaderScope { + /** The function the scope's values belong to. */ + fn: string; + locals: ReadonlyMap; + /** The header of each receiver value read from one of `locals`. */ + values: Map; +} + +/** Reads the header fields of the views `loop` accesses through a const + * local or a parameter it does not rebind, at the current block, which must + * dominate every copy of the loop. Null when there are none. */ +export function emitLoopBytesHeaders( + emitter: LlEmitter, + loop: IrStmt, + parameters: ReadonlySet, +): BytesHeaderScope | null { + const receivers = new Set(); + const rebound = new Set(); + const receiver = (e: IrExpr): void => { + if (e.kind === "varRef") receivers.add(e.localId); + }; + everyStmt(loop, { + expr: (e) => { + if (e.kind === "bytesIntrinsic") receiver(e.receiver); + else if (e.kind === "assignExpr" || e.kind === "incDec") rebound.add(e.localId); + return true; + }, + stmt: (s) => { + if (s.kind === "bytesSet") receiver(s.arr); + else if (s.kind === "assign" || s.kind === "forOf" || s.kind === "varDecl") + rebound.add(s.localId); + else if (s.kind === "tryCatch" && s.catchLocalId !== null) rebound.add(s.catchLocalId); + return true; + }, + }); + const locals = new Map(); + for (const id of receivers) { + if (rebound.has(id)) continue; + const local = emitter.currentLocals.get(id); + if (local === undefined || local.boxed === true || local.type.kind !== "bytes") continue; + if (local.mutable && !parameters.has(id)) continue; + if (emitter.integerLoopBindings.has(id) || emitter.int32Locals.has(id)) continue; + const B = emitter.B; + const view = B.tmp(); + B.line(`${view} = load ptr, ptr ${emitter.binding(id).slot}${emitter.slotAlias(id)}`); + locals.set(id, { + length: emitBytesHeaderLength(emitter, view), + data: emitBytesData(emitter, view), + shared: emitter.mod.workers ? emitBytesSharedStorage(emitter, view) : null, + }); + } + if (locals.size === 0) return null; + return { fn: emitter.currentFunctionName, locals, values: new Map() }; +} diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts index 0fa8fea348..742d256b79 100644 --- a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -320,16 +320,17 @@ test("worker programs test each receiver's shared storage once in the guard", () const fast = /^bytes\.version\.fast\d+:$[^]*?^bytes\.version\.fast\.shared\d+:$/m.exec(body)![0]; expect(fast).not.toContain("bytes.shared"); expect(fast).not.toMatch(/(load|store) atomic double/); - // The checked copy keeps the shared-memory path for each access, and so - // does the copy for shared views whose bounds hold. + // The checked copy keeps the shared-memory path for each access. The copy + // for shared views whose bounds hold is a counted loop that polls between + // blocks, so its accesses at the counter are plain. const slow = body.slice(body.search(/^bytes\.version\.slow\d+:$/m), body.indexOf(fast)); expect(slow).toContain("load atomic double"); expect(slow).toContain("store atomic double"); const shared = /^bytes\.version\.fast\.shared\d+:$[^]*?^bytes\.version\.done\d+:$/m.exec( body, )![0]; - expect(shared).toContain("load atomic double"); - expect(shared).toContain("store atomic double"); + expect(shared).toMatch(/^loop\.poll\d+:$/m); + expect(shared).not.toMatch(/(load|store) atomic double/); }); test("index temps of a version's counter are integers in the fast copy", () => { diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 699755157e..74880453f7 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -573,13 +573,21 @@ export function classIdLoadLines( * 64-bit word. The count is the i32 at offset 0. */ export const CLASS_ID_OFFSET = 4; +/** The bound on layout ids (scr_rc.h SCR_RC_LAYOUT_ID_LIMIT): the high + * half of a 64-bit rc word keeps the layout id in its low 30 bits, beside + * the program-lifetime and frozen bits. A class id is the layout id of its + * class's canonical layout. */ +export const LAYOUT_ID_LIMIT = 0x3fffffff; + /** The rc word an LlClassMeta.idInRc instance is created with: the class * id, its preorder number plus 1, in the high 32 bits, and in the low 32 bits * a count of 1, or for a program-lifetime instance the midpoint of the 32-bit - * count (as programLifetimeRc is of the 32-bit word). The id is never 0, - * 0x80000000 or 0xffffffff, the high halves of other objects' rc words, so - * the runtime can tell an object with a class id (scr_rc.h scr_rc_class_id). */ + * count (as programLifetimeRc is of the 32-bit word). The id is a layout id + * (LAYOUT_ID_LIMIT), so it never sets the program-lifetime or frozen bit + * and the runtime reads it back (scr_rc.h scr_rc_layout_id). */ export function classIdRcWord(meta: LlClassMeta, lifetime: boolean): string { + if (meta.pre + 1 >= LAYOUT_ID_LIMIT) + throw new InternalCompilerError(`class id ${meta.pre + 1} is not a layout id`); return (BigInt(meta.pre + 1) * 2n ** 32n + (lifetime ? 2n ** 31n : 1n)).toString(); } diff --git a/packages/compiler/src/backend/llvm/constant-choices.test.ts b/packages/compiler/src/backend/llvm/constant-choices.test.ts new file mode 100644 index 0000000000..f19b5b19d8 --- /dev/null +++ b/packages/compiler/src/backend/llvm/constant-choices.test.ts @@ -0,0 +1,109 @@ +import { expect, test } from "vitest"; +import { BOOL, F64, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { matchConstantChoice } from "./constant-choices.js"; + +const loc = { file: "choices.ts", start: 0, end: 0 }; +const bytes = { kind: "bytes", elem: "u8" } as const; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string, type: IrExpr["type"] = F64): IrExpr => ({ + kind: "varRef", + localId, + type, + loc, +}); +const is = (left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "bin", + op: "===", + left, + right, + type: BOOL, + loc, +}); +const store = (value: IrExpr, index: IrExpr = ref("i"), arr = "a"): IrStmt => ({ + kind: "bytesSet", + arr: ref(arr, bytes), + index, + value, + loc, +}); +const branch = (cond: IrExpr, then: IrStmt[], else_: IrStmt[] | null): IrStmt => ({ + kind: "if", + cond, + then, + else_, + loc, +}); +/** `if (x === c0) a[i] = r0; else if ... else a[i] = fallback`. */ +function chain(cases: [number, number][], fallback: IrStmt[] | null, subject = "x"): IrStmt { + let rest = fallback; + for (let k = cases.length - 1; k >= 0; k--) { + const [value, result] = cases[k]!; + rest = [branch(is(ref(subject), num(value)), [store(num(result))], rest)]; + } + return rest![0]!; +} + +test("a chain of literal stores to one element is a constant choice", () => { + const choice = matchConstantChoice( + chain( + [ + [97, 0], + [116, 1], + [99, 2], + ], + [store(num(3))], + ), + ); + expect(choice?.cases).toEqual([ + { value: 97, result: 0 }, + { value: 116, result: 1 }, + { value: 99, result: 2 }, + ]); + expect(choice?.fallback).toBe(3); + expect(choice?.stores).toHaveLength(4); +}); + +test("a literal on the left, -0 and a repeated value keep strict equality's first arm", () => { + const first: IrStmt = branch( + is(num(-0), ref("x")), + [store(num(5))], + [ + chain( + [ + [1, 6], + [0, 7], + [2, 8], + ], + [store(num(9))], + ), + ], + ); + expect(matchConstantChoice(first)?.cases).toEqual([ + { value: 0, result: 5 }, + { value: 1, result: 6 }, + { value: 2, result: 8 }, + ]); +}); + +test("chains that are not one choice of literals keep their arms", () => { + const three: [number, number][] = [ + [1, 10], + [2, 20], + [3, 30], + ]; + // Two values, no final else, a fraction, or cases spanning more than 256 values. + expect(matchConstantChoice(chain(three.slice(0, 2), [store(num(0))]))).toBeNull(); + expect(matchConstantChoice(chain(three, null))).toBeNull(); + expect(matchConstantChoice(chain([...three, [1.5, 40]], [store(num(0))]))).toBeNull(); + expect(matchConstantChoice(chain([...three, [256, 40]], [store(num(0))]))).not.toBeNull(); + expect(matchConstantChoice(chain([...three, [257, 40]], [store(num(0))]))).toBeNull(); + // A final else that is not a literal, or that stores elsewhere. + expect(matchConstantChoice(chain(three, [store(ref("y"))]))).toBeNull(); + expect(matchConstantChoice(chain(three, [store(num(0), ref("j"))]))).toBeNull(); + expect(matchConstantChoice(chain(three, [store(num(0), ref("i"), "b")]))).toBeNull(); + // An arm with a second statement. + expect(matchConstantChoice(chain(three, [store(num(0)), store(num(1))]))).toBeNull(); + // A second subject. + const mixed = branch(is(ref("y"), num(0)), [store(num(1))], [chain(three, [store(num(0))])]); + expect(matchConstantChoice(mixed)).toBeNull(); +}); diff --git a/packages/compiler/src/backend/llvm/constant-choices.ts b/packages/compiler/src/backend/llvm/constant-choices.ts new file mode 100644 index 0000000000..6e7bb4c84d --- /dev/null +++ b/packages/compiler/src/backend/llvm/constant-choices.ts @@ -0,0 +1,241 @@ +import { createHash } from "node:crypto"; +import { InternalCompilerError } from "../../errors.js"; +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { exactInteger, widenInteger } from "./integer-values.js"; +import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; +import { f64Lit } from "./common.js"; + +type BytesSet = Extract; + +/** An if/else-if chain that stores one of several numeric literals to the + * same typed-array element, choosing it by comparing one number local with + * integer literals: + * + * if (x === 97) a[i] = 0; else if (x === 116) a[i] = 1; ... else a[i] = 3; + * + * The chain is emitted as one store of a value read from a constant table, + * with no branch on the comparisons. Branches on unpredictable input (the + * bases of a DNA sequence, the bytes of a text) mispredict about half the + * time; LLVM keeps them, because sparse cases (97, 99, 116) are too far + * apart for its own switch tables. */ +export interface ConstantChoice { + subject: IrExpr; + /** The first arm for each distinct value, in chain order. */ + cases: readonly { value: number; result: number }[]; + fallback: number; + /** Every arm's store, the final else's last. */ + stores: readonly BytesSet[]; +} + +/** At least this many distinct values: shorter chains predict as well as a + * table load. */ +const MIN_CASES = 3; +/** The table covers the subject's values from the smallest case (or the + * subject's proven minimum) up to this many entries. */ +const MAX_TABLE = 256; + +function integerLiteral(expr: IrExpr): number | null { + if (expr.kind !== "numLit" || !Number.isSafeInteger(expr.value)) return null; + // `x === -0` and `x === 0` select the same values. + return expr.value === 0 ? 0 : expr.value; +} + +function sameVarRef(a: IrExpr, b: IrExpr): boolean { + return a.kind === "varRef" && b.kind === "varRef" && a.localId === b.localId; +} + +/** A local read or a number literal: reading it has no effect. */ +function sameOperand(a: IrExpr, b: IrExpr): boolean { + if (a.kind === "numLit" && b.kind === "numLit") return Object.is(a.value, b.value); + return sameVarRef(a, b); +} + +/** The store of a single-statement arm, when it stores a number literal to + * an element named by a local and a local or a literal index (so evaluating + * them once, after the comparisons, reads what every arm would). */ +function literalStore(body: readonly IrStmt[] | null): BytesSet | null { + if (body === null || body.length !== 1) return null; + const s = body[0]!; + if (s.kind !== "bytesSet" || s.value.kind !== "numLit") return null; + if (s.arr.kind !== "varRef" || (s.index.kind !== "varRef" && s.index.kind !== "numLit")) + return null; + if (s.arr.type.kind !== "bytes" || s.index.type.kind !== "f64") return null; + return s; +} + +export function matchConstantChoice(s: IrStmt): ConstantChoice | null { + if (s.kind !== "if") return null; + let subject: IrExpr | null = null; + const cases: { value: number; result: number }[] = []; + const seen = new Set(); + const stores: BytesSet[] = []; + let current: IrStmt = s; + for (;;) { + if (current.kind !== "if") return null; + const cond = current.cond; + if (cond.kind !== "bin" || cond.op !== "===") return null; + const [local, literal] = + cond.left.kind === "varRef" ? [cond.left, cond.right] : [cond.right, cond.left]; + if (local.kind !== "varRef" || local.type.kind !== "f64") return null; + if (subject === null) subject = local; + else if (!sameVarRef(subject, local)) return null; + const value = integerLiteral(literal); + if (value === null) return null; + const store = literalStore(current.then); + if (store === null) return null; + stores.push(store); + // A repeated value never reaches its later arm. + if (!seen.has(value)) { + seen.add(value); + cases.push({ value, result: (store.value as IrExpr & { kind: "numLit" }).value }); + } + const rest: readonly IrStmt[] | null = current.else_; + if (rest === null || rest.length !== 1) return null; + if (rest[0]!.kind === "if") { + current = rest[0]!; + continue; + } + const last = literalStore(rest); + if (last === null) return null; + stores.push(last); + break; + } + const first = stores[0]!; + for (const store of stores) { + if (!sameVarRef(store.arr, first.arr) || !sameOperand(store.index, first.index)) return null; + if (store.arr.type.kind !== "bytes" || first.arr.type.kind !== "bytes") return null; + if (store.arr.type.elem !== first.arr.type.elem) return null; + } + if (cases.length < MIN_CASES) return null; + const values = cases.map((c) => c.value); + if (Math.max(...values) - Math.min(...values) >= MAX_TABLE) return null; + const fallback = (stores[stores.length - 1]!.value as IrExpr & { kind: "numLit" }).value; + return { subject: subject!, cases, fallback, stores }; +} + +function int32(value: number): boolean { + return ( + Number.isInteger(value) && value >= -2147483648 && value <= 2147483647 && !Object.is(value, -0) + ); +} + +/** The chosen number: the table entry for the subject's value, or the final + * else's literal for a value no case names (including NaN, a fraction and + * every value outside the table). */ +function emitChoice(host: LlvmEmitterContext, choice: ConstantChoice): LlValue { + const B = host.B; + const subject = host.emitExpr(choice.subject); + const results = [...choice.cases.map((c) => c.result), choice.fallback]; + const integral = results.every(int32); + const element = integral ? "i32" : "double"; + const literal = (value: number): string => (integral ? String(value) : f64Lit(value)); + + const proven = exactInteger(host, subject, choice.subject); + const values = choice.cases.map((c) => c.value); + let low = Math.min(...values); + let high = Math.max(...values); + // A proven range that fits a table needs no test: every value it allows + // has an entry. + const covered = + proven !== null && proven.range.max - proven.range.min < MAX_TABLE ? proven : null; + if (covered !== null) { + low = covered.range.min; + high = covered.range.max; + } + const size = high - low + 1; + const entries: string[] = []; + for (let value = low; value <= high; value++) { + const hit = choice.cases.find((c) => c.value === value); + entries.push(`${element} ${literal(hit === undefined ? choice.fallback : hit.result)}`); + } + const table = `@sc_choice_${createHash("sha256") + .update(`${element}:${entries.join(",")}`) + .digest("hex") + .slice(0, 16)}`; + host.declare( + `${table} = private unnamed_addr constant [${size} x ${element}] [${entries.join(", ")}]`, + ); + + let index: string; + let ok: string | null = null; + if (covered !== null) { + const wide = widenInteger(host, covered); + index = B.tmp(); + B.line(`${index} = sub i64 ${wide}, ${low}`); + } else { + // freeze keeps NaN and out-of-range conversions from producing poison; + // the round trip then rejects every value that is not an exact integer + // (-0 converts to 0, which `=== 0` selects too). + const raw = B.tmp(); + const selector = B.tmp(); + const restored = B.tmp(); + const exact = B.tmp(); + const offset = B.tmp(); + const inside = B.tmp(); + ok = B.tmp(); + index = B.tmp(); + B.line(`${raw} = fptosi double ${subject.name} to i64`); + B.line(`${selector} = freeze i64 ${raw}`); + B.line(`${restored} = sitofp i64 ${selector} to double`); + B.line(`${exact} = fcmp oeq double ${subject.name}, ${restored}`); + B.line(`${offset} = sub i64 ${selector}, ${low}`); + B.line(`${inside} = icmp ult i64 ${offset}, ${size}`); + B.line(`${ok} = and i1 ${exact}, ${inside}`); + B.line(`${index} = select i1 ${ok}, i64 ${offset}, i64 0`); + } + const slot = B.tmp(); + const entry = B.tmp(); + B.line( + `${slot} = getelementptr inbounds [${size} x ${element}], ptr ${table}, i64 0, i64 ${index}`, + ); + B.line(`${entry} = load ${element}, ptr ${slot}`); + let chosen = entry; + if (ok !== null) { + chosen = B.tmp(); + B.line( + `${chosen} = select i1 ${ok}, ${element} ${entry}, ${element} ${literal(choice.fallback)}`, + ); + } + if (!integral) return { name: chosen, type: choice.stores[0]!.value.type }; + const number = B.tmp(); + B.line(`${number} = sitofp i32 ${chosen} to double`); + return { + name: number, + type: choice.stores[0]!.value.type, + uint32: chosen, + integer: { + name: chosen, + type: "i32", + signed: Math.min(...results) < 0, + range: { min: Math.min(...results), max: Math.max(...results) }, + }, + }; +} + +/** Emits a matched chain as one store of the chosen literal. The element + * write is the arms' own: every arm names the same receiver and index. */ +export function emitConstantChoice(host: LlvmEmitterContext, choice: ConstantChoice): void { + const first = choice.stores[0]!; + if (first.arr.type.kind !== "bytes") + throw new InternalCompilerError("llvm emitter bug: constant choice on non-bytes"); + // An arm's index carries the ranges proven inside that arm: an index that + // is the compared local holds the arm's value there. The one store runs + // for every value, so such an index reads the local through the first + // comparison's operand, whose range holds before the chain branches. + const indexExpr = sameVarRef(first.index, choice.subject) ? choice.subject : first.index; + const arr = host.emitStableReceiver(first.arr, [indexExpr]); + const index = host.emitExpr(indexExpr); + const value = emitChoice(host, choice); + const scopes = choice.stores.map((store) => host.bytesAliasScopes.get(store)); + host.emitBytesSet( + first.arr.type.elem, + arr.name, + index, + value, + indexExpr, + undefined, + choice.stores.every((store) => host.bytesBounds.has(store)), + choice.stores.every((store) => host.bytesOwned.has(store)), + scopes.every((scope) => scope === scopes[0]) ? (scopes[0] ?? "") : "", + ); +} diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 8c9ace9f31..a901445d16 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -83,6 +83,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { emitter.unwindCleanups.clear(); emitter.jumpTargets = []; emitter.currentLocals = new Map(fn.locals.map((l) => [l.id, l])); + emitter.currentParameters = new Set(fn.params.map((p) => p.localId)); emitter.lazyCaptureBoxes.clear(); emitter.lazyStackBoxes.clear(); // An unchanged captured parameter keeps its value in a plain slot; the diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index e56c035550..d25427e4fb 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -67,6 +67,7 @@ import { InternalCompilerError } from "../../errors.js"; import { targetCpuAttribute } from "../target-cpu.js"; import { CPU_LEVEL_INIT_CALL, cpuDispatchAttributes, cpuLevelDefs } from "./cpu-dispatch.js"; import type { IrFunction } from "../../ir/ir.js"; +import { type FunctionLayout, layoutTierOf } from "./function-layout.js"; /** The function bodies emitted in `chunks` contiguous ranges, each by its * own emitter (LlEmitter.chunkEmitter), then merged in function order: the @@ -233,6 +234,61 @@ function partitionGroupLines(functions: readonly IrFunction[]): string[] { /** Ends the group of the definitions that follow. */ const PARTITION_GROUP_END = "; scriptc-group"; +/** Writes the generated helpers, then the function definitions with their + * group lines. With a layout, the definitions it lists, helpers included, + * move after the unlisted helpers and before the other functions, by tier, + * each in its tier's group (`layout-`). The helper compiles each + * partition's definitions in text order, so listed ones come first in their + * partitions, and it places a group's definitions in the same partitions. + * Everything else keeps its order and its source-file group. */ +function writeDefinitions( + out: string[], + layout: FunctionLayout | undefined, + helperTexts: readonly (readonly string[])[], + fnDefs: readonly string[], + groups: readonly string[], +): void { + const tierLine = (tier: number) => `${PARTITION_GROUP_END} layout-${tier}`; + const listed = new Map(); + const listIn = (tier: number, lines: readonly string[]) => { + const list = listed.get(tier) ?? []; + if (!listed.has(tier)) listed.set(tier, list); + for (const line of lines) list.push(line); + }; + if (layout === undefined || layout.size === 0) { + for (const texts of helperTexts) for (const text of texts) out.push(text); + } else { + // A helper's text can span several entries (one per line), so the + // definitions are found in the joined text. A listed definition moves + // through its closing brace; what follows it stays in place. + const joined: string[] = []; + for (const texts of helperTexts) for (const text of texts) joined.push(text); + for (const part of joined.join("\n").split(/\n(?=define )/)) { + const tier = part.startsWith("define ") ? layoutTierOf(layout, part) : undefined; + const close = tier === undefined ? -1 : part.search(/\n}(\n|$)/); + if (tier === undefined || close < 0) { + out.push(part); + continue; + } + listIn(tier, [tierLine(tier), part.slice(0, close + 2)]); + const rest = part.slice(close + 3); + if (rest !== "") out.push(rest); + } + } + const unlisted: number[] = []; + fnDefs.forEach((text, i) => { + const tier = layout === undefined || layout.size === 0 ? undefined : layoutTierOf(layout, text); + if (tier === undefined) unlisted.push(i); + else listIn(tier, ["", tierLine(tier), text]); + }); + for (const tier of [...listed.keys()].sort((a, b) => a - b)) + for (const line of listed.get(tier)!) out.push(line); + for (const [k, i] of unlisted.entries()) { + if (k > 0 || listed.size > 0) out.push(""); + out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]!); + } +} + export function emitModuleParts(emitter: LlEmitter): string[] { // Function bodies first (the literal/unit/fn-value tables fill as they // emit), then the file assembles around them — the runtime ABI’s order. @@ -948,21 +1004,24 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ); } if (globals.length > 0) out.push(``); - for (const line of helpers) out.push(line); - for (const line of ffiCallbacks.defs) out.push(line); - for (const line of shapes.defs) out.push(line); - for (const line of classShapes.defs) out.push(line); - for (const line of classObjDefs) out.push(line); - for (const line of emitter.walkers.defs) out.push(line); - for (const line of emitter.dyn.defs) out.push(line); - for (const line of wrappers) out.push(line); - for (const line of asyncDefs) out.push(line); - for (const line of emitter.resolveThunkDefs) out.push(line); - const groups = partitionGroupLines(emitter.mod.functions); - for (let i = 0; i < fnDefs.length; i++) { - if (i > 0) out.push(""); - out.push(groups[i] ?? PARTITION_GROUP_END, fnDefs[i]!); - } + writeDefinitions( + out, + emitter.targetOptions.functionLayout, + [ + helpers, + ffiCallbacks.defs, + shapes.defs, + classShapes.defs, + classObjDefs, + emitter.walkers.defs, + emitter.dyn.defs, + wrappers, + asyncDefs, + emitter.resolveThunkDefs, + ], + fnDefs, + partitionGroupLines(emitter.mod.functions), + ); if (fnDefs.length === 0) out.push(""); out.push(PARTITION_GROUP_END, ""); // WebAssembly modules translated into this program, and the registry. diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index cc0af8ebb6..c1f3cf18fe 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -1,4 +1,6 @@ /* LlEmitter's statement emission: the statement switch, try/catch, switch, loop copies, speculative writes and string self-concatenation. */ +import { emitLoopBytesHeaders } from "./bytes-headers.js"; +import { noteBytesReceiver } from "./expr-bytes.js"; import { type LlEmitter, type LlScopeEntry, @@ -81,6 +83,7 @@ import { planProvenIntegerLoop, planSpeculativeLoop, type ProvenLoopPlan, + type SpeculativeLoopPlan, speculativeReads, speculativeWriteTarget, } from "./speculative-integers.js"; @@ -93,6 +96,7 @@ import { import { emitStringInputs } from "./string-lifetimes.js"; import { LOOPS } from "./loop-state.js"; import { loopAccessesTypedArrays } from "../loop-polls.js"; +import { emitConstantChoice, matchConstantChoice } from "./constant-choices.js"; /** A callback loop version copies the loop; keep the copies small. */ const MAX_CALLBACK_VERSION_COST = 400; @@ -576,6 +580,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // IrBytesElem is static, so never rediscover it through the // generic runtime switch in a hot loop. const arr = emitter.emitStableReceiver(s.arr, [s.index, s.value]); + noteBytesReceiver(emitter, s.arr, arr.name); const idx = emitter.emitExpr(s.index); const v = emitter.emitExpr(s.value); if (s.arr.type.kind !== "bytes") @@ -837,6 +842,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { break; } case "if": { + // A chain storing one of several literals to one element stores a + // table entry instead (debug builds keep the arms to step through). + const choice = emitter.debug === null ? matchConstantChoice(s) : null; + if (choice !== null) { + emitConstantChoice(emitter, choice); + break; + } const cond = emitter.emitCondition(s.cond); const trueLabel = B.newLabel("if.t"); const joinLabel = B.newLabel("if.j"); @@ -1265,6 +1277,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { if (stripMined) { const counter: StripMinedCounter = countedLoop ? { + localId: countedLoop.localId, slot: integerSlot!, type: "i64", signed: true, @@ -1276,6 +1289,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { limit: () => countedLimit!, } : { + localId: integerLoop!.localId, slot: integerSlot!, type: emitter.sizeType, signed: false, @@ -2371,6 +2385,22 @@ export function emitLoopCopies( else ordinary(exit); return; } + const outer = emitter.bytesHeaders; + emitter.bytesHeaders = emitLoopBytesHeaders(emitter, loop, emitter.currentParameters) ?? outer; + try { + emitSpeculativeLoop(emitter, plan, exit, emitCopy); + } finally { + emitter.bytesHeaders = outer; + } +} + +/** The copies of a speculative loop (emitLoopCopies). */ +function emitSpeculativeLoop( + emitter: LlEmitter, + plan: SpeculativeLoopPlan, + exit: string, + emitCopy: (exit: string) => void, +): void { const B = emitter.B; const entries = new Map(); let ok = "true"; @@ -2705,10 +2735,11 @@ function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number { return trips * (1 + nested); } -/** The counter of a strip-mined loop: its slot, the LLVM integer type and - * signedness of its values, its step, the counters stepped beside it, and - * the limit, emitted in the outer header before each block. */ +/** The counter of a strip-mined loop: its binding, its slot, the LLVM + * integer type and signedness of its values, its step, the counters stepped + * beside it, and the limit, emitted in the outer header before each block. */ interface StripMinedCounter { + localId: string; slot: string; type: string; signed: boolean; @@ -2800,7 +2831,7 @@ function emitStripMinedCountedLoop( const after = runSlot === null ? poll : B.newLabel("loop.end"); B.condBr(inBlock, lb, after); B.startBlock(lb); - emitter.jumpTargets.push({ + const target: LlEmitter["jumpTargets"][number] = { kind: "loop", brkLabel: exit, contLabel: lu, @@ -2808,8 +2839,19 @@ function emitStripMinedCountedLoop( frameDepth: emitter.frames.length, scopeDepth: emitter.scopes.length, finallyDepth: emitter.finallyStack.length, - }); - emitter.emitBlock(s.body); + }; + emitter.jumpTargets.push(target); + // The counter steps by a nonzero constant and the body does not assign + // it, so an element at the counter is a different element on every + // iteration: its plain load or store is not merged with another + // iteration's or moved out of the loop. + const outer = emitter.plainSharedLoop; + emitter.plainSharedLoop = { counter: counter.localId, target, depth: emitter.loopDepth }; + try { + emitter.emitBlock(s.body); + } finally { + emitter.plainSharedLoop = outer; + } emitter.jumpTargets.pop(); B.br(lu); B.startBlock(lu); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index b00eaa9453..459dd2d9e1 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,5 @@ import type { TargetCpu } from "../target-cpu.js"; +import type { FunctionLayout } from "./function-layout.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; import { jsTruthy } from "./jsval.js"; @@ -63,6 +64,7 @@ import { captureFreeFunctions, observesStacks } from "../stack-observations.js"; import { findInitializedByteLoopBindings } from "./byte-windows.js"; import { matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { globalSlotAlias } from "./bytes-alias.js"; +import type { BytesHeaderScope } from "./bytes-headers.js"; import type { IntegerCountedForLoop, IntegerPairedForLoop } from "../../ir/integer-loops.js"; import { materializeSplitPiece, @@ -499,6 +501,8 @@ export interface LlvmTargetOptions { /** The host's process primitives, where it can fork (the native * compiler's). */ forkPrimitives?: ForkPrimitives | undefined; + /** Functions to write first, by tier (function-layout.ts). */ + functionLayout?: FunctionLayout; /** Exact frontend sources for dev-build locations and variable metadata. */ debugSources?: ReadonlyMap; /** Pointer width of the target C ABI. Native targets are 64-bit today. */ @@ -1823,11 +1827,26 @@ export class LlEmitter { /** Worker programs: typed-array accesses whose views an enclosing version * guard proved thread-owned. */ bytesOwned: ReadonlySet = new Set(); + /** Worker programs: the strip-mined counted loop whose body is being + * emitted (emitStripMinedCountedLoop), with its counter, its jump target + * and its loopDepth. An element access of a view that may be shared, at + * the counter and directly in that loop's body, is a plain load or store + * (plainSharedElement in expr-bytes.ts). */ + plainSharedLoop: { + counter: string; + target: LlEmitter["jumpTargets"][number]; + depth: number; + } | null = null; /** The alias-scope attachment of each typed-array element read and write * in the current loop version's fast copy (emitVersionedLoop), and the * one the element access being emitted carries (bytesElementAlias). */ bytesAliasScopes: ReadonlyMap = new Map(); bytesAccessScope = ""; + /** Header fields a speculative loop's entry read for its views + * (bytes-headers.ts), while its copies are emitted. */ + bytesHeaders: BytesHeaderScope | null = null; + /** The parameters of the function being emitted. */ + currentParameters: ReadonlySet = new Set(); /** Loop versions with alias scopes in the current function; it names * their scopes. */ aliasScopedVersions = 0; diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index b380767d4f..72ab4ec139 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -1,6 +1,7 @@ /* Focused LLVM expression emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; +import type { BytesHeader } from "./bytes-headers.js"; import { BYTES_ELEMENT_SIZE, F64, type IrBytesElem, type IrExpr } from "../../ir/ir.js"; import type { LlvmEmitterContext, LlValue } from "./expr-context.js"; import { F64_INF, exactIndexLines, f64Lit, sizeToDouble } from "./common.js"; @@ -160,6 +161,22 @@ export function emitBytesIndex( return idx; } +/** Records that `value` is the view `receiver` reads, so the header reads of + * an access through it use the values a loop entry read (bytes-headers.ts). */ +export function noteBytesReceiver(host: LlvmEmitterContext, receiver: IrExpr, value: string): void { + const scope = host.bytesHeaders; + if (scope === null || receiver.kind !== "varRef" || scope.fn !== host.currentFunctionName) return; + const header = scope.locals.get(receiver.localId); + if (header !== undefined) scope.values.set(value, header); +} + +/** The header fields a loop entry read for the view `receiver`, if any. */ +function cachedHeader(host: LlvmEmitterContext, receiver: string): BytesHeader | undefined { + const scope = host.bytesHeaders; + if (scope === null || scope.fn !== host.currentFunctionName) return undefined; + return scope.values.get(receiver); +} + /** The element count of a typed array, in the target's size type. `value`: * the program reads the count itself (bytesHeaderValueAlias), rather than * only bounding an access. */ @@ -168,6 +185,8 @@ export function emitBytesHeaderLength( receiver: string, value = false, ): string { + const cached = cachedHeader(host, receiver); + if (cached !== undefined) return cached.length; const p = host.B.tmp(); const len = host.B.tmp(); const alias = value ? bytesHeaderValueAlias(host) : bytesHeaderAlias(host); @@ -177,6 +196,8 @@ export function emitBytesHeaderLength( } export function emitBytesData(host: LlvmEmitterContext, receiver: string): string { + const cached = cachedHeader(host, receiver); + if (cached !== undefined) return cached.data; const p = host.B.tmp(); const data = host.B.tmp(); host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 3`); @@ -203,6 +224,8 @@ export function emitBytesLength( /** A view's shared backing (null for a buffer this thread owns). It is * set at construction, before script code can see the view. */ export function emitBytesSharedStorage(host: LlvmEmitterContext, receiver: string): string { + const cached = cachedHeader(host, receiver)?.shared; + if (cached !== undefined && cached !== null) return cached; const B = host.B; const slot = B.tmp(), storage = B.tmp(); @@ -317,7 +340,7 @@ function emitBytesLoad( owned: boolean, ): LlValue { // `owned`: an enclosing version guard proved the view thread-owned. - if (host.mod.workers && !owned) { + if (host.mod.workers && !owned && !plainSharedElement(host, elem, expr)) { const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); return emitSharedOrLocalLoad(host, elem, receiver, idx); } @@ -328,6 +351,31 @@ function emitBytesLoad( return emitBytesLoadAt(host, elem, data, idx); } +/** Worker programs: whether an element access at `index` of a view that + * may be shared is a plain load or store, like a thread-owned view's. It is + * when the index is the counter of the strip-mined counted loop whose body + * holds the access, outside any loop nested in that body (its condition and + * update included: a nested loop that waits on the element reads it on + * every iteration, so the read stays atomic): every iteration then reaches + * a different element, so LLVM cannot merge an element's reads across + * iterations or move them out of the loop, and the loop can be vectorized. A racing write from another thread can still be seen or + * missed within one iteration, the documented gap of non-atomic shared + * accesses. Float16 elements keep their runtime path. */ +function plainSharedElement( + host: LlvmEmitterContext, + elem: IrBytesElem, + index: IrExpr | undefined, +): boolean { + const loop = host.plainSharedLoop; + if (loop === null || elem === "f16" || index?.kind !== "varRef") return false; + // A nested loop raises loopDepth before its condition is emitted. + if (index.localId !== loop.counter || host.loopDepth !== loop.depth) return false; + let innermost: (typeof host.jumpTargets)[number] | undefined; + for (let i = host.jumpTargets.length - 1; i >= 0 && innermost === undefined; i--) + if (host.jumpTargets[i]!.kind === "loop") innermost = host.jumpTargets[i]; + return innermost === loop.target; +} + /** Worker programs: the element at a checked index of a view that may be * shared. An integer element joins as its 32-bit value and converts to a * number after the join, as the local load does, so compares and @@ -666,7 +714,7 @@ function emitScopedBytesSet( ty = "i32"; operand = stored!; } - if (host.mod.workers && !owned) { + if (host.mod.workers && !owned && !plainSharedElement(host, elem, indexExpr)) { // A shared view's element takes a relaxed atomic store of its width. const local = emitSharedBytesSplit(host, receiver); const p = B.tmp(); @@ -788,6 +836,7 @@ export function emitBytesIntrinsic( const r = directElementAccess ? host.emitStableReceiver(e.receiver, e.args) : emitBorrowedInput(host, e.receiver); + if (directElementAccess) noteBytesReceiver(host, e.receiver, r.name); // Byte operations borrow every input; views and identity results return // independent owners, while writes retain no operand beyond the call. const args = e.args.map((a) => emitBorrowedInput(host, a)); diff --git a/packages/compiler/src/backend/llvm/function-layout.ts b/packages/compiler/src/backend/llvm/function-layout.ts new file mode 100644 index 0000000000..00cf233dbd --- /dev/null +++ b/packages/compiler/src/backend/llvm/function-layout.ts @@ -0,0 +1,41 @@ +/** Function layouts: the functions that runs of a program execute, in tiers + * (scripts/refresh-cli-layout.mjs writes the native compiler's). The emitter + * writes a listed function before the others, in a partition group of its + * tier, so the code that runs together sits together in the executable. */ + +/** Layout key → tier (1 first). */ +export type FunctionLayout = ReadonlyMap; + +/** The layout key of an LLVM symbol: module numbers (`%m12.`, `%init.12`), + * which change when modules are added, are left out. */ +export function functionLayoutKey(symbol: string): string { + return symbol.replace(/_x25_m\d+_/g, "_x25_m_").replace(/_x25_init_\d+$/, "_x25_init_"); +} + +/** Reads a layout's lines, each ` `. A function listed in two + * tiers keeps the first. */ +export function parseFunctionLayout(text: string): FunctionLayout { + const layout = new Map(); + for (const line of text.split("\n")) { + const space = line.indexOf(" "); + if (space < 1) continue; + const tier = Number(line.slice(0, space)); + const key = functionLayoutKey(line.slice(space + 1).trim()); + if (Number.isInteger(tier) && tier > 0 && !layout.has(key)) layout.set(key, tier); + } + return layout; +} + +/** The lowest tier of the functions `text` defines, or undefined when the + * layout lists none of them. */ +export function layoutTierOf(layout: FunctionLayout, text: string): number | undefined { + let tier: number | undefined; + for (let at = text.indexOf("define "); at >= 0; at = text.indexOf("\ndefine ", at + 1)) { + const name = text.indexOf("@", at); + const open = text.indexOf("(", name); + if (name < 0 || open < 0) break; + const listed = layout.get(functionLayoutKey(text.slice(name + 1, open))); + if (listed !== undefined && (tier === undefined || listed < tier)) tier = listed; + } + return tier; +} diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 18f907c789..182ac49aba 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -423,6 +423,7 @@ export const LIB_FN_SYMS: Record = { "fs.rmRetrySync": "scr_fs_rm_opts_retry", "fs.rmdirSync": "scr_fs_rmdir", "fs.readdirSync": "scr_fs_readdir", + "fs.readdirRecursiveSync": "scr_fs_readdir_recursive", "fs.globSync": "scr_fs_glob_strings", "fs.callbackGlob": "scr_fs_glob_callback", "fsp.globStrings": "scr_fsp_glob_strings", @@ -1715,6 +1716,7 @@ export const LIB_FN_SYMS: Record = { "dyn.objEntries": "scr_dyn_obj_entries", "dyn.structuredClone": "scr_structured_clone", "dyn.declAdopt": "scr_dyn_decl_adopt", + "dyn.adoptShared": "scr_dyn_adopt_shared", "dyn.cloneMissing": "scr_structured_clone_missing", "dyn.cloneTransferFail": "scr_structured_clone_transfer_fail", "dyn.cloneTransferFailWith": "scr_structured_clone_transfer_fail_with", diff --git a/packages/compiler/src/backend/llvm/shape-union-tags.test.ts b/packages/compiler/src/backend/llvm/shape-union-tags.test.ts new file mode 100644 index 0000000000..e07626e77a --- /dev/null +++ b/packages/compiler/src/backend/llvm/shape-union-tags.test.ts @@ -0,0 +1,80 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import { validateModule } from "../../ir/validate.js"; +import { emitLlvmModule } from "./emitter.js"; + +async function llvmFor(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-shape-union-tags-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + const module = deserializeModule(await readFile(output, "utf8")); + expect(validateModule(module)).toEqual([]); + return emitLlvmModule(module); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function tagHelpers(ir: string): string[] { + return [...ir.matchAll(/^define internal i32 @sc_stag\.[^\n]*\{[^]*?^}/gm)].map( + (match) => match[0], + ); +} + +test("a union whose arms span few shape numbers switches on the shape word", async () => { + const ir = await llvmFor(` +type Num = { kind: "num"; value: number }; +type Neg = { kind: "neg"; operand: Node }; +type Add = { kind: "add"; left: Node; right: Node }; +type Node = Num | Neg | Add; +function evaluate(node: Node): number { + switch (node.kind) { + case "num": return node.value; + case "neg": return -evaluate(node.operand); + case "add": return evaluate(node.left) + evaluate(node.right); + } +} +const one: Node = { kind: "num", value: 1 }; +console.log(evaluate({ kind: "add", left: one, right: { kind: "neg", operand: one } })); +`); + expect(ir).not.toMatch(/@sc_stagtab\./); + const helpers = tagHelpers(ir); + expect(helpers.length).toBeGreaterThan(0); + for (const helper of helpers) expect(helper).toMatch(/switch i32 %w/); +}); + +test("a union whose arms span many shape numbers reads the arm from its table", async () => { + const arms = Array.from({ length: 40 }, (_, i) => i); + const ir = await llvmFor(` +${arms.map((i) => `type K${i} = { kind: "k${i}"; v${i}: number };`).join("\n")} +type Any = ${arms.map((i) => `K${i}`).join(" | ")}; +function value(item: Any): number { + switch (item.kind) { +${arms.map((i) => ` case "k${i}": return item.v${i};`).join("\n")} + } +} +const items: Any[] = [${arms.map((i) => `{ kind: "k${i}", v${i}: ${i} }`).join(", ")}]; +let total = 0; +for (const item of items) total += value(item); +console.log(total); +`); + // One byte per shape number from the arms' lowest to highest; 255 names no arm. + const table = + /^(@sc_stagtab\.\S+) = private unnamed_addr constant \[(\d+) x i8\] \[([^\]]*)\]$/m.exec(ir); + expect(table, "arm table").not.toBeNull(); + const entries = table![3]!.split(", ").map((entry) => Number(entry.replace("i8 ", ""))); + expect(entries).toHaveLength(Number(table![2])); + expect(entries.filter((entry) => entry !== 255).sort((a, b) => a - b)).toEqual(arms); + const helper = tagHelpers(ir).find((text) => text.includes(table![1]!)); + expect(helper, "tag helper").toBeDefined(); + expect(helper).toContain(`getelementptr inbounds [${table![2]} x i8], ptr ${table![1]}`); + expect(helper).not.toMatch(/switch i32 %w/); +}); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6d03db222e..79a79f36ce 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -313,19 +313,46 @@ function shapeUnionRc(host: ShapeHost, union: ShapeUnion): { retain: string; rel }; } +/** The widest span of shape numbers a union's arms may cover for its tag + * helper to switch on the shape word (emitShapeUnionHelpers). */ +const SHAPE_TAG_SWITCH_SPAN = 32; + /** The tag, release and trace helpers of every shape union * (nullable-unions.ts). The tag helper maps a value to its arm: NULL and * the null sentinel are the unit arms, anything else a tagged record whose - * shape word names its arm (the switch folds to a lookup table). */ + * shape word names its arm. + * - Arms within SHAPE_TAG_SWITCH_SPAN shape numbers: the helper switches on + * the shape word. Inlined, the switch folds into the caller's branch on + * the arm, one indirect jump with a jump table of at most that many + * entries per site. + * - Wider unions: the shape word indexes the union's arm table, one entry + * per shape number from the arms' lowest to highest, and callers branch + * on the small arm tag. Folded switches would give every site of such a + * union a jump table spanning all of its arms' shape numbers. */ export function emitShapeUnionHelpers(host: ShapeHost): string[] { const out: string[] = []; const S = host.sizeType; for (const union of host.nullableUnions.allShapeUnions()) { const header = shapeWordHeader(S); const tagFn = shapeTagFn(union.unionId); - const cases = union.arms - .map((arm) => `i32 ${host.nullableUnions.shapeNumber(arm.shapeId)}, label %a${arm.tag}`) - .join(" "); + const numbers = union.arms.map((arm) => host.nullableUnions.shapeNumber(arm.shapeId)); + const lo = Math.min(...numbers); + const span = Math.max(...numbers) - lo + 1; + const byTable = span > SHAPE_TAG_SWITCH_SPAN; + // Entries are bytes, or 16-bit words for a union of 255 arms or more. + // A shape number that names no arm maps to the all-ones entry, which is + // above every arm tag, so callers take their path for no arm as they do + // for the -1 of a shape number outside the table. + const wide = union.arms.some((arm) => arm.tag >= 255); + const entry = wide ? "i16" : "i8"; + const none = wide ? 65535 : 255; + const table = new Array(span).fill(none); + union.arms.forEach((arm, i) => (table[numbers[i]! - lo] = arm.tag)); + const tableName = tagFn.replace("@sc_stag.", "@sc_stagtab."); + if (byTable) + out.push( + `${tableName} = private unnamed_addr constant [${span} x ${entry}] [${table.map((t) => `${entry} ${t}`).join(", ")}]`, + ); out.push( `define internal i32 ${tagFn}(ptr %u) alwaysinline ${FN_ATTRS} { ; shape union tag`, `entry:`, @@ -343,16 +370,35 @@ export function emitShapeUnionHelpers(host: ShapeHost): string[] { ` ret i32 ${union.nullTag}`, `rec:`, ); - out.push( - ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, - ` %w = load i32, ptr %wp`, - ` switch i32 %w, label %bad [ ${cases} ]`, - ...union.arms.flatMap((arm) => [`a${arm.tag}:`, ` ret i32 ${arm.tag}`]), - `bad:`, - ` ret i32 -1`, - `}`, - ``, - ); + const cases = union.arms.map((arm, i) => `i32 ${numbers[i]}, label %a${arm.tag}`).join(" "); + if (!byTable) + out.push( + ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, + ` %w = load i32, ptr %wp`, + ` switch i32 %w, label %bad [ ${cases} ]`, + ...union.arms.flatMap((arm) => [`a${arm.tag}:`, ` ret i32 ${arm.tag}`]), + `bad:`, + ` ret i32 -1`, + `}`, + ``, + ); + else + out.push( + ` %wp = getelementptr inbounds ${header}, ptr %u, i32 0, i32 ${SHAPE_WORD_INDEX}`, + ` %w = load i32, ptr %wp`, + ` %i = sub i32 %w, ${lo}`, + ` %inrange = icmp ult i32 %i, ${span}`, + ` br i1 %inrange, label %look, label %bad`, + `look:`, + ` %tp = getelementptr inbounds [${span} x ${entry}], ptr ${tableName}, i32 0, i32 %i`, + ` %t8 = load ${entry}, ptr %tp`, + ` %tag = zext ${entry} %t8 to i32`, + ` ret i32 %tag`, + `bad:`, + ` ret i32 -1`, + `}`, + ``, + ); // Release and trace: one definition for the unions over the same // record arms (helperOwner). Unit encodings are NULL or the immortal // null sentinel; the shape word picks the arm. diff --git a/packages/compiler/src/backend/llvm/unwind-abi.test.ts b/packages/compiler/src/backend/llvm/unwind-abi.test.ts index 5f7f5895bd..bcef1c3796 100644 --- a/packages/compiler/src/backend/llvm/unwind-abi.test.ts +++ b/packages/compiler/src/backend/llvm/unwind-abi.test.ts @@ -283,3 +283,43 @@ test("calls that cannot unwind are marked nounwind", () => { expect(out).toContain("%r = call ptr %b(ptr %b) nounwind"); expect(out).toContain(" call void @scr_unwind_raise()\n"); }); + +test("functions without a plain call of a raising function change only their call lines", () => { + const raising = [ + "define internal void @sc_bf_raises() #0 {", + "entry:", + " call void @scr_unwind_raise()", + " unreachable", + "}", + ].join("\n"); + const other = [ + "define internal void @sc_bf_raises.copy() #0 {", + "entry:", + " ret void", + "}", + ].join("\n"); + const plain = [ + "define internal double @sc_f_plain(ptr %p) #0 {", + "entry: ; call sites below", + " %a = call double @sc_f_other(ptr %p)", + " %b = tail call double @sc_f_other(ptr %p) ; mentions @ and @sc_bf_raises.copy", + " store ptr @sc_bf_raises.copy, ptr %p", + " ret double %a", + "}", + ].join("\n"); + const parts = [`${raising}\n${other}`, plain, "@table = internal constant ptr @sc_f_plain"]; + const out = bridgePlainUnwindCalls(parts); + expect(out[0]).toBe(parts[0]); + expect(out[1]).toBe( + plain + .replace("@sc_f_other(ptr %p)\n", "@sc_f_other(ptr %p) nounwind\n") + .replace("@sc_f_other(ptr %p) ;", "@sc_f_other(ptr %p) nounwind ;"), + ); + expect(out[2]).toBe(parts[2]); + expect(() => + bridgePlainUnwindCalls([ + raising, + plain.replace("store ptr @sc_bf_raises.copy", "store ptr @sc_bf_raises"), + ]), + ).toThrow(UnwindAbiViolation); +}); diff --git a/packages/compiler/src/backend/llvm/unwind-abi.ts b/packages/compiler/src/backend/llvm/unwind-abi.ts index 4dc6119132..91722fb1dd 100644 --- a/packages/compiler/src/backend/llvm/unwind-abi.ts +++ b/packages/compiler/src/backend/llvm/unwind-abi.ts @@ -301,6 +301,46 @@ function markNounwind(line: string): string { : `${line.slice(0, close + 1)} nounwind${line.slice(close + 1)}`; } +/** The functions of `part`: its text split before each `define` line + * (`part.split(/\n(?=define )/)`), found with indexOf. */ +function splitFunctions(part: string): string[] { + const chunks: string[] = []; + let start = 0; + for (let at = part.indexOf("\ndefine "); at >= 0; at = part.indexOf("\ndefine ", at + 1)) { + chunks.push(part.slice(start, at)); + start = at + 1; + } + chunks.push(part.slice(start)); + return chunks; +} + +/** Whether `code` can be part of a symbol name (`[\w.$%-]`). */ +function isNameCode(code: number): boolean { + return ( + (code >= 97 && code <= 122) || + (code >= 65 && code <= 90) || + (code >= 48 && code <= 57) || + code === 95 /* _ */ || + code === 46 /* . */ || + code === 36 /* $ */ || + code === 37 /* % */ || + code === 45 /* - */ + ); +} + +/** Calls `each` with every line of `chunk` that holds `call `, by its start + * and end offsets, once per line and in order. Lines without `call ` match + * neither CALL_LINE nor PLAIN_CALL. */ +function eachCallLine(chunk: string, each: (start: number, end: number) => void): void { + for (let at = chunk.indexOf("call "); at >= 0;) { + const start = chunk.lastIndexOf("\n", at) + 1; + let end = chunk.indexOf("\n", at); + if (end < 0) end = chunk.length; + each(start, end); + at = chunk.indexOf("call ", end); + } +} + /** The value a bridged call yields on its landing pad: the dummy a * pending-flag callee returns with an exception pending. */ function dummyValue(type: string): string { @@ -318,7 +358,7 @@ function dummyValue(type: string): string { * these functions that emitDirectCall lowered are invokes already. */ export function bridgePlainUnwindCalls(parts: string[]): string[] { const raisers = new Set(); - const split = parts.map((part) => part.split(/\n(?=define )/)); + const split = parts.map(splitFunctions); for (const chunks of split) for (const chunk of chunks) { const m = DEFINE_NAME.exec(chunk); @@ -359,20 +399,29 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] { } } for (const name of invoked) if (!raisers.has(name) && testsFlag.has(name)) violations.add(name); + // Every mention of a raising function outside its `define` line, other + // than as the callee of a call or invoke, found with indexOf. for (const chunks of split) for (const chunk of chunks) - for (const line of chunk.split("\n")) { - if (line.startsWith("define ")) continue; - for (const m of line.matchAll(/@([\w.$%-]+)/g)) { - if (!raisers.has(m[1]!)) continue; - const at = m.index!; - if ( - /(?:call|invoke) [^@]*$/.test(line.slice(0, at)) && - line.charAt(at + m[0].length) === "(" - ) - continue; - violations.add(m[1]!); + for (let at = chunk.indexOf("@"); at >= 0;) { + let end = at + 1; + while (end < chunk.length && isNameCode(chunk.charCodeAt(end))) end++; + const next = chunk.indexOf("@", end); + if (end === at + 1) { + at = next; + continue; } + const name = chunk.slice(at + 1, end); + if (raisers.has(name)) { + const start = chunk.lastIndexOf("\n", at) + 1; + const line = chunk.slice(start, at); + if ( + !line.startsWith("define ") && + !(/(?:call|invoke) [^@]*$/.test(line) && chunk.charAt(end) === "(") + ) + violations.add(name); + } + at = next; } if (violations.size > 0) throw new UnwindAbiViolation(violations); let personality = false; @@ -381,6 +430,34 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] { chunks .map((chunk) => { if (!DEFINE_NAME.test(chunk) || !chunk.includes("call ")) return chunk; + // A function without a plain call of a raising function only marks + // its calls nounwind: rewrite just its call lines. + let bridged = false; + const marks: { start: number; end: number; line: string }[] = []; + eachCallLine(chunk, (start, end) => { + if (bridged) return; + const line = chunk.slice(start, end); + const m = PLAIN_CALL.exec(line); + const callee = m === null ? undefined : (m[4] ?? m[7]); + if (m !== null && callee !== undefined && raisers.has(callee)) { + bridged = true; + return; + } + if (!CALL_LINE.test(line)) return; + const marked = markNounwind(line); + if (marked !== line) marks.push({ start, end, line: marked }); + }); + if (!bridged) { + if (marks.length === 0) return chunk; + const pieces: string[] = []; + let from = 0; + for (const mark of marks) { + pieces.push(chunk.slice(from, mark.start), mark.line); + from = mark.end; + } + pieces.push(chunk.slice(from)); + return pieces.join(""); + } const lines = chunk.split("\n"); const resume = resumers.has(DEFINE_NAME.exec(chunk)![1]!); const result: string[] = []; diff --git a/packages/compiler/src/backend/native-codegen.test.ts b/packages/compiler/src/backend/native-codegen.test.ts index 5463b868ea..64961bcaa1 100644 --- a/packages/compiler/src/backend/native-codegen.test.ts +++ b/packages/compiler/src/backend/native-codegen.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, expect, test } from "vitest"; @@ -26,6 +26,7 @@ async function fakePackage( emptyOutput?: boolean; missingOutput?: boolean; changePackageDuringEmit?: boolean; + profileReport?: string; } = {}, ) { const root = await mkdtemp(join(tmpdir(), "scriptc-native-helper-test-")); @@ -72,6 +73,7 @@ ${ ? ":" : 'for each in $outputs; do cp "$input" "$each"; done' } +${options.profileReport === undefined ? "" : `printf '%s\\n' '${options.profileReport}'`} `, ); await chmod(bin, 0o755); @@ -196,6 +198,41 @@ test("profiles reach the helper and key the native cache by their bytes", async expect(calls[2]).toContain("--profile-use /profiles/a.profdata"); }); +test("a cached artifact keeps the profile match of the build that produced it", async () => { + const pkg = await fakePackage({ + profileReport: + '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":0}}', + }); + const used = { + ...request(pkg.root, pkg.packageJson, join(pkg.root, "used.o")), + profile: { use: { path: "/profiles/a.profdata", digest: "a".repeat(64) } }, + }; + const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 }; + expect((await emitNativeArtifact(used)).profileMatch).toEqual(match); + const again = await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "again.o") }); + expect(again.profileMatch).toEqual(match); + expect((await readFile(pkg.log, "utf8")).trim().split("\n")).toHaveLength(1); + // Without the cached match, the cached partition's counts are unknown. + const cache = join(pkg.root, "cache", "native-codegen-v1"); + const [shard] = await readdir(cache); + for (const name of await readdir(join(cache, shard!))) + if (name.includes(".profile-match")) await rm(join(cache, shard!, name)); + const unknown = await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "unknown.o") }); + expect(unknown.profileMatch).toEqual({ + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: 1, + }); + // Builds without a profile carry no match, from the helper or the cache. + const plain = request(pkg.root, pkg.packageJson, join(pkg.root, "plain.o")); + expect((await emitNativeArtifact(plain)).profileMatch).toBeUndefined(); + expect( + (await emitNativeArtifact({ ...plain, outputPath: join(pkg.root, "plain-again.o") })) + .profileMatch, + ).toBeUndefined(); +}); + test("chunked LLVM input preserves emitted bytes and the native cache identity", async () => { const pkg = await fakePackage(); const first = request(pkg.root, pkg.packageJson, join(pkg.root, "chunks.o")); diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 413fb7e02f..84c76ce91b 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -2,7 +2,17 @@ import { execFile } from "node:child_process"; import { createHash, type Hash } from "node:crypto"; import { closeSync, constants, openSync, readSync } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, copyFile, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { + access, + chmod, + copyFile, + mkdir, + readFile, + rename, + rm, + stat, + writeFile, +} from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -436,7 +446,10 @@ export async function emitNativeArtifact( dependencies: helper.dependencies, outputPaths: outputs, } satisfies NativeCodegenArtifact; - if (await installCachedPartitions(cached, outputs)) return artifact; + if (await installCachedPartitions(cached, outputs)) { + if (options.profile?.use === undefined) return artifact; + return { ...artifact, profileMatch: await cachedProfileMatch(cached[0]!, outputs.length) }; + } // Partitioned programs also reuse individual partition artifacts, so an // edit recompiles only the partitions it affects. The directory names the // exact helper build and target; the helper keys entries by content. @@ -459,6 +472,7 @@ export async function emitNativeArtifact( // The helper reads a private copy: the caller's .ll may be shared with a // concurrent build of another entry that has the same output stem. const input = privateSiblingPath(options.outputPath, "native-input"); + const matchStage = privateSiblingPath(options.outputPath, "native-profile-match"); try { if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); else await writeLlvmSource(input, options.llvm, 0o600); @@ -493,6 +507,8 @@ export async function emitNativeArtifact( } await chmod(stage, artifactMode()); } + const profileMatch = + options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); // Cache publication is an optimization boundary. The helper has already // produced a valid caller artifact, so a read-only/full cache must not // discard it or turn an otherwise successful build into an exception. @@ -500,23 +516,68 @@ export async function emitNativeArtifact( root !== null && (await nativeArtifactDependenciesStillMatch(helper.dependencies).catch(() => false)) ) { + // The match goes in first: a reader that finds every partition finds it too. + if (profileMatch !== null) { + await writeFile( + matchStage, + `${JSON.stringify({ profile: helperProfileMatch(profileMatch) })}\n`, + { + mode: 0o600, + }, + ) + .then(() => publishCachedFile(matchStage, cachedProfileMatchPath(cached[0]!))) + .catch(() => undefined); + } for (let index = 0; index < stages.length; index++) await publishCachedFile(stages[index]!, cached[index]!).catch(() => undefined); } for (let index = 0; index < stages.length; index++) await rename(stages[index]!, outputs[index]!); await pruneBuildCache(root); - const profileMatch = - options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout); return profileMatch === null ? artifact : { ...artifact, profileMatch }; } finally { await Promise.all([ ...stages.map((stage) => rm(stage, { force: true }).catch(() => undefined)), rm(input, { force: true }).catch(() => undefined), + rm(matchStage, { force: true }).catch(() => undefined), ]); } } +/** The cache entry beside a cached artifact's first partition that holds the + * profile match of the build that produced it. */ +function cachedProfileMatchPath(firstPartition: string): string { + return `${firstPartition}.profile-match`; +} + +/** The match in the helper's own JSON form, as parseNativeProfileMatch reads it. */ +function helperProfileMatch(match: NativeProfileMatch): Record { + return { + annotated: match.annotated, + mismatched: match.mismatched, + missing: match.missing, + unknown_partitions: match.unknownPartitions, + }; +} + +/** The profile match of a cached artifact. Without a valid entry, every + * partition's counts are unknown. */ +async function cachedProfileMatch( + firstPartition: string, + partitions: number, +): Promise { + const path = cachedProfileMatchPath(firstPartition); + const text = (await validCachedFile(path)) ? await readFile(path, "utf8").catch(() => "") : ""; + return ( + parseNativeProfileMatch(text) ?? { + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: partitions, + } + ); +} + /** Install a cached artifact only when every partition is present and valid. */ async function installCachedPartitions( cached: readonly (string | null)[], diff --git a/packages/compiler/src/backend/native-profile.test.ts b/packages/compiler/src/backend/native-profile.test.ts index b2cb0c75d1..18f0c6f736 100644 --- a/packages/compiler/src/backend/native-profile.test.ts +++ b/packages/compiler/src/backend/native-profile.test.ts @@ -3,6 +3,7 @@ import { executableLinkInputs } from "./link-plan-core.js"; import { parseNativeProfileMatch, nativeProfileHelperArgs, + renderNativeProfileMatch, nativeProfileKey, nativeProfileRefusal, profileGenerateLinkerArgs, @@ -84,3 +85,49 @@ test("the helper's profile match is read from its output", () => { expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull(); expect(parseNativeProfileMatch('{"ok":true,"profile":{"annotated":-1}}\n')).toBeNull(); }); + +test("the build reports the match, and warns when the profile is stale", () => { + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 9170, + mismatched: 0, + missing: 28636, + unknownPartitions: 0, + }), + ).toBe( + "scriptc: profile app.profdata: 9170 functions annotated, 0 mismatched, 28636 without a record\n", + ); + // 10% of the described functions changed: at the threshold, no warning. + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 90, + mismatched: 10, + missing: 5, + unknownPartitions: 1, + }), + ).toBe( + "scriptc: profile app.profdata: 90 functions annotated, 10 mismatched, 5 without a record, 1 partition without counts\n", + ); + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 5208, + mismatched: 3739, + missing: 28859, + unknownPartitions: 2, + }), + ).toBe( + "scriptc: profile app.profdata: 5208 functions annotated, 3739 mismatched, 28859 without a record, 2 partitions without counts\n" + + "scriptc: warning: 3739 of the 8947 functions app.profdata describes (41.8%) changed since it was collected and are optimized without their counts; collect the profile again\n", + ); + // Nothing described: no share to warn about. + expect( + renderNativeProfileMatch("app.profdata", { + annotated: 0, + mismatched: 0, + missing: 12, + unknownPartitions: 0, + }), + ).toBe( + "scriptc: profile app.profdata: 0 functions annotated, 0 mismatched, 12 without a record\n", + ); +}); diff --git a/packages/compiler/src/backend/native-profile.ts b/packages/compiler/src/backend/native-profile.ts index 904994a7dc..85b1fdca41 100644 --- a/packages/compiler/src/backend/native-profile.ts +++ b/packages/compiler/src/backend/native-profile.ts @@ -83,6 +83,32 @@ export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | nu return null; } +/** The share of the functions a profile describes whose control flow changed + * since it was collected, above which `scriptc build` warns that the profile + * is stale. A freshly collected profile has none. */ +export const STALE_PROFILE_SHARE = 0.1; + +/** `scriptc build --profile-use`'s report on stderr: one line with the match, + * and a warning line when more than STALE_PROFILE_SHARE of the functions the + * profile describes changed since it was collected. */ +export function renderNativeProfileMatch(profile: string, match: NativeProfileMatch): string { + const unknown = + match.unknownPartitions === 0 + ? "" + : `, ${match.unknownPartitions} partition${match.unknownPartitions === 1 ? "" : "s"} without counts`; + const lines = [ + `scriptc: profile ${profile}: ${match.annotated} functions annotated, ${match.mismatched} mismatched, ${match.missing} without a record${unknown}`, + ]; + const described = match.annotated + match.mismatched; + if (described > 0 && match.mismatched / described > STALE_PROFILE_SHARE) { + const percent = ((match.mismatched / described) * 100).toFixed(1); + lines.push( + `scriptc: warning: ${match.mismatched} of the ${described} functions ${profile} describes (${percent}%) changed since it was collected and are optimized without their counts; collect the profile again`, + ); + } + return `${lines.join("\n")}\n`; +} + /** Cache-key part; empty without a profile so other keys are unchanged. */ export function nativeProfileKey(profile: NativeProfileOptions | undefined): string { if (profile?.generate !== undefined) return `profile-generate:${profile.generate}`; diff --git a/packages/compiler/src/backend/native-toolchain.test.ts b/packages/compiler/src/backend/native-toolchain.test.ts index f1bcde404e..0ac21e8533 100644 --- a/packages/compiler/src/backend/native-toolchain.test.ts +++ b/packages/compiler/src/backend/native-toolchain.test.ts @@ -24,6 +24,7 @@ import { compileLibArchive, executableSectionEliminationFlags, executableNativeEnvironmentFingerprint, + implicitDependencyIncludeDirective, implicitDependencyProbeIncludes, parseLinkTraceFiles, resolveBuildCacheRoot, @@ -624,6 +625,46 @@ test("implicit dependency seeds include separately compiled vendor system header expect(includes).toContain(""); }); +test.skipIf(clangExecutable === undefined)( + "the implicit toolchain probe guards target-specific compiler headers", + async () => { + // clang ships its x86 intrinsic headers (, ...) for every + // target, so __has_include finds them on arm64, where including them is + // an #error, and the same holds for the ARM headers on x86. The probe + // must guard them by target. Only the compiler's own freestanding + // headers are visible here, so the result does not depend on the host's + // C library. + const includes = (await implicitDependencyProbeIncludes(runtimeSrcDir())).filter( + (include) => + include !== "" && /intrin\.h>$|^$|^$/.test(include), + ); + expect(includes).toContain(""); + const resourceDir = execFileSync(clangExecutable!, ["-print-resource-dir"], { + encoding: "utf8", + }).trim(); + const source = includes.map(implicitDependencyIncludeDirective).join(""); + for (const target of ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]) { + const result = spawnSync( + clangExecutable!, + [ + `--target=${target}`, + "-ffreestanding", + "-nostdinc", + "-isystem", + join(resourceDir, "include"), + "-fsyntax-only", + "-x", + "c", + "-", + ], + { input: source, encoding: "utf8" }, + ); + expect(result.stderr, target).toBe(""); + expect(result.status, target).toBe(0); + } + }, +); + test.skipIf( process.platform === "win32" || clangExecutable === undefined || diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index 05f1f71225..6c024c6a57 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -26,7 +26,10 @@ export { export { CcCompileError, subprocessFailureDetail } from "./native/process.js"; export { compileLibArchive } from "./native/library.js"; export { localizeLibraryObjects } from "./native/object-merge.js"; -export { implicitDependencyProbeIncludes } from "./native/compiler-fingerprint.js"; +export { + implicitDependencyIncludeDirective, + implicitDependencyProbeIncludes, +} from "./native/compiler-fingerprint.js"; export { parseLinkTraceFiles, nativeLinkerDependencyPaths } from "./native/linker-fingerprint.js"; export { clearCcCaches, stageRuntimeObjects } from "./native/runtime-objects.js"; export { diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index 2f1b88ed05..9d6f2cf944 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -8,7 +8,12 @@ import { validateNativeCodegenVersion, type NativeCodegenOutputKind, } from "./native-codegen-core.js"; -import { nativeProfileHelperArgs, type NativeProfileOptions } from "./native-profile.js"; +import { + nativeProfileHelperArgs, + parseNativeProfileMatch, + type NativeProfileMatch, + type NativeProfileOptions, +} from "./native-profile.js"; export function runNativeTool( executable: string, @@ -48,10 +53,10 @@ export function emitNativeObject(options: { partitionCache?: string; /** Profile-guided optimization: instrumentation or an indexed profile. */ profile?: NativeProfileOptions; -}): string[] { +}): { outputs: string[]; profileMatch: NativeProfileMatch | null } { verifyNativeHelper(options); const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1); - runNativeTool(options.executable, [ + const stdout = runNativeTool(options.executable, [ "emit", "--input", options.inputPath, @@ -74,7 +79,10 @@ export function emitNativeObject(options: { ...nativeProfileHelperArgs(options.profile), ]); for (const output of outputs) requireNativeArtifact(output); - return outputs; + return { + outputs, + profileMatch: options.profile?.use === undefined ? null : parseNativeProfileMatch(stdout), + }; } /** Merges raw or indexed IR profiles into one indexed profile. */ diff --git a/packages/compiler/src/backend/native/compiler-fingerprint.ts b/packages/compiler/src/backend/native/compiler-fingerprint.ts index ec427ce0a0..6f92707f63 100644 --- a/packages/compiler/src/backend/native/compiler-fingerprint.ts +++ b/packages/compiler/src/backend/native/compiler-fingerprint.ts @@ -195,15 +195,25 @@ export async function implicitDependencyProbeIncludes(rtDir: string): Promise" || include === "" - ? " && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64))" - : include === "" - ? " && defined(_WIN32)" - : include === "" || include === "" || include === "" - ? " && (defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64))" - : ""; +/** Compiler-provided x86 headers: clang ships them for every target, so + * __has_include finds them on arm64 too, where including them is an + * #error. Covers the SSE/AVX intrinsic families (, + * , ...), , and . */ +function x86OnlyHeader(include: string): boolean { + return include === "" || /^<(x86|x86gpr|[a-z]*mm)intrin\.h>$/.test(include); +} + +/** One guarded line of the implicit-toolchain probe source: the header is + * included only when the compiler can find it and it belongs to the target + * being probed. */ +export function implicitDependencyIncludeDirective(include: string): string { + const targetCondition = x86OnlyHeader(include) + ? " && (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64))" + : include === "" + ? " && defined(_WIN32)" + : include === "" || include === "" || include === "" + ? " && (defined(__arm__) || defined(__aarch64__) || defined(_M_ARM) || defined(_M_ARM64))" + : ""; return `#if __has_include(${include})${targetCondition}\n#include ${include}\n#endif\n`; } diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 6444b4c3b3..abacd5c8c1 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -1519,6 +1519,8 @@ export async function compileCInternal( [], programPublicSymbols, `${stem}.program`, + undefined, + { inProcessElf: true }, ); const publishable = entries.filter((entry) => entry.missed); if (publishable.length > 0) { diff --git a/packages/compiler/src/backend/native/library.ts b/packages/compiler/src/backend/native/library.ts index f0577f3ca9..06b4ce1df9 100644 --- a/packages/compiler/src/backend/native/library.ts +++ b/packages/compiler/src/backend/native/library.ts @@ -517,6 +517,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise [], programPublicSymbols!, `${stem}.program`, + undefined, + { inProcessElf: true }, ); // Keep the canonical archive member spelling. The fact that native // compilation used shards is an implementation detail; consumers and diff --git a/packages/compiler/src/backend/native/object-merge.ts b/packages/compiler/src/backend/native/object-merge.ts index 39a14785ec..43f15fef8f 100644 --- a/packages/compiler/src/backend/native/object-merge.ts +++ b/packages/compiler/src/backend/native/object-merge.ts @@ -5,6 +5,17 @@ import { localizeElfObject, mergeAndLocalizeCoffObjects } from "../object-locali import { type CcDriver, targetPlatform } from "./driver.js"; import { execFileAsync, subprocessFailureDetail } from "./process.js"; +/* Demotes in process (localizeElfObject), rewriting the object in place. */ +async function localizeInProcess(object: string, keepSymbols: readonly string[]): Promise { + try { + await writeFile(object, localizeElfObject(await readFile(object), new Set(keepSymbols))); + } catch (err) { + throw new Error( + `ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`, + ); + } +} + /* Multi-instance library mode's localization step: combine the program * object with exactly the runtime/vendor members it reaches into ONE * relocatable object, then demote every scriptc external definition except @@ -46,7 +57,11 @@ import { execFileAsync, subprocessFailureDetail } from "./process.js"; * multi-archive link from discarding one archive's copies), * then binutils objcopy --keep-global-symbols localizes every * other DEFINED global (objcopy leaves undefined symbols - * global by its own rule). The exact historical recipe. + * global by its own rule). The exact historical recipe. With + * `inProcessElf` (program-shard merges), localizeElfObject + * demotes instead of objcopy: a sanitized dev program merges + * into an object with hundreds of thousands of sections, which + * objcopy takes minutes over and localizeElfObject seconds. * ELF, cross triples from any host — `zig cc -target -r` * merges (zig is already the cross driver's hard requirement), * then localizeElfObject demotes in process and resolves @@ -68,6 +83,7 @@ export async function localizeLibraryObjects( keepSymbols: readonly string[], stem: string, platform = targetPlatform(driver), + options: { inProcessElf?: boolean } = {}, ): Promise { const combined = join(buildDir, `${stem}.localized.o`); const staging = join(buildDir, `${stem}.localize-staging.a`); @@ -79,7 +95,7 @@ export async function localizeLibraryObjects( const stderr = subprocessFailureDetail(err); throw new Error( `${argv[0]} failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n` + - `Runtime symbol localization needs the ${platform === "darwin" ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`, + `Runtime symbol localization needs the ${platform === "darwin" || (driver.target === null && options.inProcessElf === true) ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`, ); } }; @@ -122,7 +138,6 @@ export async function localizeLibraryObjects( keepFile, ]); } else if (platform === "linux" && driver.target === null) { - await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); await run([ "ld", "-r", @@ -132,7 +147,12 @@ export async function localizeLibraryObjects( "-o", combined, ]); - await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); + if (options.inProcessElf === true) { + await localizeInProcess(combined, keepSymbols); + } else { + await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); + await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); + } } else if (platform === "linux") { // Cross ELF: the cross driver's own lld performs the relocatable merge // (with the staging archive's member semantics); demotion and section- @@ -152,13 +172,7 @@ export async function localizeLibraryObjects( "-o", combined, ]); - try { - await writeFile(combined, localizeElfObject(await readFile(combined), new Set(keepSymbols))); - } catch (err) { - throw new Error( - `ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`, - ); - } + await localizeInProcess(combined, keepSymbols); } else { throw new InternalCompilerError( `runtime symbol localization (abi.localize_runtime) has no ${platform} arm; compileLibrary admits darwin, linux, and win32 builds only`, diff --git a/packages/compiler/src/backend/native/program-shards.ts b/packages/compiler/src/backend/native/program-shards.ts index 1afd87f7bc..2948299b22 100644 --- a/packages/compiler/src/backend/native/program-shards.ts +++ b/packages/compiler/src/backend/native/program-shards.ts @@ -52,13 +52,13 @@ export async function resolveProgramShardMergeIdentity(driver: CcDriver): Promis } if (platform === "linux") { if (driver.target === null) { - const [ld, objcopy] = await Promise.all([resolvedTool("ld"), resolvedTool("objcopy")]); - return ld === null || objcopy === null + // ld -r merges; localizeElfObject demotes in process. + const ld = await resolvedTool("ld"); + return ld === null ? null - : rememberFingerprintDependencies( - `program-shard-merge-linux-v1\0${ld.cacheIdentity}\0${objcopy.cacheIdentity}`, - [ld.canonicalPath, objcopy.canonicalPath], - ); + : rememberFingerprintDependencies(`program-shard-merge-linux-v2\0${ld.cacheIdentity}`, [ + ld.canonicalPath, + ]); } const arch = driver.target.split("-", 1)[0]; if (arch !== "x86_64" && arch !== "aarch64") return null; diff --git a/packages/compiler/src/backend/object-localize.test.ts b/packages/compiler/src/backend/object-localize.test.ts index 9e4e302856..1a1e9d4bd4 100644 --- a/packages/compiler/src/backend/object-localize.test.ts +++ b/packages/compiler/src/backend/object-localize.test.ts @@ -28,7 +28,10 @@ const SHF_EXECINSTR = 0x4n; const SHF_GROUP = 0x200n; const STB_GLOBAL = 1; const STB_WEAK = 2; +const SHT_SYMTAB_SHNDX = 18; +const SHN_LORESERVE = 0xff00; const SHN_COMMON = 0xfff2; +const SHN_XINDEX = 0xffff; interface ElfSymbolSpec { name: string; @@ -53,7 +56,11 @@ function buildStrtab(names: string[]): { data: Uint8Array; offsets: Map s.name).filter((n) => n !== "")); const symData = new Uint8Array((symbols.length + 1) * 24); const symView = new DataView(symData.buffer); + const shndxData = new Uint8Array((symbols.length + 1) * 4); + let usesShndx = false; symbols.forEach((sym, i) => { const at = (i + 1) * 24; symView.setUint32(at, sym.name === "" ? 0 : strtab.offsets.get(sym.name)!, true); symView.setUint8(at + 4, (sym.binding << 4) | 0); - symView.setUint16(at + 6, sym.shndx, true); + if (sym.shndx >= SHN_LORESERVE && sym.shndx !== SHN_COMMON) { + symView.setUint16(at + 6, SHN_XINDEX, true); + new DataView(shndxData.buffer).setUint32((i + 1) * 4, sym.shndx, true); + usesShndx = true; + } else { + symView.setUint16(at + 6, sym.shndx, true); + } symView.setBigUint64(at + 8, sym.value ?? 0n, true); }); const all: ElfSectionSpec[] = [ @@ -82,9 +97,22 @@ function buildElf( }, { name: ".strtab", type: SHT_STRTAB, data: strtab.data }, { name: ".shstrtab", type: SHT_STRTAB }, + ...(usesShndx + ? [ + { + name: ".symtab_shndx", + type: SHT_SYMTAB_SHNDX, + data: shndxData, + link: sections.length + 1, + entsize: 4n, + }, + ] + : []), ]; + const shstrndx = sections.length + 3; + const extended = all.length >= SHN_LORESERVE; const shstr = buildStrtab(all.map((s) => s.name).filter((n) => n !== "")); - all[all.length - 1]!.data = shstr.data; + all[shstrndx]!.data = shstr.data; let offset = 64; const offsets = all.map((s) => { const data = s.data ?? new Uint8Array(0); @@ -106,8 +134,8 @@ function buildElf( view.setBigUint64(40, BigInt(shoff), true); view.setUint16(52, 64, true); view.setUint16(58, 64, true); - view.setUint16(60, all.length, true); - view.setUint16(62, all.length - 1, true); + view.setUint16(60, extended ? 0 : all.length, true); + view.setUint16(62, shstrndx >= SHN_LORESERVE ? SHN_XINDEX : shstrndx, true); all.forEach((s, i) => { const at = shoff + i * 64; const data = s.data ?? new Uint8Array(0); @@ -116,8 +144,8 @@ function buildElf( view.setUint32(at + 4, s.type, true); view.setBigUint64(at + 8, s.flags ?? 0n, true); view.setBigUint64(at + 24, BigInt(s.type === 0 ? 0 : offsets[i]!), true); - view.setBigUint64(at + 32, BigInt(data.length), true); - view.setUint32(at + 40, s.link ?? 0, true); + view.setBigUint64(at + 32, BigInt(i === 0 && extended ? all.length : data.length), true); + view.setUint32(at + 40, i === 0 && shstrndx >= SHN_LORESERVE ? shstrndx : (s.link ?? 0), true); view.setUint32(at + 44, s.info ?? 0, true); view.setBigUint64(at + 48, s.addralign ?? 1n, true); view.setBigUint64(at + 56, s.entsize ?? 0n, true); @@ -142,8 +170,10 @@ interface ReadElf { function readElf(bytes: Uint8Array): ReadElf { const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const shoff = Number(view.getBigUint64(40, true)); - const shnum = view.getUint16(60, true); - const shstrndx = view.getUint16(62, true); + let shnum = view.getUint16(60, true); + let shstrndx = view.getUint16(62, true); + if (shnum === 0) shnum = Number(view.getBigUint64(shoff + 32, true)); + if (shstrndx === SHN_XINDEX) shstrndx = view.getUint32(shoff + 40, true); const raw = [] as { nameOff: number; type: number; @@ -182,12 +212,16 @@ function readElf(bytes: Uint8Array): ReadElf { const symtabIdx = raw.findIndex((s) => s.type === SHT_SYMTAB); const symtab = raw[symtabIdx]!; const strtab = raw[symtab.link]!; + const shndxTable = raw.find((s) => s.type === SHT_SYMTAB_SHNDX && s.link === symtabIdx); const symbols = [] as ReadElf["symbols"]; for (let at = symtab.offset; at < symtab.offset + symtab.size; at += 24) { + let shndx = view.getUint16(at + 6, true); + if (shndx === SHN_XINDEX) + shndx = view.getUint32(shndxTable!.offset + ((at - symtab.offset) / 24) * 4, true); symbols.push({ name: name(strtab, view.getUint32(at, true)), binding: view.getUint8(at + 4) >> 4, - shndx: view.getUint16(at + 6, true), + shndx, }); } return { sections, symbols, symtabInfo: symtab.info }; @@ -241,6 +275,65 @@ describe("localizeElfObject", () => { expect(info & 0xffffffffn).toBe(2n); }); + test("keeps extended section numbering when the object has SHN_LORESERVE or more sections", () => { + // A sanitized dev program merges into an object with more sections than + // e_shnum and st_shndx can index: section 0 carries the count and the + // section-name table's index, and .symtab_shndx the symbols' indices. + const count = SHN_LORESERVE + 400; + const texts: ElfSectionSpec[] = Array.from({ length: count }, (_, i) => ({ + name: `.text.f${i}`, + type: SHT_PROGBITS, + flags: SHF_ALLOC | SHF_EXECINSTR, + data: new Uint8Array(4), + })); + const rela = new Uint8Array(24); + new DataView(rela.buffer).setBigUint64(8, (2n << 32n) | 2n, true); // sym 2 (high) + const object = buildElf( + [ + ...texts, + { + name: ".rela.text.last", + type: SHT_RELA, + data: rela, + link: count + 2, + info: count, + entsize: 24n, + }, + ], + [ + { name: "low", binding: STB_GLOBAL, shndx: 1 }, + { name: "high", binding: STB_GLOBAL, shndx: count }, + { name: "main", binding: STB_GLOBAL, shndx: count - 1 }, + { name: "libc_ref", binding: STB_GLOBAL, shndx: 0 }, + { name: "shared_common", binding: STB_GLOBAL, shndx: SHN_COMMON, value: 8n }, + ], + 0, + ); + const output = localizeElfObject(object, new Set(["main"])); + const header = new DataView(output.buffer, output.byteOffset, output.byteLength); + expect(header.getUint16(60, true)).toBe(0); + expect(header.getUint16(62, true)).toBe(SHN_XINDEX); + const localized = readElf(output); + expect(localized.sections.length).toBe(count + 6); + expect(localized.sections.filter((s) => s.type === SHT_SYMTAB_SHNDX)).toHaveLength(1); + const byName = new Map(localized.symbols.map((s) => [s.name, s])); + expect(byName.get("low")!.binding).toBe(0); + expect(byName.get("high")!.binding).toBe(0); + expect(localized.sections[byName.get("high")!.shndx]!.name).toBe(`.text.f${count - 1}`); + expect(byName.get("main")!.binding).toBe(STB_GLOBAL); + expect(localized.sections[byName.get("main")!.shndx]!.name).toBe(`.text.f${count - 2}`); + expect(byName.get("libc_ref")!.shndx).toBe(0); + expect(byName.get("shared_common")!.shndx).toBe(SHN_COMMON); + const relaOut = localized.sections.find((s) => s.name === ".rela.text.last")!; + expect(localized.sections[relaOut.info]!.name).toBe(`.text.f${count - 1}`); + const info = new DataView(relaOut.data.buffer, relaOut.data.byteOffset).getBigUint64(8, true); + expect(localized.symbols[Number(info >> 32n)]!.name).toBe("high"); + // The output reads back as the same object. + expect(readElf(localizeElfObject(output, new Set(["main"]))).symbols).toEqual( + localized.symbols, + ); + }); + test("resolves section groups: SHT_GROUP dropped, SHF_GROUP cleared, members and indices intact", () => { const group = new Uint8Array(8); const groupView = new DataView(group.buffer); diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 302027484b..fb91ea85b6 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -84,9 +84,13 @@ interface ElfSymbol { name: string; info: number; other: number; - /** Resolved section index (SHT_SYMTAB_SHNDX applied), or a reserved - * SHN_* value below/at SHN_LORESERVE semantics. */ + /** Section index (SHT_SYMTAB_SHNDX applied), or the reserved SHN_* value + * when `reserved`. */ shndx: number; + /** st_shndx holds a reserved value (SHN_ABS, SHN_COMMON, ...), not a + * section index. An index from SHT_SYMTAB_SHNDX can itself be at or above + * SHN_LORESERVE. */ + reserved: boolean; } /** Demote every defined global/weak symbol not named in `keep` to a local @@ -143,8 +147,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) if (strtab === undefined) fail("ELF symbol table names an invalid string table"); // SHT_SYMTAB_SHNDX (only present when section indices overflow st_shndx): - // resolve the real indices up front and drop the section — the rebuilt - // object's section count only shrinks, so inline indices always fit. + // resolve the real indices up front and drop the section. The rebuilt + // object writes a new one when its own section count still overflows. const shndxSection = sections.find( (s) => s.type === SHT_SYMTAB_SHNDX && sections[s.link] === symtab, ); @@ -170,9 +174,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) for (let i = 0; i < symCount; i++) { const at = i * 24; let shndx: number = symView.getUint16(at + 6, true); + let reserved = shndx >= SHN_LORESERVE; if (shndx === SHN_XINDEX) { if (shndxView === null) fail("ELF symbol uses SHN_XINDEX without a SHT_SYMTAB_SHNDX table"); shndx = shndxView.getUint32(i * 4, true); + reserved = false; } symbols.push({ nameOffset: symView.getUint32(at, true), @@ -180,6 +186,7 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) info: symView.getUint8(at + 4), other: symView.getUint8(at + 5), shndx, + reserved, }); } @@ -212,11 +219,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) const symbolKept: boolean[] = []; for (let i = 0; i < symCount; i++) { const sym = symbols[i]!; - const regular = sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE; + const regular = !sym.reserved && sym.shndx !== SHN_UNDEF; symbolKept.push(!(regular && sectionKept[sym.shndx] !== true)); if (i === 0) continue; const binding = sym.info >> 4; - const defined = sym.shndx !== SHN_UNDEF && sym.shndx !== SHN_COMMON; + const defined = sym.shndx !== SHN_UNDEF && !(sym.reserved && sym.shndx === SHN_COMMON); if (binding !== STB_LOCAL && defined && !keep.has(sym.name)) { sym.info = (STB_LOCAL << 4) | (sym.info & 0xf); } @@ -236,6 +243,16 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) symbolMap[oldIndex] = newIndex; }); + // Extended section numbering: with SHN_LORESERVE or more sections, an + // index from SHN_LORESERVE up does not fit st_shndx, e_shnum or + // e_shstrndx. Such a symbol's st_shndx is SHN_XINDEX and its index goes in + // a SHT_SYMTAB_SHNDX section (one entry per symbol, appended after the + // kept sections); the counts go in section 0 (sh_size, sh_link). + let keptCount = 0; + for (const keptSection of sectionKept) if (keptSection) keptCount++; + const extended = keptCount >= SHN_LORESERVE; + const shndxOut = extended ? new Uint8Array(symbolOrder.length * 4) : null; + const shndxOutView = shndxOut === null ? null : new DataView(shndxOut.buffer); const newSymtabData = new Uint8Array(symbolOrder.length * 24); const newSymtabView = new DataView(newSymtabData.buffer); symbolOrder.forEach((oldIndex, newIndex) => { @@ -244,9 +261,14 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) newSymtabView.setUint32(at, sym.nameOffset, true); newSymtabView.setUint8(at + 4, sym.info); newSymtabView.setUint8(at + 5, sym.other); - const shndx = - sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE ? sectionMap[sym.shndx]! : sym.shndx; - newSymtabView.setUint16(at + 6, shndx, true); + const regular = !sym.reserved && sym.shndx !== SHN_UNDEF; + const shndx = regular ? sectionMap[sym.shndx]! : sym.shndx; + if (regular && shndx >= SHN_LORESERVE) { + newSymtabView.setUint16(at + 6, SHN_XINDEX, true); + shndxOutView!.setUint32(newIndex * 4, shndx, true); + } else { + newSymtabView.setUint16(at + 6, shndx, true); + } newSymtabView.setBigUint64(at + 8, symView.getBigUint64(oldIndex * 24 + 8, true), true); newSymtabView.setBigUint64(at + 16, symView.getBigUint64(oldIndex * 24 + 16, true), true); }); @@ -278,6 +300,33 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) // Rebuild. sh_link is a section index in every use this object can carry; // sh_info is one only on relocation sections. const kept = sections.filter((_, i) => sectionKept[i] === true); + const newSymtabIndex = sectionMap[symtabIndex]!; + if (shndxOut !== null) { + // The extended index table goes last, named in the section-name table. + const shstrtab = sections[shstrndx]!; + let nameOffset = shndxSection?.nameOffset; + if (nameOffset === undefined) { + const name = textEncoder.encode(".symtab_shndx\0"); + const grown = new Uint8Array(shstrtab.data.length + name.length); + grown.set(shstrtab.data); + grown.set(name, shstrtab.data.length); + nameOffset = shstrtab.data.length; + shstrtab.data = grown; + } + kept.push({ + nameOffset, + type: SHT_SYMTAB_SHNDX, + flags: 0n, + addr: 0n, + size: BigInt(shndxOut.length), + link: symtabIndex, + info: 0, + addralign: 4n, + entsize: 4n, + data: shndxOut, + }); + } + const newShstrndx = sectionMap[shstrndx]!; let offset = 64; const offsets: number[] = []; for (const section of kept) { @@ -301,8 +350,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) outView.setBigUint64(40, BigInt(shoffOut), true); outView.setUint16(54, 0, true); // e_phentsize outView.setUint16(56, 0, true); // e_phnum - outView.setUint16(60, kept.length, true); - outView.setUint16(62, sectionMap[shstrndx]!, true); + outView.setUint16(60, extended ? 0 : kept.length, true); + outView.setUint16(62, newShstrndx >= SHN_LORESERVE ? SHN_XINDEX : newShstrndx, true); kept.forEach((section, i) => { if (section.type !== SHT_NULL && section.type !== SHT_NOBITS) { out.set(section.data, offsets[i]!); @@ -313,12 +362,27 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) outView.setBigUint64(at + 8, section.flags, true); outView.setBigUint64(at + 16, section.addr, true); outView.setBigUint64(at + 24, BigInt(offsets[i]!), true); + if (i === 0) { + // Section 0 carries the counts that overflow the ELF header. + outView.setBigUint64(at + 32, extended ? BigInt(kept.length) : 0n, true); + outView.setUint32(at + 40, newShstrndx >= SHN_LORESERVE ? newShstrndx : 0, true); + outView.setUint32(at + 44, 0, true); + return; + } outView.setBigUint64( at + 32, section.type === SHT_NOBITS ? section.size : BigInt(section.data.length), true, ); - outView.setUint32(at + 40, section.link === 0 ? 0 : (sectionMap[section.link] ?? 0), true); + outView.setUint32( + at + 40, + section.type === SHT_SYMTAB_SHNDX + ? newSymtabIndex + : section.link === 0 + ? 0 + : (sectionMap[section.link] ?? 0), + true, + ); outView.setUint32( at + 44, section.type === SHT_RELA || section.type === SHT_REL diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index eace775f28..945803a5d0 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -28,6 +28,7 @@ import { import { isTypePatchCode } from "../diagnostics/diagnostic.js"; import { isPerformanceCode, renderPerformanceWarnings } from "../diagnostics/dynamic-tier.js"; import { isNativeOptimization } from "../backend/optimization.js"; +import { renderNativeProfileMatch } from "../backend/native-profile.js"; import { hostTargetCpu, isTargetCpuOption, @@ -713,6 +714,15 @@ async function main(args: string[], host: CliHost): Promise { } } if (result.artifact.kind === "exe") { + // How the profile matched the program's functions; `run` keeps stderr + // for the program's own output. + const profileUse = values["profile-use"]; + if ( + command === "build" && + profileUse !== undefined && + result.artifact.profileMatch !== undefined + ) + process.stderr.write(renderNativeProfileMatch(profileUse, result.artifact.profileMatch)); if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); } else if (result.artifact.kind === "obj") { nativeLinkInfo = result.artifact.nativeLinkInfo; diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index 6b1e001d28..5d75e65b45 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -38,7 +38,7 @@ Options: (native-link-info implies --emit=obj and never links; diagnostics prints the versioned JSON diagnostics envelope for build and coverage) - --fail-on + --fail-on coverage only: exit 1 when the entry path has blockers (a build would fail), or also when it has Node divergences, likely-hot functions that use untyped @@ -86,8 +86,10 @@ Options: code is laid out, inlined and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is - optimized as without one (release or speed; --emit=exe - only) + optimized as without one. build prints how the profile + matched on stderr, and warns when more than 10% of the + functions it describes changed (release or speed; + --emit=exe only) --keep-llvm keep generated LLVM IR beside the executable (default) --no-keep-llvm delete generated LLVM IR after compiling --emit-ir also write IR beside an executable or library archive; diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index cccb67bcd9..da78905b33 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -59,6 +59,11 @@ export interface CompileBaseOptions { * merge-profiles`). Code the profile does not describe, or describes * from an older version of the program, is optimized as without one. */ profileUse?: string; + /** A function layout (scripts/refresh-cli-layout.mjs; gzip-compressed when + * the name ends in .gz): the listed functions are written first, by tier, + * so the code that runs together sits together in the executable. The + * native compiler's build uses it. Executables and LLVM output. */ + functionLayout?: string; /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */ npmStatic?: readonly string[] | "auto"; diff --git a/packages/compiler/src/coverage/report.ts b/packages/compiler/src/coverage/report.ts index 58a2cafa64..d78120f89d 100644 --- a/packages/compiler/src/coverage/report.ts +++ b/packages/compiler/src/coverage/report.ts @@ -22,9 +22,18 @@ import { type TiersEnvelope, } from "../diagnostics/envelope.js"; import { renderDiagnostics } from "../diagnostics/render.js"; +import { + dynamicSlotLabel, + isDynamicSlotCode, + type DynamicSlotEntry, +} from "../diagnostics/dynamic-slots.js"; import type { LowerStats } from "../frontend/lowering/lowerer.js"; import type { ProvenanceSources } from "../frontend/provenance-registry.js"; -import type { DemotionUnit, TierReport } from "../frontend/tier-analysis.js"; +import type { + DemotionUnit, + LocalizedFailureReport, + TierReport, +} from "../frontend/tier-analysis.js"; import type { WasmModuleReport } from "../wasm/discover.js"; import { describe as describeReason, @@ -55,6 +64,9 @@ export interface CoverageInput { * leaves untyped. Never blockers; `--fail-on=dynamic` opts into failing * on it. */ performance?: ScrDiagnostic[]; + /** Localized failure: the checked dynamic slots (SC6104, also in + * `performance`). `--fail-on=dynamic-slots` fails on them. */ + dynamicSlots?: DynamicSlotEntry[]; /** Each dependency's native compilation outcome, with the first refusal * reason for a refused package. */ npmStatic?: NpmStaticStatus[]; @@ -234,6 +246,10 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions ` ${c(DIM, `(${skipped} function${skipped === 1 ? "" : "s"} not analyzed — signature blocked; their statements count as not static)`)}`, ); } + renderDynamicSlots(input.dynamicSlots ?? [], out, c, (loc) => { + const p = locator.position(loc.file, loc.start); + return `${displayPath(loc.file, opts.root)}:${p.line}:${p.column}`; + }); out.push(""); // Report native package compilation and the first reason for each refusal. @@ -391,7 +407,7 @@ export function renderCoverage(input: CoverageInput, opts: CoverageRenderOptions renderRows(groupBlockers(divergences), YELLOW); }; // The performance warning: its message names the functions. - const performance = input.performance ?? []; + const performance = (input.performance ?? []).filter((d) => !isDynamicSlotCode(d.code)); const renderPerformance = () => { for (const d of performance) { out.push(""); @@ -525,6 +541,7 @@ export type CoverageFailOn = | "blockers" | "divergences" | "dynamic" + | "dynamic-slots" | "node-islands" | "eager-node-islands" | "island-gaps"; @@ -533,6 +550,7 @@ export const COVERAGE_FAIL_ON: readonly CoverageFailOn[] = [ "blockers", "divergences", "dynamic", + "dynamic-slots", "node-islands", "eager-node-islands", "island-gaps", @@ -548,7 +566,9 @@ export function coveragePasses(input: CoverageInput, failOn?: CoverageFailOn): b // runtime fences build (the statement throws only if it executes). if (input.diagnostics.length > 0) return false; if (failOn === "divergences" && (input.divergences?.length ?? 0) > 0) return false; - if (failOn === "dynamic" && (input.performance?.length ?? 0) > 0) return false; + if (failOn === "dynamic" && (input.performance ?? []).some((d) => !isDynamicSlotCode(d.code))) + return false; + if (failOn === "dynamic-slots" && (input.dynamicSlots?.length ?? 0) > 0) return false; // The Node island gates (report-only islands count: they are what a // build would choose). Every island evaluates eagerly until deferred // evaluation lands, so eager-node-islands equals node-islands for now. @@ -620,6 +640,19 @@ export function coverageEnvelope( percentStatic: total === 0 ? 100 : Math.floor((ok / total) * 100), }, }), + ...(input.dynamicSlots !== undefined + ? { + dynamicSlots: input.dynamicSlots.map((slot) => ({ + file: slot.loc.file, + ...locator.position(slot.loc.file, slot.loc.start), + name: slot.name, + kind: slot.slot, + declaredType: slot.declaredType, + code: slot.code, + reason: slot.reason, + })), + } + : {}), ...(input.tiers !== undefined ? { tiers: tiersEnvelope(input.tiers, locator) } : {}), ...(input.wasmModules !== undefined ? { wasmModules: input.wasmModules } : {}), ...(input.nodeIslands !== undefined @@ -668,6 +701,26 @@ function tiersEnvelope(tiers: TierReport, locator: SourceLocator): TiersEnvelope ...at(r.loc), })), units: tiers.units.map((u) => ({ ...unit(u), requests: u.requests, sites: u.sites })), + localizedFailure: { + errors: tiers.localizedFailure.errors, + typeFailures: tiers.localizedFailure.typeFailures, + cascades: tiers.localizedFailure.cascades, + clearedThroughL2: tiers.localizedFailure.clearedThroughL2, + clearedThroughL4: tiers.localizedFailure.clearedThroughL4, + remainingByCode: tiers.localizedFailure.remainingByCode, + slots: tiers.localizedFailure.slots.map((slot) => ({ + ...at(slot.loc), + slot: slot.slot, + name: slot.name, + declaredType: slot.declaredType, + family: slot.family, + code: slot.code, + message: slot.message, + stage: slot.stage, + ...(slot.rejects !== null ? { rejects: slot.rejects } : {}), + cascade: slot.cascade, + })), + }, }; } @@ -695,6 +748,59 @@ function unitMatches(u: DemotionUnit, query: string): boolean { ); } +const DYNAMIC_SLOTS_SHOWN = 20; + +/** The dynamic slots under the statement counts: each slot's declared + * type, and why it has no static lowering. */ +function renderDynamicSlots( + slots: readonly DynamicSlotEntry[], + out: string[], + c: (code: string, s: string) => string, + where: (loc: SrcLoc) => string, +): void { + if (slots.length === 0) return; + out.push( + ` dynamic slots ${slots.length} ${c(DIM, "(declared types with no static lowering, held as checked-dynamic values; SC6104)")}`, + ); + for (const slot of slots.slice(0, DYNAMIC_SLOTS_SHOWN)) + out.push(` ${dynamicSlotLabel(slot)} ${c(DIM, where(slot.loc))}`); + if (slots.length > DYNAMIC_SLOTS_SHOWN) + out.push(` ${c(DIM, `and ${slots.length - DYNAMIC_SLOTS_SHOWN} more`)}`); +} + +const SLOTS_SHOWN = 40; + +/** The localized-failure section of --explain-dynamic (report-only): the + * committed type failures that would become checked dynamic slots, and + * the cascade each one causes today. */ +function renderLocalizedFailure( + report: LocalizedFailureReport, + out: string[], + c: (code: string, s: string) => string, + where: (loc: { file: string; start: number }) => string, +): void { + if (report.typeFailures.degradable === 0 && report.typeFailures.refused === 0) return; + out.push(""); + out.push( + ` ${report.slots.length} would-be dynamic slot${report.slots.length === 1 ? "" : "s"} ${c(DIM, "(localized failure, report-only: type failures a checked dynamic slot would hold)")}`, + ); + out.push( + ` ${c(DIM, `${report.errors} errors: ${report.clearedThroughL2} clear with declaration slots (L1-L2), ${report.clearedThroughL4} with member and expression slots too (L3-L4); ${report.cascades} cascade${report.cascades === 1 ? "" : "s"} attributed; ${report.typeFailures.refused} type failure${report.typeFailures.refused === 1 ? " stays" : "s stay"} refused`)}`, + ); + for (const slot of report.slots.slice(0, SLOTS_SHOWN)) { + const cascade = slot.cascade > 0 ? `, clears ${slot.cascade}` : ""; + out.push( + ` ${slot.stage} ${slot.slot} ${slot.name}: ${capMessage(slot.declaredType)} ${c(DIM, `${where(slot.loc)} (${slot.code} ${slot.family}${cascade})`)}`, + ); + if (slot.rejects !== null) + out.push(` ${c(DIM, `rejects ${slot.rejects.split(" @")[0]}`)}`); + } + if (report.slots.length > SLOTS_SHOWN) + out.push( + ` ${c(DIM, `${report.slots.length - SLOTS_SHOWN} more (--print=diagnostics lists all)`)}`, + ); +} + /** `scriptc coverage --explain-dynamic[=]`: the dynamic tier today * and the demotion chains the report-only analysis computed. With a query, * only units whose name matches it (`x`, `Point.x`, `parse:x`). */ @@ -735,6 +841,7 @@ export function renderExplainDynamic( (excluded > 0 ? c(DIM, `, ${excluded} excluded by policy`) : ""), ); } + if (query === undefined) renderLocalizedFailure(tiers.localizedFailure, out, c, where); const units = tiers.units.filter((u) => query === undefined || unitMatches(u, query)); if (query !== undefined && units.length === 0) { out.push(` no demoted unit matches '${query}'`); diff --git a/packages/compiler/src/coverage/verdict-cache.ts b/packages/compiler/src/coverage/verdict-cache.ts index c92cdc45e3..dfaaa5c98c 100644 --- a/packages/compiler/src/coverage/verdict-cache.ts +++ b/packages/compiler/src/coverage/verdict-cache.ts @@ -7,6 +7,7 @@ * still hold. With a semantic matcher (the Node frontend's), edits that * only touch TypeScript comments also reuse it, with every reported * location moved to the edited text. */ +import type { DynamicSlotEntry } from "../diagnostics/dynamic-slots.js"; import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -23,7 +24,7 @@ import { type FrontendInputSnapshot, } from "../frontend/input-tracker.js"; -const SCHEMA = "scriptc.coverage-verdict.v3"; +const SCHEMA = "scriptc.coverage-verdict.v4"; /** Key-value storage for verdict entries (best effort: failures miss). */ export interface VerdictStore { @@ -41,6 +42,7 @@ export interface CachedCoverage { unreached?: { stats: LowerStats; diagnostics: ScrDiagnostic[]; runtimeFences?: ScrDiagnostic[] }; divergences?: ScrDiagnostic[]; performance?: ScrDiagnostic[]; + dynamicSlots?: DynamicSlotEntry[]; npmStatic?: NpmStaticStatus[]; wasmModules?: WasmModuleReport[]; nodeIslands?: NodeIslandReport; @@ -158,6 +160,7 @@ function cacheable(coverage: AnalyzeResult["coverage"]): CachedCoverage | null { if (coverage.unreached !== undefined) out.unreached = coverage.unreached; if (coverage.divergences !== undefined) out.divergences = coverage.divergences; if (coverage.performance !== undefined) out.performance = coverage.performance; + if (coverage.dynamicSlots !== undefined) out.dynamicSlots = coverage.dynamicSlots; if (coverage.npmStatic !== undefined) out.npmStatic = coverage.npmStatic; if (coverage.wasmModules !== undefined) out.wasmModules = coverage.wasmModules; if (coverage.nodeIslands !== undefined) out.nodeIslands = coverage.nodeIslands; @@ -177,6 +180,7 @@ function restore(cached: CachedCoverage, sources: CachedSource[]): AnalyzeResult if (cached.unreached !== undefined) coverage.unreached = cached.unreached; if (cached.divergences !== undefined) coverage.divergences = cached.divergences; if (cached.performance !== undefined) coverage.performance = cached.performance; + if (cached.dynamicSlots !== undefined) coverage.dynamicSlots = cached.dynamicSlots; if (cached.npmStatic !== undefined) coverage.npmStatic = cached.npmStatic; if (cached.wasmModules !== undefined) coverage.wasmModules = cached.wasmModules; if (cached.nodeIslands !== undefined) coverage.nodeIslands = cached.nodeIslands; diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 872d832cb4..cc6a5a627c 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -113,6 +113,64 @@ export interface ScrDiagnostic { * and class member body: it throws when the module initializes, which is * program startup for a module the entry imports. */ startup?: true; + /** Localized failure, report-only (stage L0): set where a type with no + * static lowering poisoned a slot (Lowerer.badType and the dyn class + * field policy). Never part of the rendered message. */ + typeFailure?: TypeFailure; + /** Localized failure, report-only: the declaration this diagnostic + * rejected, as a failure-group key. Set when a reference flushes a + * deferred declaration's diagnostics, and on the diagnostics of a + * declaration statement whose bindings were blocked. */ + rejects?: string; + /** Localized failure, report-only: the failure groups (rejected + * declarations, blocked statements) that the statement committing this + * diagnostic used. A cascade clears when every one of them would + * degrade instead of failing. */ + cause?: readonly string[]; +} + +/** Where a type failure sits (localized-failure design §3.6): a + * declaration slot, a member of a declared type, or an expression's own + * type. */ +export type TypeFailureSlot = + | "field" + | "param" + | "return" + | "local" + | "global" + | "loop" + | "member" + | "expression"; + +/** Which badType branch (or policy fence) refused the type. */ +export type TypeFailureFamily = + | "node-types" + | "fetch-ambient" + | "enum-object" + | "stdlib-index" + | "index-signature" + | "package" + | "generic-signature" + | "overloads" + | "intersection" + | "component" + | "stdlib-nominal" + | "record-member" + | "unsupported" + | "dyn-field-policy"; + +/** A type failure's structured payload: what the commit points of the + * localized-failure design inspect (PoisonError.failure carries the same + * object). `degradable` is the §3.7 classification: a checked-dynamic slot + * would keep the program's meaning. */ +export interface TypeFailure { + slot: TypeFailureSlot; + /** The slot's name (`_customTools`, `listAll`, `x`), or the expression + * text for expression slots. */ + name: string; + declaredType: string; + family: TypeFailureFamily; + degradable: boolean; } /** SC3002–SC3004 — native-helper target, installation, and execution diff --git a/packages/compiler/src/diagnostics/dynamic-slots.ts b/packages/compiler/src/diagnostics/dynamic-slots.ts new file mode 100644 index 0000000000..09b4cd6706 --- /dev/null +++ b/packages/compiler/src/diagnostics/dynamic-slots.ts @@ -0,0 +1,49 @@ +/* SC6104, the dynamic-slot performance warning (localized-failure design + * §3.9): a slot whose declared TypeScript type has no static lowering holds + * a checked-dynamic value instead of rejecting the declaration around it. + * Its wording lives here; the slots come from the lowering's commit points + * (Lowerer.degradeAtCommit). Imports nothing heavy, like dynamic-tier.ts. */ +import type { SrcLoc } from "../ir/ir.js"; +import type { ScrDiagnostic, TypeFailureSlot } from "./diagnostic.js"; + +export const DYNAMIC_SLOT_CODE = "SC6104"; + +/** One dynamic slot, as coverage reports it. */ +export interface DynamicSlotEntry { + loc: SrcLoc; + name: string; + slot: TypeFailureSlot; + declaredType: string; + /** The diagnostic the slot's declaration would have failed with. */ + code: string; + reason: string; +} + +const SLOT_WORD: Record = { + field: "field", + param: "parameter", + return: "return value of", + local: "local", + global: "module global", + loop: "loop variable", + member: "member", + expression: "expression", +}; + +/** `'_customTools' (declared 'ToolDefinition[]')`, with the slot kind. */ +export function dynamicSlotLabel(entry: DynamicSlotEntry): string { + return `${SLOT_WORD[entry.slot]} '${entry.name}' (declared '${entry.declaredType}')`; +} + +export function dynamicSlotDiag(entry: DynamicSlotEntry): ScrDiagnostic { + return { + code: DYNAMIC_SLOT_CODE, + message: `${dynamicSlotLabel(entry)} is held as a checked-dynamic value: its type has no static lowering (${entry.reason}); reads into typed code are checked`, + loc: entry.loc, + hint: "restate the declared type inside the compilable set to keep this slot static; scriptc coverage lists dynamic slots, and --fail-on=dynamic-slots fails on them", + }; +} + +export function isDynamicSlotCode(code: string): boolean { + return code === DYNAMIC_SLOT_CODE; +} diff --git a/packages/compiler/src/diagnostics/envelope.ts b/packages/compiler/src/diagnostics/envelope.ts index bdb545ff3f..f6f8b9d3da 100644 --- a/packages/compiler/src/diagnostics/envelope.ts +++ b/packages/compiler/src/diagnostics/envelope.ts @@ -114,6 +114,27 @@ export interface TiersEnvelope { message: string; })[]; units: (TierUnitPosition & { requests: number; sites: number })[]; + /** Localized failure (report-only): committed type failures that would + * become checked dynamic slots, and what they would clear. */ + localizedFailure: { + errors: number; + typeFailures: { degradable: number; refused: number }; + cascades: number; + clearedThroughL2: number; + clearedThroughL4: number; + remainingByCode: Record; + slots: (SourcePosition & { + slot: string; + name: string; + declaredType: string; + family: string; + code: string; + message: string; + stage: string; + rejects?: string; + cascade: number; + })[]; + }; } export interface DiagnosticsEnvelope { @@ -143,6 +164,15 @@ export interface DiagnosticsEnvelope { }; /** build only: the produced artifact path. */ artifact?: string; + /** coverage only, when there are any: the checked dynamic slots + * (localized failure, SC6104). */ + dynamicSlots?: (SourcePosition & { + name: string; + kind: string; + declaredType: string; + code: string; + reason: string; + })[]; /** coverage --explain-dynamic only: the report-only tier assignment. */ tiers?: TiersEnvelope; /** coverage only: the WebAssembly modules the build can see, with their diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index b3f02c8a79..3ba6a7c546 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -220,7 +220,7 @@ test("early executable cache misses on source and resolution changes", async () expect(await readEarlyExecutableCache(f.root, f.options)).toBeNull(); }); -test("early executable keys isolate compile modes, paths, implementation, checker, and FFI manifests", async () => { +test("early executable keys isolate compile modes, paths, implementation, checker, function layouts, and FFI manifests", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); tracker.run(() => trackedReadFile(f.source)); @@ -242,6 +242,7 @@ test("early executable keys isolate compile modes, paths, implementation, checke { ...f.options, cpuDispatch: true }, { ...f.options, profile: "profile-generate:/tmp/%p.profraw" }, { ...f.options, profile: "profile-use:digest" }, + { ...f.options, functionLayout: "layout-digest" }, { ...f.options, checker: "tsc-ts-executable-identity" }, { ...f.options, npmStatic: "auto" }, { ...f.options, npmStatic: ["commander"] }, diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index e50a811786..b337b64ee5 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -12,6 +12,7 @@ import { } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { NativeProfileMatch } from "../backend/native-profile.js"; import type { FrontendInputSnapshot } from "../frontend/input-tracker.js"; import { frontendInputsStillMatch, validFrontendInputSnapshot } from "../frontend/input-tracker.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; @@ -129,6 +130,8 @@ interface EarlyExecutableCacheStamp { native: EarlyExecutableNativeFeatures; /** The build's performance warnings (SC61xx), which a cache hit repeats. */ performance?: ScrDiagnostic[]; + /** How a --profile-use build's profile matched, which a cache hit reports. */ + profileMatch?: NativeProfileMatch; integrity: string; } @@ -154,6 +157,8 @@ export interface EarlyExecutableCacheOptions { /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; + /** Omitted without a function layout; otherwise the digest of its bytes. */ + functionLayout?: string; /** Omitted for the default tsgo checker; with SCRIPTC_CHECKER=tsc-ts, the * tsc-ts executable's file identity. */ checker?: string; @@ -216,12 +221,23 @@ export interface EarlyExecutableCacheHit { frontend: FrontendInputSnapshot; /** The performance warnings the build reported (SC61xx), repeated on a hit. */ performance?: ScrDiagnostic[]; + /** How a --profile-use build's profile matched, reported again on a hit. */ + profileMatch?: NativeProfileMatch; } export interface EarlyExecutableCachePublish extends EarlyExecutableCacheHit { nativeDependencies?: NativeArtifactDependency[]; } +function isProfileMatch(value: unknown): value is NativeProfileMatch { + if (typeof value !== "object" || value === null) return false; + const match = value as Record; + return ["annotated", "mismatched", "missing", "unknownPartitions"].every((key) => { + const count = match[key]; + return typeof count === "number" && Number.isInteger(count) && count >= 0; + }); +} + const BOOLEAN_NATIVE_KEYS = [ "regex", "copying", @@ -300,6 +316,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...targetCpuKeyParts(options.targetCpu), ...cpuDispatchKeyParts(options.cpuDispatch), ...(options.profile !== undefined ? [options.profile] : []), + ...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), options.npmStatic === null @@ -344,6 +361,7 @@ function routeKey(options: Omit 0 ? { performance: stamp.performance } : {}), + ...(isProfileMatch(stamp.profileMatch) ? { profileMatch: stamp.profileMatch } : {}), ...(ir === null ? {} : { irPath: paths.irPath }), ...(llvmSourcePath === undefined ? {} : { llvmSourcePath }), ...(irSourcePath === undefined ? {} : { irSourcePath }), @@ -864,6 +883,7 @@ export async function publishEarlyExecutableCache( ...(result.performance === undefined || result.performance.length === 0 ? {} : { performance: result.performance }), + ...(result.profileMatch === undefined ? {} : { profileMatch: result.profileMatch }), }; const stamp: EarlyExecutableCacheStamp = { ...unsigned, integrity: stampIntegrity(unsigned) }; await writeFile(join(stage, "stamp.json"), `${JSON.stringify(stamp)}\n`, { mode: 0o600 }); diff --git a/packages/compiler/src/frontend/analysis.ts b/packages/compiler/src/frontend/analysis.ts index 848306b66b..d2e8814737 100644 --- a/packages/compiler/src/frontend/analysis.ts +++ b/packages/compiler/src/frontend/analysis.ts @@ -132,6 +132,7 @@ export function analyzeWithFrontend( ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), ...(lowered.divergences !== undefined ? { divergences: lowered.divergences } : {}), ...(lowered.performance !== undefined ? { performance: lowered.performance } : {}), + ...(lowered.dynamicSlots !== undefined ? { dynamicSlots: lowered.dynamicSlots } : {}), ...(lowered.unreached ? { unreached: lowered.unreached } : {}), ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), // --provenance-sources: the per-package attribution inputs (the diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts new file mode 100644 index 0000000000..5d75ae8065 --- /dev/null +++ b/packages/compiler/src/frontend/canon.ts @@ -0,0 +1,479 @@ +/* Canonical type normalization (localized-failure design part 2, §4), + * stage N0: report-only. Nothing in the compiler consumes it yet. + * + * `canon(type, env)` resolves a checker type, under the current generic + * bindings, to a small set of canonical shapes: primitive, literal, unit, + * top, object (nominal or structural, with lazily resolved members), array, + * tuple, flat union, unbound parameter, and an opaque leaf for what it + * cannot reduce. The per-form rules of type-mapper.ts (generic aliases, + * equal-member intersections, narrowed parameters, recursive placeholders) + * are what this replaces family by family in later stages; N1 runs it in + * shadow mode beside mapType, N2 as mapType's fallback. + * + * Nodes intern by checker type identity and the bindings of the type + * parameters the walk consulted, so a recursive type closes on itself once + * (a back-reference returns the node under construction). */ +import * as ts from "./ts7/adapter.js"; + +export type CanonPrim = "number" | "string" | "boolean" | "bigint" | "symbol"; + +export interface CanonProp { + name: string; + optional: boolean; + readonly: boolean; + type: () => Canon; +} + +export interface CanonSig { + params: { name: string; optional: boolean; rest: boolean; type: () => Canon }[]; + ret: () => Canon; + /** The signature keeps type parameters of its own. */ + generic: boolean; +} + +export interface CanonIndex { + key: Canon; + value: () => Canon; +} + +export interface AliasRef { + name: string; + args: readonly Canon[]; + /** The checker type the alias reference came from (identity preserved + * for tables keyed by ts.Type, such as boundAliasInstances). */ + tsType: ts.Type; +} + +export type Canon = + | { k: "prim"; p: CanonPrim } + | { k: "lit"; value: string | number | boolean } + | { k: "unit"; u: "undefined" | "null" | "void" | "never" } + | { k: "top"; t: "unknown" | "any" | "object" } + | { + k: "obj"; + /** Class or standard-library declaration name, when nominal. */ + nominal?: string; + /** The type arguments of a nominal generic instantiation. */ + typeArgs: () => readonly Canon[]; + alias?: AliasRef; + props: () => readonly CanonProp[]; + index: () => readonly CanonIndex[]; + call: () => readonly CanonSig[]; + construct: () => readonly CanonSig[]; + } + | { k: "array"; elem: Canon; readonly: boolean } + | { k: "tuple"; elems: readonly { type: Canon; optional: boolean; rest: boolean }[] } + | { k: "union"; arms: readonly Canon[] } + | { k: "param"; name: string } + | { k: "opaque"; ts: ts.Type; reason: string }; + +/** Type-parameter bindings: the checker type a parameter is bound to. */ +export type CanonEnv = ReadonlyMap; + +const PRIM_FLAGS: readonly [number, CanonPrim][] = [ + [ts.TypeFlags.Number, "number"], + [ts.TypeFlags.String, "string"], + [ts.TypeFlags.Boolean, "boolean"], + [ts.TypeFlags.BigInt, "bigint"], + [ts.TypeFlags.ESSymbol | ts.TypeFlags.UniqueESSymbol, "symbol"], +]; + +/** A stable key for a canonical node, for deduplication and tests. */ +export function canonKey(c: Canon, depth = 0): string { + if (depth > 6) return "…"; + switch (c.k) { + case "prim": + return c.p; + case "lit": + return JSON.stringify(c.value); + case "unit": + return c.u; + case "top": + return c.t; + case "param": + return `param ${c.name}`; + case "opaque": + return `opaque(${c.reason})`; + case "array": + return `${c.readonly ? "readonly " : ""}${canonKey(c.elem, depth + 1)}[]`; + case "tuple": + return `[${c.elems.map((e) => `${e.rest ? "..." : ""}${canonKey(e.type, depth + 1)}${e.optional ? "?" : ""}`).join(", ")}]`; + case "union": + return c.arms.map((a) => canonKey(a, depth + 1)).join(" | "); + case "obj": { + if (c.nominal !== undefined) { + const args = c.typeArgs(); + return args.length === 0 + ? c.nominal + : `${c.nominal}<${args.map((a) => canonKey(a, depth + 1)).join(", ")}>`; + } + const props = c + .props() + .map((p) => `${p.name}${p.optional ? "?" : ""}: ${canonKey(p.type(), depth + 1)}`); + const sig = (s: CanonSig): string => + `(${s.params + .map( + (p) => `${p.rest ? "..." : ""}${p.optional ? "?" : ""}${canonKey(p.type(), depth + 1)}`, + ) + .join(", ")}) => ${canonKey(s.ret(), depth + 1)}`; + const calls = c.call().map(sig); + const constructs = c.construct().map((s) => `new ${sig(s)}`); + const index = c + .index() + .map((i) => `[${canonKey(i.key, depth + 1)}]: ${canonKey(i.value(), depth + 1)}`); + return `{ ${[...calls, ...constructs, ...index, ...props].join("; ")} }`; + } + } +} + +/** The pairs a comparison takes as equal while it runs, and the order they + * were assumed in, so that a failed comparison withdraws what it assumed. */ +interface Assumptions { + pairs: Map>; + trail: [Canon, Canon][]; +} + +/** Whether two canonical nodes are the same type, structurally: unlike + * canonKey, every part is compared (signature parameters, index + * signatures, type arguments, any depth). A pair met again while it is + * being compared is taken as equal, which closes the comparison of + * recursive types; when the comparison fails, the pairs assumed during it + * are withdrawn. */ +export function canonEqual( + a: Canon, + b: Canon, + assumed: Assumptions = { pairs: new Map(), trail: [] }, +): boolean { + if (a === b) return true; + if (a.k !== b.k) return false; + let pairs = assumed.pairs.get(a); + if (pairs?.has(b)) return true; + if (pairs === undefined) assumed.pairs.set(a, (pairs = new Set())); + const mark = assumed.trail.length; + pairs.add(b); + assumed.trail.push([a, b]); + const equal = sameCanon(a, b, assumed); + if (!equal) + while (assumed.trail.length > mark) { + const [x, y] = assumed.trail.pop()!; + assumed.pairs.get(x)!.delete(y); + } + return equal; +} + +function sameCanon(a: Canon, b: Canon, assumed: Assumptions): boolean { + const eq = (x: Canon, y: Canon): boolean => canonEqual(x, y, assumed); + const sigsEqual = (xs: readonly CanonSig[], ys: readonly CanonSig[]): boolean => + xs.length === ys.length && + xs.every((x, i) => { + const y = ys[i]!; + return ( + x.generic === y.generic && + x.params.length === y.params.length && + x.params.every( + (p, j) => + p.optional === y.params[j]!.optional && + p.rest === y.params[j]!.rest && + eq(p.type(), y.params[j]!.type()), + ) && + eq(x.ret(), y.ret()) + ); + }); + switch (a.k) { + case "prim": + return a.p === (b as typeof a).p; + case "lit": + return a.value === (b as typeof a).value; + case "unit": + return a.u === (b as typeof a).u; + case "top": + return a.t === (b as typeof a).t; + case "param": + return a.name === (b as typeof a).name; + case "opaque": + return a.ts === (b as typeof a).ts; + case "array": { + const o = b as typeof a; + return a.readonly === o.readonly && eq(a.elem, o.elem); + } + case "tuple": { + const o = b as typeof a; + return ( + a.elems.length === o.elems.length && + a.elems.every( + (e, i) => + e.optional === o.elems[i]!.optional && + e.rest === o.elems[i]!.rest && + eq(e.type, o.elems[i]!.type), + ) + ); + } + case "union": { + const o = b as typeof a; + return ( + a.arms.length === o.arms.length && + a.arms.every((x) => o.arms.some((y) => eq(x, y))) && + o.arms.every((y) => a.arms.some((x) => eq(x, y))) + ); + } + case "obj": { + const o = b as typeof a; + if (a.nominal !== o.nominal) return false; + if (a.nominal !== undefined) { + const ax = a.typeArgs(); + const ox = o.typeArgs(); + return ax.length === ox.length && ax.every((x, i) => eq(x, ox[i]!)); + } + const ap = a.props(); + const op = o.props(); + if (ap.length !== op.length) return false; + for (const p of ap) { + const q = op.find((x) => x.name === p.name); + if (q === undefined || q.optional !== p.optional || !eq(p.type(), q.type())) return false; + } + const ai = a.index(); + const oi = o.index(); + return ( + ai.length === oi.length && + ai.every((x, i) => eq(x.key, oi[i]!.key) && eq(x.value(), oi[i]!.value())) && + sigsEqual(a.call(), o.call()) && + sigsEqual(a.construct(), o.construct()) + ); + } + } +} + +export class Normalizer { + private readonly interned = new Map(); + /** A number for each type parameter an environment binds: parameters + * with the same name are distinct keys. */ + private readonly paramIds = new Map(); + + constructor(private readonly checker: ts.TypeChecker) {} + + canon(type: ts.Type, env: CanonEnv = new Map()): Canon { + const id = (type as { id?: number }).id; + const key = id === undefined ? null : `${id}|${this.envKey(env)}`; + if (key !== null) { + const known = this.interned.get(key); + if (known !== undefined) return known; + } + const node = this.build(type, env, key); + if (key !== null && !this.interned.has(key)) this.interned.set(key, node); + return node; + } + + private envKey(env: CanonEnv): string { + if (env.size === 0) return ""; + const parts: string[] = []; + for (const [param, bound] of env) { + let id = this.paramIds.get(param); + if (id === undefined) this.paramIds.set(param, (id = this.paramIds.size)); + parts.push(`${id}=${(bound as { id?: number }).id ?? "?"}`); + } + return parts.sort().join(","); + } + + private build(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const flags = type.flags; + if (flags & ts.TypeFlags.Any) return { k: "top", t: "any" }; + if (flags & ts.TypeFlags.Unknown) return { k: "top", t: "unknown" }; + if (flags & ts.TypeFlags.NonPrimitive) return { k: "top", t: "object" }; + if (flags & ts.TypeFlags.Never) return { k: "unit", u: "never" }; + if (flags & ts.TypeFlags.Void) return { k: "unit", u: "void" }; + if (flags & ts.TypeFlags.Undefined) return { k: "unit", u: "undefined" }; + if (flags & ts.TypeFlags.Null) return { k: "unit", u: "null" }; + if (flags & ts.TypeFlags.TemplateLiteral) return { k: "prim", p: "string" }; + if (flags & (ts.TypeFlags.StringLiteral | ts.TypeFlags.NumberLiteral)) { + const value = (type as { value?: string | number }).value; + if (value !== undefined) return { k: "lit", value }; + } + if (flags & ts.TypeFlags.BooleanLiteral) + return { k: "lit", value: this.checker.typeToString(type) === "true" }; + if (!type.isUnionType()) + for (const [flag, p] of PRIM_FLAGS) if (flags & flag) return { k: "prim", p }; + if (flags & ts.TypeFlags.TypeParameter) { + const symbol = type.getSymbol(); + const bound = symbol === undefined ? undefined : env.get(symbol); + if (bound !== undefined && bound !== type) return this.canon(bound, env); + return { k: "param", name: symbol === undefined ? "?" : ts.symbolName(symbol) }; + } + if (type.isUnionType()) return this.union(ts.constituentTypes(type), env); + if (type.isIntersectionType()) return this.intersection(type, env, key); + if (flags & (ts.TypeFlags.Conditional | ts.TypeFlags.IndexedAccess | ts.TypeFlags.Index)) + return { k: "opaque", ts: type, reason: "symbolic type operator" }; + if (flags & ts.TypeFlags.Object) return this.object(type, env, key); + return { k: "opaque", ts: type, reason: `flags ${flags}` }; + } + + /** Flat, deduplicated, never-free; `void` reads as `undefined`. */ + private union(parts: readonly ts.Type[], env: CanonEnv): Canon { + const arms: Canon[] = []; + const add = (c: Canon): void => { + if (c.k === "union") { + for (const a of c.arms) add(a); + return; + } + if (c.k === "unit" && c.u === "never") return; + const arm: Canon = c.k === "unit" && c.u === "void" ? { k: "unit", u: "undefined" } : c; + // Objects dedupe by identity (they intern by checker type): comparing + // their members would walk into a recursive type still being built. + if (arms.some((a) => (arm.k === "obj" ? a === arm : canonEqual(a, arm)))) return; + arms.push(arm); + }; + for (const part of parts) add(this.canon(part, env)); + // Units last, in a fixed order; the rest keep the checker's order. + const unitRank = (c: Canon): number => + c.k !== "unit" ? 0 : c.u === "null" ? 1 : c.u === "undefined" ? 2 : 3; + arms.sort((a, b) => unitRank(a) - unitRank(b)); + if (arms.length === 0) return { k: "unit", u: "never" }; + if (arms.length === 1) return arms[0]!; + // `true | false` is `boolean`. + const bools = arms.filter((a) => a.k === "lit" && typeof a.value === "boolean"); + if (bools.length === 2) { + const rest = arms.filter((a) => !(a.k === "lit" && typeof a.value === "boolean")); + const merged: Canon[] = [{ k: "prim", p: "boolean" }, ...rest]; + return merged.length === 1 ? merged[0]! : { k: "union", arms: merged }; + } + return { k: "union", arms }; + } + + /** Equal constituents collapse; object-like constituents without + * signatures merge through the checker's resolved properties; a + * primitive branded with an object is the primitive. */ + private intersection(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const parts = ts.constituentTypes(type).map((t) => this.canon(t, env)); + if (parts.every((p) => canonEqual(p, parts[0]!))) return parts[0]!; + // Narrowed parameters: `T & {}` (and `T & ({} | null)`'s object part) + // keep T without its undefined and null arms. + const empty = (p: Canon): boolean => + p.k === "obj" && + p.nominal === undefined && + p.props().length === 0 && + p.index().length === 0 && + p.call().length === 0 && + p.construct().length === 0; + const rest = parts.filter((p) => !empty(p)); + if (rest.length === 1 && rest.length < parts.length) { + const only = rest[0]!; + if (only.k !== "union") return only.k === "unit" ? { k: "unit", u: "never" } : only; + const arms = only.arms.filter((a) => a.k !== "unit"); + return arms.length === 1 ? arms[0]! : { k: "union", arms }; + } + const prims = parts.filter((p) => p.k === "prim" || p.k === "lit"); + if (prims.length === 1 && parts.every((p) => p === prims[0] || p.k === "obj")) return prims[0]!; + if (parts.every((p) => p.k === "obj" || (p.k === "top" && p.t === "object"))) + return this.object(type, env, key); + return { k: "opaque", ts: type, reason: "intersection" }; + } + + private object(type: ts.Type, env: CanonEnv, key: string | null): Canon { + const checker = this.checker; + // Arrays and tuples enter the intern table before their elements are + // normalized, so a recursive alias (`type Tree = Tree[] | number`) + // closes on the node under construction. + if (checker.isArrayType(type)) { + const [elem] = checker.getTypeArguments(type as ts.TypeReference); + const node: Canon & { k: "array" } = { + k: "array", + elem: { k: "top", t: "unknown" }, + readonly: + type.getSymbol() !== undefined && ts.symbolName(type.getSymbol()!) === "ReadonlyArray", + }; + if (key !== null) this.interned.set(key, node); + if (elem !== undefined) node.elem = this.canon(elem, env); + return node; + } + if (checker.isTupleType(type)) { + const ref = type as ts.TupleTypeReference; + // The element flags live on the tuple shape: the type itself, or the + // target of a reference to it. + const shape = + (ref.elementFlags as ts.ElementFlags[] | undefined) !== undefined + ? ref + : (ref.getTarget() as ts.TupleType | undefined); + const flags = shape?.elementFlags ?? []; + const elems: { type: Canon; optional: boolean; rest: boolean }[] = []; + const node: Canon = { k: "tuple", elems }; + if (key !== null) this.interned.set(key, node); + checker.getTypeArguments(ref).forEach((a, i) => { + const f = flags[i] ?? ts.ElementFlags.Required; + elems.push({ + type: this.canon(a, env), + optional: (f & ts.ElementFlags.Optional) !== 0, + rest: (f & (ts.ElementFlags.Rest | ts.ElementFlags.Variadic)) !== 0, + }); + }); + return node; + } + const symbol = type.getSymbol(); + const decl = symbol?.declarations[0]; + const nominal = + decl !== undefined && (ts.isClassDeclaration(decl) || ts.isClassExpression(decl)) + ? ts.symbolName(symbol!) + : decl !== undefined && + ts.isInterfaceDeclaration(decl) && + ts.getSourceFileOfNode(decl).isDeclarationFile && + /\/lib\.[^/]*\.d\.ts$/.test(ts.getSourceFileOfNode(decl).fileName) + ? ts.symbolName(symbol!) + : undefined; + const aliasSymbol = type.getAliasSymbol(); + const alias: AliasRef | undefined = + aliasSymbol === undefined + ? undefined + : { + name: ts.symbolName(aliasSymbol), + args: type.getAliasTypeArguments().map((a) => this.canon(a, env)), + tsType: type, + }; + let typeArgs: readonly Canon[] | undefined; + let props: readonly CanonProp[] | undefined; + let index: readonly CanonIndex[] | undefined; + let call: readonly CanonSig[] | undefined; + let construct: readonly CanonSig[] | undefined; + const sigs = (list: readonly ts.Signature[]): CanonSig[] => + list.map((s) => ({ + params: s.getParameters().map((p) => { + const pd = p.declarations[0]; + const param = pd !== undefined && ts.isParameter(pd) ? pd : undefined; + return { + name: ts.symbolName(p), + optional: param?.questionToken !== undefined || param?.initializer !== undefined, + rest: param?.dotDotDotToken !== undefined, + type: () => this.canon(checker.getTypeOfSymbol(p), env), + }; + }), + ret: () => this.canon(checker.getReturnTypeOfSignature(s), env), + generic: s.getTypeParameters().length > 0, + })); + const node: Canon = { + k: "obj", + ...(nominal !== undefined ? { nominal } : {}), + ...(alias !== undefined ? { alias } : {}), + typeArgs: () => + (typeArgs ??= + nominal !== undefined && + ((type as ts.ObjectType).objectFlags & ts.ObjectFlags.Reference) !== 0 + ? checker.getTypeArguments(type as ts.TypeReference).map((a) => this.canon(a, env)) + : []), + props: () => + (props ??= checker.getPropertiesOfType(type).map((p) => ({ + name: ts.symbolName(p), + optional: (p.flags & ts.SymbolFlags.Optional) !== 0, + readonly: false, + type: () => this.canon(checker.getTypeOfSymbol(p), env), + }))), + index: () => + (index ??= checker.getIndexInfosOfType(type).map((info) => ({ + key: this.canon(info.keyType, env), + value: () => this.canon(info.type, env), + }))), + call: () => (call ??= sigs(checker.getCallSignatures(type))), + construct: () => (construct ??= sigs(checker.getConstructSignatures(type))), + }; + // Recursion closes here: members resolve lazily, so a back-reference + // reaches this node through the intern table. + if (key !== null) this.interned.set(key, node); + return node; + } +} diff --git a/packages/compiler/src/frontend/lowering/array-values.ts b/packages/compiler/src/frontend/lowering/array-values.ts index ed52b8da5c..a8c2d699b0 100644 --- a/packages/compiler/src/frontend/lowering/array-values.ts +++ b/packages/compiler/src/frontend/lowering/array-values.ts @@ -3,8 +3,10 @@ import { InternalCompilerError } from "../../errors.js"; import { dynUndefinedExpr, helperLoc, varRef } from "../../ir/build.js"; import { BOOL, + DYN, F64, type IrExpr, + type IrLocal, type IrStmt, type IrType, isIntegerBytesElem, @@ -446,6 +448,167 @@ export function appendElement( ]; } +/** The argument a helper passes in place of an element its callback never + * reads (lowerArrayCallback's `unread`): a constant of the parameter's + * type, so the slot stays unread and an element the type cannot hold + * (stored through a dynamic view) passes. Null for a type without one. */ +export function unreadArgument(lowerer: Lowerer, type: IrType, loc: SrcLoc): IrExpr | null { + if (type.kind === "f64") return { kind: "numLit", value: NaN, type: F64, loc }; + if (type.kind === "bool") return { kind: "boolLit", value: false, type: BOOL, loc }; + if (type.kind === "string") return { kind: "strLit", value: "", type: STRING, loc }; + if (type.kind === "union" && lowerer.armTag(type.unionId, UNDEFINED_T) >= 0) + return lowerer.wrappedUndefined(type, loc) ?? null; + return null; +} + +/** The locals of an element snapshot (arraySlotSnapshot): whether the slot + * held an element outside the element type, the value read otherwise, and + * that element's dynamic value otherwise. */ +export interface SlotSnapshot { + isForeign: string; + value: string; + foreign: string; +} + +/** The locals a snapshot of an `elem` element uses. */ +export function slotSnapshotLocals(lowerer: Lowerer, elem: IrType, ids: SlotSnapshot): IrLocal[] { + return [ + { id: ids.isForeign, name: "foreign", type: BOOL, mutable: false }, + { id: ids.value, name: "value", type: arrayValueType(lowerer, elem), mutable: false }, + { id: ids.foreign, name: "element", type: DYN, mutable: false }, + ]; +} + +/** Read the element at `arr[index]` as it is stored, before a callback runs, + * as JavaScript reads it before calling a callback that ignores it: a value + * or a present undefined into `ids.value`, or an element outside the element + * type (stored through a dynamic view) as its dynamic value into + * `ids.foreign`, which `ids.isForeign` tells apart. */ +export function arraySlotSnapshot( + lowerer: Lowerer, + arr: IrExpr, + index: IrExpr, + elem: IrType, + ids: SlotSnapshot, + loc: SrcLoc, +): IrStmt[] { + const valueT = arrayValueType(lowerer, elem); + const isForeign: IrExpr = { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }; + const missing = + valueT.kind === "dyn" ? dynUndefinedExpr(loc) : lowerer.wrappedUndefined(valueT, loc)!; + return [ + { + kind: "varDecl", + localId: ids.isForeign, + init: { + kind: "bin", + op: "===", + left: { kind: "arrayState", arr, index, type: F64, loc }, + right: { kind: "numLit", value: 3, type: F64, loc }, + type: BOOL, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: ids.value, + init: { + kind: "ternary", + cond: isForeign, + then: missing, + else_: arrayValueRead(lowerer, arr, index, elem, loc), + type: valueT, + loc, + }, + loc, + }, + { + kind: "varDecl", + localId: ids.foreign, + init: { + kind: "ternary", + cond: isForeign, + then: { kind: "libCall", fn: "array.foreign", args: [arr, index], type: DYN, loc }, + else_: dynUndefinedExpr(loc), + type: DYN, + loc, + }, + loc, + }, + ]; +} + +/** Append a snapshot (arraySlotSnapshot) of an element to `out`, an array of + * the same element type, as it was stored: an element outside the element + * type stays one, appended as the view's push appends it. `out` is a + * binding. */ +export function arraySnapshotAppend( + lowerer: Lowerer, + out: IrExpr, + elem: IrType, + ids: SlotSnapshot, + loc: SrcLoc, +): IrStmt { + const end: IrExpr = { + kind: "arrIntrinsic", + method: "length", + receiver: out, + args: [], + type: F64, + loc, + }; + const value: IrExpr = { + kind: "varRef", + localId: ids.value, + type: arrayValueType(lowerer, elem), + loc, + }; + return { + kind: "if", + cond: { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }, + then: [ + { + kind: "exprStmt", + expr: { + kind: "dynInvoke", + recv: { kind: "dynFrom", value: out, liveRef: true, type: DYN, loc }, + method: "push", + calleeName: "push", + args: [{ kind: "varRef", localId: ids.foreign, type: DYN, loc }], + type: DYN, + loc, + }, + loc, + }, + ], + else_: [arrayValueStore(lowerer, out, end, value, elem, loc)], + loc, + }; +} + +/** A snapshot (arraySlotSnapshot) as a value of `type`: the value read, or + * the element outside the element type through the checked conversion, + * which such an element fails as a typed read does. `typed` converts the + * value read. */ +export function arraySnapshotValue( + lowerer: Lowerer, + elem: IrType, + ids: SlotSnapshot, + type: IrType, + typed: (value: IrExpr) => IrExpr, + loc: SrcLoc, +): IrExpr { + return { + kind: "ternary", + cond: { kind: "varRef", localId: ids.isForeign, type: BOOL, loc }, + then: lowerer.coerceToExpected({ kind: "varRef", localId: ids.foreign, type: DYN, loc }, type), + else_: typed({ kind: "varRef", localId: ids.value, type: arrayValueType(lowerer, elem), loc }), + type, + loc, + }; +} + /** Store a stabilized value without changing the array's element ABI. */ export function arrayValueStore( lowerer: Lowerer, @@ -529,6 +692,9 @@ export function lowerSafeIndexRead( const elem = arr.type.elem; if (elem.kind === "void") return null; const resultT = arrayValueType(lowerer, elem); + // An element union that cannot take an undefined arm has no value for a + // missing element. + if (resultT.kind !== "dyn" && lowerer.wrappedUndefined(resultT, loc) === null) return null; const key = `idxOr:${typeKey(elem)}`; let name = lowerer.arrHofHelpers.get(key); if (!name) { diff --git a/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts b/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts index 375c3f1e72..866358b068 100644 --- a/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts +++ b/packages/compiler/src/frontend/lowering/builtins/disposable-stack.ts @@ -73,8 +73,7 @@ export function lowerDisposableStackUseCall( if (valueT?.kind !== "object") return null; const loc = locOf(call); const className = valueT.className; - const info = lowerer.classes.get(className); - if (!info) lowerer.flushDeferredClass(className); + const info = lowerer.classes.get(className) ?? lowerer.flushDeferredClass(className); const found = info ? lowerer.findMethodOn(info, "sym:dispose") : null; if (!found) return null; if (found.sig.params.length !== 0) { diff --git a/packages/compiler/src/frontend/lowering/builtins/filesystem.ts b/packages/compiler/src/frontend/lowering/builtins/filesystem.ts index 6b988c491d..d8272c7951 100644 --- a/packages/compiler/src/frontend/lowering/builtins/filesystem.ts +++ b/packages/compiler/src/frontend/lowering/builtins/filesystem.ts @@ -1306,6 +1306,7 @@ export function lowerFsReaddirTypesCall( ); } let sawWithFileTypes = false; + let recursive: ts.Expression | null = null; for (const p of optsNode.properties.nodes) { const m = optionMember(p); if (!m) { @@ -1334,9 +1335,11 @@ export function lowerFsReaddirTypesCall( "names decode as utf8 (the default); encoding: 'buffer' has no lowering", ); } + } else if (m.name === "recursive" && !promiseForm) { + recursive = m.value; } else { // The options-record stance: recursive (a documented knob with - // no lowering) fences by name; undocumented keys drop like Node. + // no lowering here) fences by name; undocumented keys drop like Node. fenceOrDropOptionKey( lowerer, p, @@ -1347,6 +1350,26 @@ export function lowerFsReaddirTypesCall( ); } } + if (recursive !== null) { + // Only the `true`/`false` keywords fold; any other expression, even one + // typed as a literal, evaluates at run time for its effects. + let flag = recursive; + while (ts.isParenthesizedExpression(flag)) flag = flag.expression; + const literal = + flag.kind === ts.SyntaxKind.TrueKeyword + ? "true" + : flag.kind === ts.SyntaxKind.FalseKeyword + ? "false" + : null; + if (sawWithFileTypes && literal !== "false") { + lowerer.noLowering( + `${operation} with withFileTypes and recursive`, + recursive, + "recursive: true lists relative names; withFileTypes: true lists one directory's Dirents", + ); + } + if (!sawWithFileTypes) return lowerFsReaddirRecursive(lowerer, call, recursive, literal, loc); + } if (!sawWithFileTypes) { lowerer.noLowering( `${operation} with 2 arguments`, @@ -1388,6 +1411,75 @@ export function lowerFsReaddirTypesCall( }; } +/** `fs.readdirSync(path, { recursive })` without withFileTypes: the + * relative names of every entry below the path when `recursive` is true + * (scr_fs_readdir_recursive, Node's queue order), the directory's own names + * when it is false. A recursive flag that is not a boolean literal selects + * between the two at run time, after the path is evaluated. */ +function lowerFsReaddirRecursive( + lowerer: Lowerer, + call: ts.CallExpression, + recursive: ts.Expression, + literal: string | null, + loc: SrcLoc, +): IrExpr { + const names = arrayOf(STRING); + // @types/node types the call `string[] | Buffer[]` when the options name + // no encoding; the names are strings, so the call is the string[] arm. + const callType = lowerer.mapTypeOf(lowerer.typeOf(call)); + if ( + !callType || + !( + typeEquals(callType, names) || + (callType.kind === "union" && lowerer.armTag(callType.unionId, names) >= 0) + ) + ) { + lowerer.noLowering( + "readdirSync(path, { recursive }) where the result is not a string array", + call, + "names decode as utf8 strings (the default encoding)", + ); + } + return lowerFsReaddirNames(lowerer, call, recursive, literal, names, loc); +} + +function lowerFsReaddirNames( + lowerer: Lowerer, + call: ts.CallExpression, + recursive: ts.Expression, + literal: string | null, + names: IrType, + loc: SrcLoc, +): IrExpr { + const path = lowerFsPath(lowerer, call.arguments.nodes[0]!); + const listing = (fn: IrLibFn, arg: IrExpr): IrExpr => ({ + kind: "libCall", + fn, + args: [arg], + type: names, + loc, + }); + if (literal === "true") return listing("fs.readdirRecursiveSync", path); + if (literal === "false") return listing("fs.readdirSync", path); + const pathLocal = lowerer.declareHiddenLocal("%readdirPath", STRING); + const pathRef = varRef(pathLocal.id, STRING, loc); + const flag = lowerer.lowerExprExpecting(recursive, BOOL); + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: pathLocal.id, init: path, loc }], + result: { + kind: "ternary", + cond: flag, + then: listing("fs.readdirRecursiveSync", pathRef), + else_: listing("fs.readdirSync", pathRef), + type: names, + loc, + }, + type: names, + loc, + }; +} + /** fs.statSync / fs.lstatSync's documented option keys. */ const FS_STAT_DOCUMENTED_OPTIONS: ReadonlySet = new Set(["bigint", "throwIfNoEntry"]); diff --git a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts index bff419d489..2a394c4d18 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts @@ -1,6 +1,6 @@ import { planRecordUnionLift, planRecordUnionWrap } from "../union-retag.js"; import type { WidthLift } from "../width-lift.js"; -import type { IrType } from "../../../ir/ir.js"; +import type { IrRecordShape, IrType } from "../../../ir/ir.js"; import { canDynCheckTo, isUnitType, typeEquals, UNDEFINED_T } from "../../../ir/ir.js"; import { type ClassInfo, findMethodOn, findGenericMethodOn } from "../lower-classes.js"; import type { Lowerer } from "../lowerer.js"; @@ -104,7 +104,15 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width }) ); }); - if (absentOnly.length === 1) selected = absentOnly; + // That layout does not win when it holds the record only by dropping + // fields and another arm keeps every field (`{ kind; type }` beside + // `{ kind; name; adapts?: true; type }`): the value is not its variant. + const dropsFields = (arm: IrType): boolean => !keepsEveryField(lowerer, source, arm); + if ( + absentOnly.length === 1 && + !(dropsFields(absentOnly[0]!.arm) && selected.some(({ arm }) => !dropsFields(arm))) + ) + selected = absentOnly; } // A record that fits several arms of a discriminated union selects one // by its discriminant at run time. Without this, an arm that a nested @@ -131,21 +139,19 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width // A record whose fields some arm keeps EVERY one of, each lifting // (`{ id; limit; opts?: A | B }` into `{ id; limit; opts?: A | B | C } | // { id; opts?: C }`): that arm is the value's own layout, widened; the - // others hold it only by dropping fields. Prefer the unique lossless arm. + // others hold it only by dropping fields. Prefer the unique lossless arm + // with the record's own fields, then the unique one that also declares + // optional fields the record lacks, which complete as absent. if (selected.length > 1 && src.kind === "record") { const source = lowerer.shapes.get(src.shapeId); - const lossless = selected.filter(({ arm }) => { - const target = arm.kind === "record" ? lowerer.shapes.get(arm.shapeId) : undefined; - return ( - source !== undefined && - target !== undefined && - !source.tuple && - !target.tuple && - target.fields.length === source.fields.length && - source.fields.every((field) => target.fields.some((kept) => kept.name === field.name)) - ); - }); - if (lossless.length === 1) selected = lossless; + const lossless = selected.filter(({ arm }) => keepsEveryField(lowerer, source, arm)); + const exact = lossless.filter( + ({ arm }) => + arm.kind === "record" && + lowerer.shapes.get(arm.shapeId)?.fields.length === source?.fields.length, + ); + if (exact.length === 1) selected = exact; + else if (exact.length === 0 && lossless.length === 1) selected = lossless; } if (selected.length !== 1) return null; return { how: "liftWrap", tag: selected[0]!.tag, arm: selected[0]!.arm }; @@ -241,6 +247,23 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width return null; } +/** True when the record arm `arm` has a field of every name the record + * `source` has. Tuples keep their own positional layouts. */ +function keepsEveryField( + lowerer: Lowerer, + source: IrRecordShape | undefined, + arm: IrType, +): boolean { + const target = arm.kind === "record" ? lowerer.shapes.get(arm.shapeId) : undefined; + return ( + source !== undefined && + target !== undefined && + !source.tuple && + !target.tuple && + source.fields.every((field) => target.fields.some((kept) => kept.name === field.name)) + ); +} + /** Two record arms a source record fits, where one arm declares a field * the other lacks and the source types that field `T | undefined` * (`{ count, errors: errs.length ? errs : undefined }` into `{ count; diff --git a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts index 2c4d5d9b41..497b50a27e 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-callbacks.ts @@ -38,6 +38,9 @@ import { } from "../lower-array-sort.js"; import { arrayIndexPresent, + arraySlotSnapshot, + arraySnapshotAppend, + arraySnapshotValue, arrayValueRead, arrayValueStore, arrayValueType, @@ -45,6 +48,9 @@ import { elementMayBeForeign, foreignElementThrow, primitiveElement, + slotSnapshotLocals, + unreadArgument, + type SlotSnapshot, } from "../array-values.js"; import { typeKey } from "../../type-mapper.js"; import { @@ -101,9 +107,12 @@ export function lowerArrayHofCall( } } const callback = booleanFilter - ? { fnArg: truthyFilterCallback(lowerer, valueT, loc), arity: 1 } + ? { fnArg: truthyFilterCallback(lowerer, valueT, loc), arity: 1, unread: false } : lowerArrayCallback(lowerer, argNode, [valueT], arrayOf(elem)); const { fnArg, arity } = callback; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = callback.unread && !present && elementMayBeForeign(lowerer, elem); const fnRet = fnArg.type.ret; if (method === "map" && (fnRet.kind === "void" || fnRet.kind === "func")) { // The result array U[] is unrepresentable here (no void elements; @@ -133,8 +142,9 @@ export function lowerArrayHofCall( : null; const outElem = filtered !== null ? filtered.elem : callbackArrayElement(lowerer, call, fnRet); // A filter that re-tags its elements keeps the single union-typed visit. + // A callback that never reads its element has no use for a typed one. const fast = - present || booleanFilter || (method === "filter" && !typeEquals(outElem, elem)) + present || booleanFilter || unread || (method === "filter" && !typeEquals(outElem, elem)) ? null : lowerFastCallback(lowerer, argNode, [valueT], 0, elem, arrayOf(elem), callback); const helper = arrayHofHelper( @@ -148,6 +158,7 @@ export function lowerArrayHofCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); const resultType: IrType = method === "map" || method === "filter" ? arrayOf(outElem) : VOID; const result: IrExpr = { @@ -541,12 +552,21 @@ function callbackValueRead( elem: IrType, present: boolean, loc: SrcLoc, + unread = false, ): IrExpr { + if (unread) { + const placeholder = unreadArgument(lowerer, callbackValueType(lowerer, elem, present), loc); + if (placeholder) return placeholder; + } return present ? { kind: "arrayGet", arr, index, type: elem, loc } : arrayValueRead(lowerer, arr, index, elem, loc); } +/** The snapshot locals of the filter and find helpers; `value` is their + * element local `v.0`. */ +const FILTER_SNAPSHOT: SlotSnapshot = { isForeign: "vf.0", value: "v.0", foreign: "vd.0" }; + /** Interned synthetic loop function for one (method, elem, fnRet, arity) * combo. Named `%arr..` ('%' keeps it out of the user * namespace); rides `liftedFns` into the module like a lifted lambda (it @@ -562,13 +582,23 @@ function arrayHofHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}:${typeKey(outElem)}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}:${typeKey(outElem)}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); lowerer.arrHofHelpers.set(key, name); - lowerer.arrHofInfo.set(name, { method, elem, fnRet, arity, outElem, present, split: !!fastT }); + lowerer.arrHofInfo.set(name, { + method, + elem, + fnRet, + arity, + outElem, + present, + split: !!fastT, + unread, + }); const fn = buildArrayHofFn( lowerer, name, @@ -580,6 +610,7 @@ function arrayHofHelper( outElem, fastT, present, + unread, ); lowerer.liftedFns.push(fastT ? withFastCallback(fn, fastT, elem) : fn); return name; @@ -794,6 +825,7 @@ function buildArrayHofFn( outElem: IrType, fastT: CallbackType | null = null, present = false, + unread = false, ): IrFunction { const arrT = arrayOf(elem); const valueT = callbackValueType(lowerer, elem, present); @@ -810,6 +842,19 @@ function buildArrayHofFn( present, loc, ); + // What the callback takes: the element, or a placeholder for one it + // ignores (callbackValueRead's `unread`). + const argElem = unread + ? callbackValueRead( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + present, + loc, + true, + ) + : getElem; const hasElem: IrExpr = { kind: "arrayHas", arr: varRef("a.0", arrT, loc), @@ -858,7 +903,7 @@ function buildArrayHofFn( kind: "if", cond: hasElem, then: [ - { kind: "varDecl", localId: "mapped.0", init: callF(getElem), loc }, + { kind: "varDecl", localId: "mapped.0", init: callF(argElem), loc }, arrayValueStore( lowerer, varRef("out.0", outT, loc), @@ -994,6 +1039,19 @@ function buildArrayHofFn( ); returnType = outT; const retained = filterRetainedValue(lowerer, varRef("v.0", valueT, loc), elem, outElem, loc); + // A predicate that ignores the element of an array that can hold + // elements outside its element type keeps a snapshot of the slot, read + // before the predicate runs, as stored (arraySlotSnapshot). + const copied = + (arity === 0 || unread) && + !present && + typeEquals(outElem, elem) && + elementMayBeForeign(lowerer, elem); + // `v.0` is declared above; the snapshot adds its other two locals. + if (copied) + locals.push( + ...slotSnapshotLocals(lowerer, elem, FILTER_SNAPSHOT).filter((l) => l.id !== "v.0"), + ); body = [ { kind: "varDecl", @@ -1006,34 +1064,62 @@ function buildArrayHofFn( { kind: "if", cond: hasElem, - then: [ - { kind: "varDecl", localId: "v.0", init: getElem, loc }, - { - kind: "if", - // ToBoolean over the predicate's answer — inert when it already - // returned bool, the per-union helper when it returned a union. - cond: filterCond(callF(varRef("v.0", valueT, loc)), fnRet, loc), - then: [ - arrayValueStore( + // A kept element is appended as it was stored, so one the element + // type cannot hold stays in the result. + then: copied + ? [ + ...arraySlotSnapshot( lowerer, - varRef("out.0", outT, loc), - { - kind: "arrIntrinsic", - method: "length", - receiver: varRef("out.0", outT, loc), - args: [], - type: F64, - loc, - }, - retained, - outElem, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + FILTER_SNAPSHOT, loc, ), + { + kind: "if", + cond: filterCond(callF(argElem), fnRet, loc), + then: [ + arraySnapshotAppend( + lowerer, + varRef("out.0", outT, loc), + elem, + FILTER_SNAPSHOT, + loc, + ), + ], + else_: null, + loc, + }, + ] + : [ + { kind: "varDecl", localId: "v.0", init: getElem, loc }, + { + kind: "if", + // ToBoolean over the predicate's answer — inert when it already + // returned bool, the per-union helper when it returned a union. + cond: filterCond(callF(varRef("v.0", valueT, loc)), fnRet, loc), + then: [ + arrayValueStore( + lowerer, + varRef("out.0", outT, loc), + { + kind: "arrIntrinsic", + method: "length", + receiver: varRef("out.0", outT, loc), + args: [], + type: F64, + loc, + }, + retained, + outElem, + loc, + ), + ], + else_: null, + loc, + }, ], - else_: null, - loc, - }, - ], else_: null, loc, }, @@ -1048,7 +1134,7 @@ function buildArrayHofFn( { kind: "if", cond: hasElem, - then: [{ kind: "exprStmt", expr: callF(getElem), loc }], + then: [{ kind: "exprStmt", expr: callF(argElem), loc }], else_: null, loc, }, @@ -1088,13 +1174,17 @@ export function lowerArrayFindLikeCall( presentElements(lowerer, call, elem) && ((method !== "find" && method !== "findLast") || (!bindUntyped && lowerer.coercibleValue(elem, lowerer.irTypeOf(call)))); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [callbackValueType(lowerer, elem, present)], arrT, bindUntyped, ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && !present && elementMayBeForeign(lowerer, elem); // JS takes the ToBoolean of the predicate's result — allowed wherever // that ToBoolean has a static answer (bool passes through; f64/string // by value; a truthy-answerable union by its arm — the @@ -1109,18 +1199,19 @@ export function lowerArrayFindLikeCall( method === "findIndex" || method === "findLastIndex" ) { - const fast = present - ? null - : lowerFastCallback( - lowerer, - argNode, - [arrayValueType(lowerer, elem)], - 0, - elem, - arrT, - { fnArg, arity }, - bindUntyped, - ); + const fast = + present || unread + ? null + : lowerFastCallback( + lowerer, + argNode, + [arrayValueType(lowerer, elem)], + 0, + elem, + arrT, + { fnArg, arity }, + bindUntyped, + ); const fastT = fast?.type ?? null; const helper = method === "findIndex" || method === "findLastIndex" @@ -1134,6 +1225,7 @@ export function lowerArrayFindLikeCall( callbackSite(fast ?? fnArg), fastT, present, + unread, ) : someEveryHelper( lowerer, @@ -1145,6 +1237,7 @@ export function lowerArrayFindLikeCall( callbackSite(fast ?? fnArg), fastT, present, + unread, ); return { kind: "call", @@ -1182,6 +1275,7 @@ export function lowerArrayFindLikeCall( retag, callbackSite(fnArg), present, + unread, ); return { kind: "call", callee: helper, args: [receiver, fnArg], type: resultT, loc }; } @@ -1275,9 +1369,10 @@ function findHelper( retag: string | null, site = "", present = false, + unread = false, ): string { const method = last ? "findLast" : "find"; - const key = `${method}:${typeKey(elem)}:${typeKey(resultT)}:${typeKey(fnRet)}:${arity}${site}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(resultT)}:${typeKey(fnRet)}:${arity}${site}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1296,28 +1391,62 @@ function findHelper( type: resultT, loc, }; - const visitBody: IrStmt[] = [ - { - kind: "varDecl", - localId: "v.0", - init: callbackValueRead( - lowerer, - varRef("a.0", arrT, loc), - varRef("i.0", F64, loc), - elem, - present, - loc, - ), + const read: IrStmt = { + kind: "varDecl", + localId: "v.0", + init: callbackValueRead( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + present, loc, - }, - { - kind: "if", - cond: predicateToBoolean(callArrayCallback(arrT, fnT, fnRet, arity, v, loc)), - then: [{ kind: "return", value: found, loc }], - else_: null, - loc, - }, - ]; + ), + loc, + }; + // A predicate that ignores the element of an array that can hold + // elements outside its element type keeps a snapshot of the slot, read + // before the predicate runs as JavaScript reads it (arraySlotSnapshot): + // the predicate is called without reading it as typed, and a found element + // outside the element type takes the checked conversion into the result. + const ignored = (arity === 0 || unread) && !present && elementMayBeForeign(lowerer, elem); + const placeholder = unread ? unreadArgument(lowerer, valueT, loc) : null; + const visitBody: IrStmt[] = ignored + ? [ + ...arraySlotSnapshot( + lowerer, + varRef("a.0", arrT, loc), + varRef("i.0", F64, loc), + elem, + FILTER_SNAPSHOT, + loc, + ), + { + kind: "if", + cond: predicateToBoolean( + callArrayCallback(arrT, fnT, fnRet, arity, placeholder ?? v, loc), + ), + then: [ + { + kind: "return", + value: arraySnapshotValue(lowerer, elem, FILTER_SNAPSHOT, resultT, () => found, loc), + loc, + }, + ], + else_: null, + loc, + }, + ] + : [ + read, + { + kind: "if", + cond: predicateToBoolean(callArrayCallback(arrT, fnT, fnRet, arity, v, loc)), + then: [{ kind: "return", value: found, loc }], + else_: null, + loc, + }, + ]; const visit: IrStmt[] = visitBody; const loop = last ? reverseArrayLoop(loc, visit) @@ -1334,6 +1463,9 @@ function findHelper( locals: [ ...arrayCallbackLocals(arrT, fnT), { id: "v.0", name: "v", type: valueT, mutable: false }, + ...(ignored + ? slotSnapshotLocals(lowerer, elem, FILTER_SNAPSHOT).filter((l) => l.id !== "v.0") + : []), ], body, loc, @@ -1375,9 +1507,10 @@ function findIndexHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { const method = last ? "findLastIndex" : "findIndex"; - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1391,6 +1524,7 @@ function findIndexHelper( outElem: elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([callbackValueType(lowerer, elem, present), F64, arrT].slice(0, arity), fnRet); @@ -1411,6 +1545,7 @@ function findIndexHelper( elem, present, loc, + unread, ), loc, ), @@ -1460,8 +1595,9 @@ function someEveryHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -1474,6 +1610,7 @@ function someEveryHelper( outElem: elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([callbackValueType(lowerer, elem, present), F64, arrT].slice(0, arity), fnRet); @@ -1490,6 +1627,7 @@ function someEveryHelper( elem, present, loc, + unread, ), loc, ), @@ -1553,12 +1691,16 @@ export function lowerArrayFlatMapCall( const argNode = call.arguments.nodes[0]; if (!argNode) lowerer.unsupported("SC1090", call, "this call form"); // tsc-guarded const present = presentElements(lowerer, call, elem); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [callbackValueType(lowerer, elem, present)], arrT, ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && !present && elementMayBeForeign(lowerer, elem); const fnRet = fnArg.type.ret; if ( fnRet.kind === "dyn" || @@ -1603,12 +1745,13 @@ export function lowerArrayFlatMapCall( if (fnRet.kind === "void" || fnRet.kind === "func") lowerer.badType(call, lowerer.typeOf(call)); requireProducedArrayElement(lowerer, call, "'.flatMap()'", fnRet); const outElem = callbackArrayElement(lowerer, call, fnRet); - const fast = present - ? null - : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { - fnArg, - arity, - }); + const fast = + present || unread + ? null + : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { + fnArg, + arity, + }); const helper = arrayHofHelper( lowerer, "flatMap", @@ -1620,6 +1763,7 @@ export function lowerArrayFlatMapCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); return { kind: "call", @@ -1629,12 +1773,13 @@ export function lowerArrayFlatMapCall( loc, }; } - const fast = present - ? null - : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { - fnArg, - arity, - }); + const fast = + present || unread + ? null + : lowerFastCallback(lowerer, argNode, [arrayValueType(lowerer, elem)], 0, elem, arrT, { + fnArg, + arity, + }); const helper = flatMapHelper( lowerer, elem, @@ -1644,6 +1789,7 @@ export function lowerArrayFlatMapCall( callbackSite(fast ?? fnArg), fast?.type ?? null, present, + unread, ); return { kind: "call", @@ -1943,8 +2089,9 @@ function flatMapHelper( site = "", fastT: CallbackType | null = null, present = false, + unread = false, ): string { - const key = `flatMap:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}`; + const key = `flatMap:${typeKey(elem)}:${typeKey(fnRet)}:${arity}${site}${fastT ? ":split" : ""}${present ? ":present" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, "%arr.flatMap", key); @@ -1957,6 +2104,7 @@ function flatMapHelper( outElem: fnRet.elem, present, split: !!fastT, + unread, }); const arrT = arrayOf(elem); const inner = fnRet.elem; @@ -2053,6 +2201,7 @@ function flatMapHelper( elem, present, loc, + unread, ), loc, ), @@ -2125,7 +2274,7 @@ export function lowerArrayReduceCall( lowerer.runtimeOptionalReduceTypes.get(call) ?? (hasInit ? lowerer.irTypeOf(call) : arrayValueType(lowerer, elem)); if (accT.kind === "void" || accT.kind === "func") lowerer.badType(call, lowerer.typeOf(call)); - const { fnArg, arity } = lowerArrayCallback( + const lowered = lowerArrayCallback( lowerer, argNode, [accT, arrayValueType(lowerer, elem)], @@ -2133,17 +2282,23 @@ export function lowerArrayReduceCall( false, accT, ); - const fast = lowerFastCallback( - lowerer, - argNode, - [accT, arrayValueType(lowerer, elem)], - 1, - elem, - arrT, - { fnArg, arity }, - false, - accT, - ); + const { fnArg, arity } = lowered; + // Only an array that can hold elements outside its element type needs to + // leave an element its callback ignores unread. + const unread = lowered.unread && elementMayBeForeign(lowerer, elem); + const fast = unread + ? null + : lowerFastCallback( + lowerer, + argNode, + [accT, arrayValueType(lowerer, elem)], + 1, + elem, + arrT, + { fnArg, arity }, + false, + accT, + ); const helper = reduceHelper( lowerer, method, @@ -2155,6 +2310,7 @@ export function lowerArrayReduceCall( loc, callbackSite(fast ?? fnArg), fast?.type ?? null, + unread, ); const args: IrExpr[] = [receiver, fnArg]; if (hasInit) args.push(lowerer.lowerExprExpecting(call.arguments.nodes[1]!, accT)); @@ -2185,8 +2341,9 @@ function reduceHelper( loc: SrcLoc, site = "", fastT: CallbackType | null = null, + unread = false, ): string { - const key = `${method}:${typeKey(elem)}:${typeKey(accT)}:${typeKey(fnRet)}:${arity}:${hasInit ? "init" : "seed"}${site}${fastT ? ":split" : ""}`; + const key = `${method}:${typeKey(elem)}:${typeKey(accT)}:${typeKey(fnRet)}:${arity}:${hasInit ? "init" : "seed"}${site}${fastT ? ":split" : ""}${unread ? ":unread" : ""}`; const existing = lowerer.arrHofHelpers.get(key); if (existing) return existing; const name = keyedHelperName(lowerer, `%arr.${method}`, key); @@ -2201,6 +2358,7 @@ function reduceHelper( present: false, split: !!fastT, hasInit, + unread, }); const arrT = arrayOf(elem); const fnT = funcOf([accT, arrayValueType(lowerer, elem), F64, arrT].slice(0, arity), fnRet); @@ -2210,6 +2368,9 @@ function reduceHelper( const right = method === "reduceRight"; const at = (index: IrExpr): IrExpr => arrayValueRead(lowerer, varRef("a.0", arrT, loc), index, elem, loc); + // The visited element, or a placeholder for a callback that never reads it. + const placeholder = unread ? unreadArgument(lowerer, arrayValueType(lowerer, elem), loc) : null; + const visited = (): IrExpr => placeholder ?? at(i); const locals: IrLocal[] = [ { id: "a.0", name: "a", type: arrT, mutable: true }, { id: "f.0", name: "f", type: fnT, mutable: true }, @@ -2320,7 +2481,7 @@ function reduceHelper( { kind: "callValue", callee: varRef("f.0", fnT, loc), - args: [varRef("acc.0", accT, loc), at(i), i, varRef("a.0", arrT, loc)].slice( + args: [varRef("acc.0", accT, loc), visited(), i, varRef("a.0", arrT, loc)].slice( 0, arity, ), diff --git a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts index 6307d871f4..5cdbe4d31d 100644 --- a/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts +++ b/packages/compiler/src/frontend/lowering/containers/array-pipelines.ts @@ -6,6 +6,7 @@ import { AmbientReceiverReaders } from "../../../ir/receiver-readers.js"; import { boolLit, countedFor, numLit, varRef } from "../../../ir/build.js"; import { BOOL, + DYN, F64, VOID, type IrBindingSource, @@ -32,9 +33,14 @@ import { import { arrayIndexPresent, arrayValueRead, + arraySlotSnapshot, + arraySnapshotAppend, + arraySnapshotValue, arrayValueStore, arrayValueType, elementMayBeForeign, + unreadArgument, + type SlotSnapshot, } from "../array-values.js"; import type { Lowerer } from "../lowerer.js"; @@ -62,6 +68,9 @@ export interface ArrayHofInfo { present: boolean; split: boolean; hasInit?: boolean; + /** The callback never reads the element it declares + * (lowerArrayCallback's `unread`). */ + unread?: boolean; } type FuncType = IrType & { kind: "func" }; @@ -610,7 +619,13 @@ function chainCallbacks(chain: Chain): Stage[] | null { /** The values a fused loop hands from one stage to the next: a local of a * known type, or (before the first read) the source slot itself. */ -type Flow = { kind: "value"; id: string; type: IrType } | { kind: "slot" }; +/** What reaches a stage: a value, the source slot not yet read, or a + * snapshot of the slot read before an earlier callback that ignored it + * (arraySlotSnapshot). */ +type Flow = + | { kind: "value"; id: string; type: IrType } + | { kind: "slot" } + | { kind: "snapshot"; ids: SlotSnapshot }; interface Plan { chain: Chain; @@ -1595,13 +1610,21 @@ class FusedLoop { stage: Stage, k: number, lead: IrExpr[], - flow: Flow & { kind: "value" }, + flow: Flow, ): { pre: IrStmt[]; value: IrExpr } | null => { const arity = stage.info.arity; - const value = varRef(flow.id, flow.type, loc); const at = lead.length; const pick = (): { arg: IrExpr & { type: FuncType }; value: IrExpr | null } | null => { if (arity <= at) return { arg: stage.slow, value: null }; + // A callback that reads the value reaches it read (`read`); one + // that never reads it takes a placeholder for the unread slot. + if (flow.kind !== "value") { + const placeholder = stage.info.unread + ? unreadArgument(this.lowerer, stage.slow.type.params[at]!, loc) + : null; + return placeholder ? { arg: stage.slow, value: placeholder } : null; + } + const value = varRef(flow.id, flow.type, loc); const fast = stage.fast; if (fast && typeEquals(fast.type.params[at]!, flow.type)) return { arg: fast, value }; const paramT = stage.slow.type.params[at]!; @@ -1630,11 +1653,62 @@ class FusedLoop { return { pre: [...before, ...called.pre], value: called.value }; }; + /** The stage's callback never reads the element at parameter `at`, and + * a placeholder of its type stands for it (unreadArgument). */ + const unreadSlot = (stage: Stage, at: number): boolean => + stage.info.unread === true && + unreadArgument(this.lowerer, stage.slow.type.params[at]!, loc) !== null; + const fail = (): IrStmt[] => { this.failed = true; return []; }; + /** Bring `flow` to the callback of `stage`, whose element parameter is + * at `at`. A slot is read for a callback that reads it. For one that + * ignores it, the slot of an array that can hold elements outside its + * element type is read as stored before the callback runs (a snapshot, + * as JavaScript reads the element first), so such an element passes the + * callback as it is; other arrays read the slot as usual. A snapshot + * reaching a callback that reads it becomes its value, and an element + * outside the element type takes the ordinary read's checked + * conversion there. */ + const reach = ( + stage: Stage, + at: number, + flow: Flow, + rest: (flow: Flow) => IrStmt[], + ): IrStmt[] => { + const reads = stage.info.arity > at && !unreadSlot(stage, at); + if (flow.kind === "slot") { + if (reads || !elementMayBeForeign(this.lowerer, arrT.elem)) + return read(stage, (value) => rest(value)); + const ids: SlotSnapshot = { + isForeign: this.local("sf", BOOL), + value: this.local("sv", arrayValueType(this.lowerer, arrT.elem)), + foreign: this.local("sd", DYN), + }; + return [ + ...arraySlotSnapshot(this.lowerer, a, i, arrT.elem, ids, loc), + ...rest({ kind: "snapshot", ids }), + ]; + } + if (flow.kind === "snapshot" && reads) { + const valueT = arrayValueType(this.lowerer, arrT.elem); + const u = this.local("u", valueT); + return [ + { + kind: "varDecl", + localId: u, + init: arraySnapshotValue(this.lowerer, arrT.elem, flow.ids, valueT, (v) => v, loc), + loc, + }, + ...rest({ kind: "value", id: u, type: valueT }), + ]; + } + return rest(flow); + }; + /** Turn the source slot into a value for the callback of `stage`. */ const read = (stage: Stage, rest: (flow: Flow & { kind: "value" }) => IrStmt[]): IrStmt[] => { if (stage.info.present) { @@ -1690,11 +1764,12 @@ class FusedLoop { }; /** The statements that run one value through stages k.. and the terminal. */ - const visit = (k: number, flow: Flow): IrStmt[] => { + const visit = (k: number, flow: Flow, reached = false): IrStmt[] => { if (k === chain.stages.length) return finish(flow); const stage = chain.stages[k]!; const info = stage.info; - if (flow.kind === "slot") return read(stage, (value) => visit(k, value)); + if (flow.kind !== "value" && !reached) + return reach(stage, 0, flow, (next) => visit(k, next, true)); const called = deliver(stage, k, [], flow); if (!called) return fail(); const { pre, value: result } = called; @@ -1791,11 +1866,23 @@ class FusedLoop { } }; + /** Append a snapshot of the source slot to the collected result: a + * filter whose callback ignored the element keeps it as it was stored + * (arraySnapshotAppend). */ + const copySlot = (flow: Flow & { kind: "snapshot" }): IrStmt[] => { + const outT = resultT as IrType & { kind: "array" }; + if (holes || !typeEquals(outT.elem, arrT.elem)) return fail(); + return [ + arraySnapshotAppend(this.lowerer, varRef(out!, resultT, loc), arrT.elem, flow.ids, loc), + ]; + }; /** The terminal's action for one value. */ - const finish = (flow: Flow): IrStmt[] => { + const finish = (flow: Flow, reached = false): IrStmt[] => { if (terminal.kind === "length") return [increment(count!, loc)]; if (terminal.kind === "collect") { - if (flow.kind !== "value") return fail(); + if (flow.kind === "snapshot") return copySlot(flow); + if (flow.kind === "slot") + return read(chain.stages[chain.stages.length - 1]!, (value) => finish(value)); const last = chain.stages[chain.stages.length - 1]!; const outT = resultT as IrType & { kind: "array" }; if (!typeEquals(outT.elem, last.info.outElem)) return fail(); @@ -1816,7 +1903,9 @@ class FusedLoop { } const stage = terminal.stage; const method = stage.info.method; - if (flow.kind === "slot") return read(stage, (value) => finish(value)); + const lead = method === "reduce" ? 1 : 0; + if (flow.kind !== "value" && !reached) + return reach(stage, lead, flow, (next) => finish(next, true)); if (method === "reduce") { const accT = stage.info.outElem; const called = deliver(stage, chain.stages.length, [varRef(acc!, accT, loc)], flow); diff --git a/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts b/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts index de027e08c5..3b504861a4 100644 --- a/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts +++ b/packages/compiler/src/frontend/lowering/containers/callback-arguments.ts @@ -53,7 +53,7 @@ export function lowerArrayCallback( arrT: IrType, bindUntyped = false, expectedReturn?: IrType, -): { fnArg: IrExpr & { type: IrType & { kind: "func" } }; arity: number } { +): { fnArg: IrExpr & { type: IrType & { kind: "func" } }; arity: number; unread: boolean } { // `xs.map(((x) => x))`: a parenthesized callback is the function itself. while (ts.isParenthesizedExpression(argNode)) argNode = argNode.expression; // The ambient `Boolean` as the callback (`flags.every(Boolean)`, @@ -72,7 +72,11 @@ export function lowerArrayCallback( if (valueT === undefined || valueT.kind === "void" || isUnitType(valueT)) lowerer.badType(argNode, lowerer.typeOf(argNode)); if (valueT.kind === "union") lowerer.requireTruthyUnion(valueT.unionId, argNode); - return { fnArg: truthyFilterCallback(lowerer, valueT, locOf(argNode)), arity: 1 }; + return { + fnArg: truthyFilterCallback(lowerer, valueT, locOf(argNode)), + arity: 1, + unread: false, + }; } // The ambient `String` (`values.map(String)`): String(value) converts its // first argument as a direct call does, whatever the element type; the @@ -86,7 +90,7 @@ export function lowerArrayCallback( const valueT = lead[0]; if (valueT === undefined || valueT.kind === "void") lowerer.badType(argNode, lowerer.typeOf(argNode)); - return { fnArg: stringConversionCallback(lowerer, valueT, argNode), arity: 1 }; + return { fnArg: stringConversionCallback(lowerer, valueT, argNode), arity: 1, unread: false }; } const full = [...lead, F64, arrT]; // A DYN-receiver HOF's callback (`parsed.flatMap((value) => ...)`): @@ -276,9 +280,32 @@ export function lowerArrayCallback( return { fnArg: fnArg as IrExpr & { type: IrType & { kind: "func" } }, arity: fnArg.type.params.length, + unread: !readsParameter(argNode, lead.length - 1), }; } +/** Whether a callback can observe its parameter at `index`: false for a + * function or arrow expression that declares no such parameter, or binds + * it to a plain name that appears nowhere else in it (any identifier of + * that name counts, including shorthand properties and shadowing + * bindings). Any other callback may. */ +function readsParameter(argNode: ts.Expression, index: number): boolean { + if (!ts.isArrowFunction(argNode) && !ts.isFunctionExpression(argNode)) return true; + const param = argNode.parameters.nodes[index]; + if (!param) return false; + if (!ts.isIdentifier(param.name) || param.initializer || param.dotDotDotToken) return true; + const name = param.name.text; + const mentions = (node: ts.Node): boolean => + (node !== param.name && + ts.isIdentifier(node) && + (node.text === name || node.text === "arguments")) || + (ts.forEachChild(node, (child) => (mentions(child) ? true : undefined)) ?? false); + return ( + argNode.parameters.nodes.some((other) => other !== param && mentions(other)) || + mentions(argNode.body) + ); +} + /** Preserve the checker-selected payload ABI when a callback's runtime * return gained only undefined. The array state byte carries that value. */ export function callbackArrayElement( diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index 29f249282f..bbeeb78918 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -1,4 +1,6 @@ /* Element reads and writes (`a[i]`, `a[i] = v`), compound element updates, and keyed record reads. */ +import { OBJECT_PROTOTYPE_MEMBERS } from "../../shape-inference/domain.js"; +import { lowerUnionTupleRead } from "./union-fields.js"; import * as ts from "../../ts7/adapter.js"; import { lowerOptionalIndex, @@ -465,6 +467,12 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre "tuple indexing with a non-literal index (tuples are fixed-shape — use t[0], t[1], ...)", ); } + // A union of tuples asserted to one tuple view (`(r as readonly + // [boolean, T | E])[0]`) still holds the union. + if (obj.type.kind === "union") { + const element = lowerUnionTupleRead(lowerer, obj, idx, locOf(expr)); + if (element) return lowerer.maybeNarrow(element, expr); + } const shapeId = obj.type.kind === "record" ? obj.type.shapeId : receiverIr.shapeId; const actualShape = lowerer.shapes.get(shapeId) ?? shape; const fieldType = actualShape.fields.find((f) => f.name === String(idx))?.type; @@ -597,6 +605,15 @@ export function lowerElementAccess(lowerer: Lowerer, expr: ts.ElementAccessExpre } } } + if (key.type.kind === "f64" && ts.isNumericLiteral(expr.argumentExpression)) { + const element = lowerUnionTupleRead( + lowerer, + value, + Number(expr.argumentExpression.text), + locOf(expr), + ); + if (element) return lowerer.maybeNarrow(element, expr); + } if (key.type.kind === "f64") { const optional = optionalArmIndexRead(lowerer, value, key, locOf(expr)); if (optional) return lowerer.maybeNarrow(optional, expr); @@ -1319,6 +1336,21 @@ export function lowerRecordKeyRead( }; return lowerer.maybeNarrow(get, expr); } + // A literal key the shape does not declare, on a shape without an index + // signature, names no own property of any of its values: the read + // answers undefined, as Node's does. The checker's key check cannot vouch + // for it here, since the receiver can be typed from a package's + // JavaScript while the checker read its declarations. Object.prototype + // members keep the keyed read below. + if (shape.indexValue === undefined && !OBJECT_PROTOTYPE_MEMBERS.has(litKey)) { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: obj, loc }], + result: dynUndefinedExpr(loc), + type: DYN, + loc, + }; + } } let key = litKey !== null @@ -1489,7 +1521,7 @@ function lowerRecordPropertyKey(lowerer: Lowerer, key: IrExpr, node: ts.Expressi * exactly the key JS derives; negative/computed keys stay runtime * expressions and canonicalize through ensureString instead). Null for * everything else. */ -function recordKeyLiteralText(node: ts.Expression): string | null { +export function recordKeyLiteralText(node: ts.Expression): string | null { if (ts.isStringLiteral(node)) return node.text; if (ts.isNumericLiteral(node)) return String(Number(node.text)); return null; diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index 26014cfd90..a6957401a8 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -318,10 +318,12 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // that does not exist. The blocked-binding cascade names the use; // without this the validator's registration check ICEs on the live // reference (`@this {T}` inference — signature 07). - const unregistered = lowerer.unregisteredClassOf(g.type, new Set()); + let unregistered = lowerer.unregisteredClassOf(g.type, new Set()); + // The class's own rejection is the root cause the cascade names + // (unless the flush registers it with dynamic slots). + if (unregistered !== null && lowerer.flushDeferredClass(unregistered) !== undefined) + unregistered = lowerer.unregisteredClassOf(g.type, new Set()); if (unregistered !== null) { - // The class's own rejection is the root cause the cascade names. - lowerer.flushDeferredClass(unregistered); lowerer.pushDiag(blockedBindingUseDiag(expr.text, loc)); throw new PoisonError(); } diff --git a/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts b/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts new file mode 100644 index 0000000000..698bf4dc3d --- /dev/null +++ b/packages/compiler/src/frontend/lowering/expressions/message-listeners.ts @@ -0,0 +1,22 @@ +import * as ts from "../../ts7/adapter.js"; + +const LISTENER_METHODS = new Set([ + "on", + "once", + "addListener", + "prependListener", + "prependOnceListener", +]); + +/** `.on("message", …)` and the other listener registrations. */ +export function isMessageListenerCall(call: ts.CallExpression): boolean { + const callee = call.expression; + const event = call.arguments.nodes[0]; + return ( + ts.isPropertyAccessExpression(callee) && + LISTENER_METHODS.has(callee.name.text) && + event !== undefined && + ts.isStringLiteral(event) && + event.text === "message" + ); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index f2e3880185..91b63cabc7 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -51,6 +51,7 @@ import { isConstAssertionTypeNode, unitOnlyUnion } from "../../type-mapper.js"; import { assertsGlobalObject, extractionCopiesReference } from "../assertion-identity.js"; import { iterableSlotValue, iterableUnionOf } from "../iterable-values.js"; import { isDeclBoundaryTypeNode } from "../../decl-projection.js"; +import { isMessageListenerCall } from "./message-listeners.js"; /** `c ? a : b`. An explicit destination constructs each branch at that * layout before joining it. Without one, retain the expression's inferred @@ -1586,7 +1587,7 @@ export function lowerAsExpression( // contains undefined, so casts over parse results keep failing on // non-string values with the usual path-annotated TypeError. if (target.kind === "union" && lowerer.dynConvertible(target)) { - return { kind: "dynCheck", value: inner, type: target, loc: locOf(expr) }; + return checkedDynCast(lowerer, expr, inner, target); } // Uint8Array targets: the checked-dynamic tree carries a bytes kind now (converted // stdin chunks) — the extraction validates the kind and copies out. @@ -1630,9 +1631,145 @@ export function lowerAsExpression( `number, string, boolean, records, arrays, and unions of those)`, ); } + return checkedDynCast(lowerer, expr, inner, target); +} + +/** The checked conversion of a dynamic value. A record or array built from + * a value another holder may still reference keeps one identity with it + * (adoptingCast); only a source nothing else can hold (freshDynSource) + * converts outright, which keeps the fused `JSON.parse(text) as T`. */ +function checkedDynCast( + lowerer: Lowerer, + expr: ts.AsExpression | ts.TypeAssertion, + inner: IrExpr, + target: IrType, +): IrExpr { + if ( + holdsReference(lowerer, target) && + !freshDynSource(lowerer, expr, inner) && + !resultOnlyRead(lowerer, expr) + ) { + return adoptingCast(lowerer, inner, target, locOf(expr), "dyn.adoptShared"); + } return { kind: "dynCheck", value: inner, type: target, loc: locOf(expr) }; } +/** A cast whose result only serves one read of a primitive member + * (`(value as T).name`, `(value as T[]).length`): the converted value is + * never stored, compared, written or called through, and the read yields no + * reference into it, so whether it is the source itself or a copy cannot be + * observed. */ +function resultOnlyRead(lowerer: Lowerer, expr: ts.AsExpression | ts.TypeAssertion): boolean { + let node: ts.Node = expr; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) node = node.parent; + const access = node.parent; + if ( + access === undefined || + !( + (ts.isPropertyAccessExpression(access) || ts.isElementAccessExpression(access)) && + access.expression === node + ) + ) + return false; + const primitive = + ts.TypeFlags.StringLike | + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.Undefined | + ts.TypeFlags.Null; + const read = lowerer.typeOf(access); + const parts = read.isUnionType() ? ts.constituentTypes(read) : [read]; + if (!parts.every((part) => (part.flags & primitive) !== 0)) return false; + let use: ts.Node = access; + while (use.parent !== undefined && ts.isParenthesizedExpression(use.parent)) use = use.parent; + const parent = use.parent; + if (parent === undefined) return true; + if (ts.isCallExpression(parent) && parent.expression === use) return false; + if ( + ts.isBinaryExpression(parent) && + parent.left === use && + ts.isAssignmentOperator(parent.operatorToken.kind) + ) + return false; + if ( + (ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent)) && + (parent.operator === ts.SyntaxKind.PlusPlusToken || + parent.operator === ts.SyntaxKind.MinusMinusToken) + ) + return false; + if (ts.isDeleteExpression(parent)) return false; + return true; +} + +/** A cast operand nothing else can hold: a value this expression creates + * (`JSON.parse(text)`, `await response.json()`), or a structured clone the + * runtime delivers once and the program reads at this site only (`workerData` + * read nowhere else; the parameter of the program's one `"message"` + * listener, read nowhere else in it). Anything else is adopted. */ +function freshDynSource( + lowerer: Lowerer, + expr: ts.AsExpression | ts.TypeAssertion, + inner: IrExpr, +): boolean { + if (inner.kind === "libCall" && inner.fn === "json.parse") return true; + if ( + inner.kind === "awaitExpr" && + inner.value.kind === "libCall" && + inner.value.fn === "fetch.responseJson" + ) + return true; + const facts = lowerer.structuredCloneReads(); + if (inner.kind === "libCall" && inner.fn === "worker.data") return facts.workerData === 1; + let operand: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(operand)) operand = operand.expression; + if (!ts.isIdentifier(operand) || inner.kind !== "varRef" || facts.messageListeners !== 1) + return false; + const symbol = lowerer.checker.getSymbolAtLocation(operand); + const param = symbol?.valueDeclaration; + if (!param || !ts.isParameter(param) || !ts.isIdentifier(param.name)) return false; + const fn = param.parent; + if ( + !(ts.isArrowFunction(fn) || ts.isFunctionExpression(fn)) || + fn.parameters.nodes[0] !== param || + !isMessageListener(fn) + ) + return false; + let reads = 0; + const name = param.name.text; + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === name && node !== param.name) reads++; + ts.forEachChild(node, visit); + }; + ts.forEachChild(fn, visit); + return reads === 1; +} + +/** The listener argument of `.on("message", fn)` (also once, + * addListener, prependListener, prependOnceListener) registered once: at + * module level, outside any function, class or loop, so one emitter can + * never hand the same message to two registrations of it. */ +function isMessageListener(fn: ts.Node): boolean { + const call = fn.parent; + if (!call || !ts.isCallExpression(call) || call.arguments.nodes[1] !== fn) return false; + if (!isMessageListenerCall(call)) return false; + for (let p = call.parent; p !== undefined && !ts.isSourceFile(p); p = p.parent) { + if ( + ts.isFunctionLike(p) || + ts.isClassDeclaration(p) || + ts.isClassExpression(p) || + ts.isClassStaticBlockDeclaration(p) || + ts.isForStatement(p) || + ts.isForInStatement(p) || + ts.isForOfStatement(p) || + ts.isWhileStatement(p) || + ts.isDoStatement(p) + ) + return false; + } + return true; +} + /** True when a type holds a record or an array somewhere (values whose * identity can be observed). */ function holdsReference(lowerer: Lowerer, t: IrType, seen = new Set()): boolean { @@ -1714,6 +1851,24 @@ function declBoundaryCast( ...(originId === undefined ? {} : { origin: originId }), }; } + return adoptingCast(lowerer, inner, target, loc, "dyn.declAdopt", originId); +} + +/** The checked conversion of `inner` to `target` followed by adoption + * (`dyn.declAdopt` at the boundary, the lenient `dyn.adoptShared` + * elsewhere): every source object or array that something else still + * references becomes a live capsule of the native value built from it. + * `originId` names the package declaration a failing check reports + * (decl-origin.ts). */ +function adoptingCast( + lowerer: Lowerer, + inner: IrExpr, + target: IrType, + loc: SrcLoc, + fn: "dyn.declAdopt" | "dyn.adoptShared", + originId?: number, +): IrExpr { + const boundary = fn === "dyn.declAdopt"; const source = lowerer.declareHiddenLocal("%boundarySource", DYN); const value = lowerer.declareHiddenLocal("%boundaryValue", target); return { @@ -1726,7 +1881,7 @@ function declBoundaryCast( init: { kind: "dynCheck", value: varRef(source.id, DYN, loc), - adopt: true, + ...(boundary ? { adopt: true as const } : {}), type: target, loc, ...(originId === undefined ? {} : { origin: originId }), @@ -1737,7 +1892,7 @@ function declBoundaryCast( kind: "exprStmt", expr: { kind: "libCall", - fn: "dyn.declAdopt", + fn, args: [ varRef(source.id, DYN, loc), { diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index 1eaf3fb87d..31479e4b01 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -419,7 +419,12 @@ export function lowerDynObjectLiteral( : !boxValue && (ts.isObjectLiteralExpression(valueExpr) || (ts.isArrayLiteralExpression(valueExpr) && - holdsEvolvingArray(lowerer, valueExpr))) + // An empty literal takes whatever the object's users + // store into it (`{ [kChildren]: [] }`), so it is a + // checked-dynamic array rather than its `never[]` layout. + (valueExpr.elements.nodes.length === 0 || + holdsEvolvingArray(lowerer, valueExpr) || + lowerer.untypedLiteralContext(valueExpr)))) ? lowerer.lowerExprExpecting(valueExpr, DYN) : lowerer.lowerExpr(valueExpr as ts.Expression); raw = fenceClosureProbe(lowerer, valueExpr, undefined, lowerValue) ?? lowerValue(); diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 6bdf9e2f05..97ec9a320e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -3376,6 +3376,18 @@ export function lowerUnionProperty( expr, ); } + // An array the checker sees through a union of arrays (`readdirSync`'s + // `string[] | Buffer[]`, whose lowering answers the string[] arm). + if (value.type.kind === "array" && expr.name.text === "length") { + return { + kind: "arrIntrinsic", + method: "length", + receiver: value, + args: [], + type: F64, + loc: locOf(expr), + }; + } if (value.type.kind !== "union") { lowerer.unsupported( "SC1090", @@ -3651,13 +3663,13 @@ export function classFieldTarget( ) { return { container: "dynamic", obj: represented, field: fieldName, fieldType: DYN }; } - const info = lowerer.classes.get(receiverType.className); - if (!info) { - // A receiver typed as a class whose collection deferred: the - // deferred diagnostics are what explains the miss. + // A receiver typed as a class whose collection deferred: the deferred + // diagnostics are what explains the miss (unless the flush registers + // the class with dynamic slots, retryDeferred). + const info = + lowerer.classes.get(receiverType.className) ?? lowerer.flushDeferredClass(receiverType.className); - return null; - } + if (!info) return null; const fieldType = info.fields.get(fieldName); const lowerObjectReceiver = (): IrExpr => { let obj = represented; diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 4bf5413a35..ae3b9c49e0 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -551,12 +551,13 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc }; } if (rhsClassval?.kind === "classval" && !caught) { - const targetInfo = lowerer.classes.get(rhsClassval.className); + const targetInfo = + lowerer.classes.get(rhsClassval.className) ?? + lowerer.flushDeferredClass(rhsClassval.className); if (!targetInfo) { // The type world names a class the lowering never registered - // (a fenced class expression, a deferred declaration): flush - // its diagnostics and poison the test site, never an ICE. - lowerer.flushDeferredClass(rhsClassval.className); + // (a fenced class expression, a deferred declaration): its + // diagnostics flushed above; poison the test site, never an ICE. lowerer.unsupported( "SC1090", expr, diff --git a/packages/compiler/src/frontend/lowering/expressions/union-fields.ts b/packages/compiler/src/frontend/lowering/expressions/union-fields.ts index 4f4532a6a9..01c841fdd3 100644 --- a/packages/compiler/src/frontend/lowering/expressions/union-fields.ts +++ b/packages/compiler/src/frontend/lowering/expressions/union-fields.ts @@ -281,3 +281,58 @@ export function lowerUnionFieldWrite( } return { kind: "seqExpr", stmts: body, result: dispatch, type: result.type, loc }; } + +/** Read the element at a literal index from a union of tuples whose element + * types differ (`readonly [true, number] | readonly [false, string]`, index + * 1): each tag reads its own tuple's slot and widens it into the union of + * the arms' element types; the caller narrows the result to the checker's + * flow type. Null when an arm is not a tuple with that position. */ +export function lowerUnionTupleRead( + lowerer: Lowerer, + receiver: IrExpr, + index: number, + loc: IrExpr["loc"], +): IrExpr | null { + if (receiver.type.kind !== "union" || !Number.isInteger(index) || index < 0) return null; + const unionId = receiver.type.unionId; + const arms = lowerer.unions.get(unionId)?.arms; + if (!arms || arms.length < 2) return null; + const name = String(index); + const fields: UnionFieldArm[] = []; + const parts: IrType[] = []; + for (const arm of arms) { + if (arm.kind !== "record" || lowerer.shapes.get(arm.shapeId)?.tuple !== true) return null; + const field = unionFieldArm(lowerer, arm, name); + if (!field || field.type.kind === "void") return null; + fields.push(field); + const own = + field.type.kind === "union" ? lowerer.unions.get(field.type.unionId)?.arms : [field.type]; + if (!own) return null; + for (const part of own) if (!parts.some((p) => typeEquals(p, part))) parts.push(part); + } + const type: IrType = + parts.length === 1 ? parts[0]! : { kind: "union", unionId: lowerer.unions.intern(parts) }; + const object = lowerer.declareHiddenLocal("%unionReadObject", receiver.type); + const objectRef = varRef(object.id, object.type, loc); + const read = (tag: number): IrExpr | null => { + const probe = unionFieldRead(unionId, tag, objectRef, fields[tag]!, name, loc); + const converted = lowerer.coerceToExpected(probe, type); + return typeEquals(converted.type, type) && isWidening(converted, probe) ? converted : null; + }; + let dispatch = read(fields.length - 1); + if (!dispatch) return null; + for (let tag = fields.length - 2; tag >= 0; tag--) { + const then = read(tag); + if (!then) return null; + dispatch = { + kind: "ternary", + cond: { kind: "unionIsTag", unionId, tag, value: objectRef, negated: false, type: BOOL, loc }, + then, + else_: dispatch, + type, + loc, + }; + } + const init: IrStmt = { kind: "varDecl", localId: object.id, init: receiver, loc }; + return { kind: "seqExpr", stmts: [init], result: dispatch, type, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 7aed5a271d..0c4afa9f8e 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -10235,8 +10235,8 @@ export function lowerObjectMethodCall( ? lowerer.coerceInto(access.expression, receiver, receiverIr) : receiver; }; - const info = lowerer.classes.get(receiverIr.className); - if (!info) lowerer.flushDeferredClass(receiverIr.className); + const info = + lowerer.classes.get(receiverIr.className) ?? lowerer.flushDeferredClass(receiverIr.className); const found = info ? lowerer.findMethodOn(info, access.name.text) : null; // The zlib stream members (flush/params/close/reset) are declared on // the zlib classes; their receivers are runtime Transform streams. @@ -10589,11 +10589,8 @@ function lowerUnionObjectMethodCall( found: NonNullable>; }[] = []; for (const arm of def.arms as (IrType & { kind: "object" })[]) { - const info = lowerer.classes.get(arm.className); - if (!info) { - lowerer.flushDeferredClass(arm.className); - return null; - } + const info = lowerer.classes.get(arm.className) ?? lowerer.flushDeferredClass(arm.className); + if (!info) return null; const found = lowerer.findMethodOn(info, method); if (!found) return null; if (found.sig.abstract === true && !lowerer.overrideBelow(info, method)) return null; diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 863d60cfaf..17c7ff59fa 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -117,6 +117,7 @@ import { requireSpecOf, } from "../program.js"; import { PoisonError, dynFallbackType, jsFuncNameOf, newFnCtx, own } from "./lowerer.js"; +import { declarationGroupKey } from "./type-failure.js"; import { isAssertClassNew } from "./assert-instances.js"; import { isCjsRequireGlobal } from "./lower-node-module.js"; import { lowerArrayConstructor } from "./containers/array-construction.js"; @@ -2305,11 +2306,21 @@ export function collectClassShapeInner( base = named; if (!base) { if (adoptCallableBase(t.expression)) continue; - lowerer.unsupported( + // A base whose own collection deferred can still register later + // (a flush retry with dynamic slots): the derived class retries + // with it (Lowerer.retryDeferred). + const diag = unsupportedDiag( "SC1090", - t, + locOf(t), `extending classes not declared in the program ('${t.expression.text}')`, ); + lowerer.retriableDiags.add(diag); + // A base the program declares but rejected: this rejection is its + // cascade (localized failure, report-only). + const rejectedBase = rejectedProgramClassGroup(lowerer, symbol); + if (rejectedBase !== null) diag.cause = [rejectedBase]; + lowerer.pushDiag(diag); + throw new PoisonError(); } } @@ -2545,13 +2556,13 @@ export function collectClassShapeInner( if (type.kind === "void") lowerer.badType(member.name, lowerer.typeOf(member.name)); if ( type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && !isJsSourceFile(ts.getSourceFileOfNode(member)) && !lowerer.classFieldStoresDyn(lowerer.typeOf(member.name)) && (member.type !== undefined || (lowerer.typeOf(member.name).flags & ts.TypeFlags.Any) === 0) ) { - lowerer.unsupported( - "SC1090", + lowerer.dynFieldPolicyFence( member.name, `${dynFieldTypeSpelling(lowerer, member.name)}-typed static fields`, ); @@ -3061,14 +3072,16 @@ export function collectClassShapeInner( if (type.kind === "void") lowerer.badType(member.name, lowerer.typeOf(member.name)); // Bundled JS declares fields without annotations before assigning // them in the constructor. Use the same native checked-dynamic - // storage as implicit constructor-assigned JS fields. + // storage as implicit constructor-assigned JS fields. In + // TypeScript the refusal is a policy fence that a flush retry lifts + // (Lowerer.liftDynFieldPolicy). if ( type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && !isJsSourceFile(ts.getSourceFileOfNode(member)) && !lowerer.classFieldStoresDyn(lowerer.typeOf(member.name)) ) { - lowerer.unsupported( - "SC1090", + lowerer.dynFieldPolicyFence( member.name, `${dynFieldTypeSpelling(lowerer, member.name)}-typed class fields`, ); @@ -3204,9 +3217,12 @@ export function collectClassShapeInner( const type = shape.bodyType ?? shape.type; if (type.kind === "void") lowerer.badType(p.name, lowerer.typeOf(p.name)); // The class-field dyn rule verbatim (KEEP NARROW). - if (type.kind === "dyn" && !lowerer.classFieldStoresDyn(lowerer.typeOf(p))) { - lowerer.unsupported( - "SC1090", + if ( + type.kind === "dyn" && + !lowerer.liftDynFieldPolicy && + !lowerer.classFieldStoresDyn(lowerer.typeOf(p)) + ) { + lowerer.dynFieldPolicyFence( p.name, `${dynFieldTypeSpelling(lowerer, p)}-typed class fields`, ); @@ -6214,6 +6230,7 @@ function lowerClassValue( const context = newFnCtx(true, null, null, VOID); info.localClass.context = context; lowerer.fnStack.push(context); + const diagsBefore = lowerer.diags.length; try { const bodies = lowerClassMembers(lowerer, info); for (const body of bodies) info.localClass.bodies.set(body.name, body); @@ -6228,6 +6245,8 @@ function lowerClassValue( }); } info.localClass.ready = true; + // A retry of the enclosing statement reuses these bodies. + for (const d of lowerer.diags.slice(diagsBefore)) lowerer.loweredOnceDiags.add(d); lowerer.onExprClassCollected?.(info); } finally { lowerer.fnStack.pop(); @@ -11334,12 +11353,13 @@ export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { ); info = lowerer.classes.get(calleeT.className); } + // The TYPE world names a class the lowering never registered — a + // fenced class expression, an abstract/deferred declaration: flush + // its own diagnostics (they tell the real story, unless the flush + // registers it with dynamic slots) and poison this construction + // site, never an ICE. + if (!info) info = lowerer.flushDeferredClass(calleeT.className); if (!info) { - // The TYPE world names a class the lowering never registered — - // a fenced class expression, an abstract/deferred declaration: - // flush its own diagnostics (they tell the real story) and - // poison this construction site, never an ICE. - lowerer.flushDeferredClass(calleeT.className); lowerer.unsupported( "SC1090", expr, @@ -11869,3 +11889,14 @@ function definedThroughReflection(lowerer: Lowerer, decl: ts.ClassLikeDeclaratio }); return found; } + +/** The failure group of a class the program declares but did not collect + * (its own declaration was rejected), or null for any other symbol. */ +function rejectedProgramClassGroup(lowerer: Lowerer, symbol: ts.Symbol | null): string | null { + if (symbol === null) return null; + const recorded = lowerer.rejectedGroups.get(symbol); + if (recorded !== undefined) return recorded; + const decl = symbol.declarations.find((d) => ts.isClassDeclaration(d) || ts.isClassExpression(d)); + if (decl === undefined || ts.getSourceFileOfNode(decl).isDeclarationFile) return null; + return declarationGroupKey(symbol); +} diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 006975085b..3bc9ed7b06 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -96,7 +96,8 @@ import { lowerUnionKeyedRead, } from "./expressions/property-access.js"; import { lowerNullishPair } from "./expressions/binary.js"; -import { lowerElementCompoundParts } from "./expressions/element-access.js"; +import { lowerElementCompoundParts, recordKeyLiteralText } from "./expressions/element-access.js"; +import { OBJECT_PROTOTYPE_MEMBERS } from "../shape-inference/domain.js"; export { DEFERRED_READ_MESSAGE, abstractPropertyDeclOf, @@ -1964,6 +1965,23 @@ export function lowerAbsenceProbe(lowerer: Lowerer, node: ts.Expression): IrExpr if (shape && !shape.tuple && shape.indexValue && shape.indexValue.kind !== "dyn") { return lowerer.lowerRecordKeyRead(expr, receiverT.shapeId, shape, true); } + // A literal key the shape does not declare is absent from every value + // (lowerRecordKeyRead answers undefined), whatever type the checker gave + // the read. An optional chain keeps its own lowering, which reads a + // nullish receiver as undefined instead of converting it. + const literal = recordKeyLiteralText(expr.argumentExpression); + if ( + !expr.questionDotToken && + !hasOptionalChainGuard(expr.expression) && + shape && + !shape.tuple && + !shape.indexValue && + literal !== null && + !OBJECT_PROTOTYPE_MEMBERS.has(literal) && + !shape.fields.some((field) => field.name === literal) + ) { + return lowerer.lowerRecordKeyRead(expr, receiverT.shapeId, shape, true); + } return null; } // `m[2]` on a JavaScript `RegExpExecArray | null` that a previous @@ -3136,11 +3154,9 @@ export function symbolFieldInfo( if (lowerer.chainBlocked(expr)) return null; const receiverIr = lowerer.mapTypeOf(lowerer.typeOf(expr.expression)); if (receiverIr?.kind !== "object") return null; - const info = lowerer.classes.get(receiverIr.className); - if (!info) { - lowerer.flushDeferredClass(receiverIr.className); - return null; - } + const info = + lowerer.classes.get(receiverIr.className) ?? lowerer.flushDeferredClass(receiverIr.className); + if (!info) return null; const key = classSymbolKeyOf(lowerer, expr.argumentExpression); if (!key) return null; const field = info.symbolFields?.get(key.identity); diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 3f4a70c744..57c58a7607 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -1,3 +1,4 @@ +import type { TypeFailureSlot } from "../../diagnostics/diagnostic.js"; import { THREADS_MODULE } from "../threads-import.js"; import { positionTag } from "./structural-positions.js"; import { EXPERIMENTAL_MODULE } from "../experimental-import.js"; @@ -178,6 +179,17 @@ export interface AppendGraph { * the require() edges preflight resolved — a CommonJS file's only edges * (the compiled CommonJS package a workspace module loaded on demand * imports, and the sibling packages and files IT requires). */ +/** Whether a class declaration has a static field with an initializer, + * which Node runs where the class statement evaluates. */ +function hasStaticFieldInitializer(decl: ts.ClassDeclaration): boolean { + return decl.members.nodes.some( + (member) => + ts.isPropertyDeclaration(member) && + member.initializer !== undefined && + member.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword) === true, + ); +} + function appendedEdgesOf( program: ts.Program, graph: AppendGraph | undefined, @@ -1035,13 +1047,11 @@ export function moduleArtifacts( } while (pendingClasses.length > 0) { const name = pendingClasses.pop()!; - const info = lowerer.classes.get(name); - if (!info) { - // Referenced by an emitted type but never registered: collection - // deferred its diagnostics — a reached reference makes them count. - lowerer.flushDeferredClass(name); - continue; - } + // Referenced by an emitted type but never registered: collection + // deferred its diagnostics — a reached reference makes them count + // (or registers the class with dynamic slots, retryDeferred). + const info = lowerer.classes.get(name) ?? lowerer.flushDeferredClass(name); + if (!info) continue; for (const field of info.def.fields) visitType(field.type); if (info.base) visitClass(info.base.def.name); if (info.def.genericOf) visitClass(info.def.genericOf); @@ -2048,7 +2058,15 @@ export function collectGlobals( // Destructuring declarations register EVERY bound identifier (the // desugar in the init function assigns the pre-registered globals, // exactly like plain declarations). - for (const nameNode of boundIdentifiersOf(decl.name)) { + const names = boundIdentifiersOf(decl.name); + // Localized failure: a degradable type failure at a module global + // degrades that type and the same name registers again + // (globalSlotRetry), at most MAX_GLOBAL_RETRIES times. + let retries = 0; + let retriedIndex = -1; + for (let nameIndex = 0; nameIndex < names.length; nameIndex++) { + const nameNode = names[nameIndex]!; + if (nameIndex !== retriedIndex) retries = 0; const diagsBefore = lowerer.diags.length; try { // Checked JavaScript factories can expose inferred callables by @@ -2700,6 +2718,12 @@ export function collectGlobals( } } catch (e) { if (!(e instanceof PoisonError)) throw e; + if (retries < MAX_GLOBAL_RETRIES && globalSlotRetry(lowerer, sf, e)) { + retries++; + retriedIndex = nameIndex; + nameIndex--; + continue; + } // diagnostic already recorded; lowering the statement poisons // too — and in a JS file BOTH defer to the statement's // runtime fence (deferJsCollectionDiags). @@ -2710,6 +2734,20 @@ export function collectGlobals( } } +/** Localized failure's commit point for module globals (design §3.3): + * collectGlobals commits a global's type failure directly, before any body + * lowers, so it degrades here and the registration runs again. */ +function globalSlotRetry(lowerer: Lowerer, sf: ts.SourceFile, e: PoisonError): boolean { + if (isJsSourceFile(sf)) return false; + const slot = lowerer.degradeAtCommit(e, GLOBAL_SLOTS); + if (slot === null) return false; + lowerer.noteDynamicSlot(slot); + return true; +} + +const MAX_GLOBAL_RETRIES = 16; +const GLOBAL_SLOTS: ReadonlySet = new Set(["global"]); + /** Registers the entry-init note for a `var` module global: JS hoists * module vars to `undefined` at module entry, so an undefined-armed slot * must hold the interned undefined arm BEFORE the body runs (a function @@ -3202,7 +3240,19 @@ export function lowerFileInit( // by position) — exactly when JS evaluates decorators, static // initializers, and blocks, so code reading an earlier module // binding sees its assigned value. - const statics = [...lowerer.classes.values()] + const runsStatics = (c: ClassInfo): boolean => + lowerer.classInitFlags.has(c.def.name) || + c.callableBase !== undefined || + c.def.baseValueGlobal !== undefined || + c.runtimeStatics !== undefined || + c.staticFields.length > 0 || + (c.staticBlocks?.length ?? 0) > 0 || + c.classDecorators !== undefined; + const statics: { + pos: number; + info: ClassInfo | null; + deferred?: { symbol: ts.Symbol; decl: ts.ClassDeclaration }; + }[] = [...lowerer.classes.values()] .filter( (c) => // Class DECLARATIONS only: expression classes run their static @@ -3214,15 +3264,30 @@ export function lowerFileInit( ts.isClassDeclaration(c.decl) && !c.mixinInstance && ts.getSourceFileOfNode(c.decl) === sf && - (lowerer.classInitFlags.has(c.def.name) || - c.callableBase !== undefined || - c.def.baseValueGlobal !== undefined || - c.runtimeStatics !== undefined || - c.staticFields.length > 0 || - (c.staticBlocks?.length ?? 0) > 0 || - c.classDecorators !== undefined), + runsStatics(c), ) .map((c) => ({ pos: ts.tokenStartOfNode(c.decl!), info: c })); + // Localized failure: a deferred class with static field initializers + // is reached at its statement, where Node runs them. Its place retries + // it (retryDeferredAtDeclaration), and once it has registered there or + // through an earlier flush, its initializers lower in place. + for (const symbol of lowerer.deferredCollectors.keys()) + for (const decl of symbol.declarations) + if ( + ts.isClassDeclaration(decl) && + ts.getSourceFileOfNode(decl) === sf && + hasStaticFieldInitializer(decl) + ) + statics.push({ pos: ts.tokenStartOfNode(decl), info: null, deferred: { symbol, decl } }); + const lowerStaticsAt = (entry: (typeof statics)[number]): IrStmt[] => { + let info = entry.info; + if (entry.deferred) { + if (!lowerer.retryDeferredAtDeclaration(entry.deferred.symbol)) return []; + info = lowerer.classes.get(lowerer.classNamer(entry.deferred.decl)) ?? null; + if (info === null || !runsStatics(info)) return []; + } + return withClassInitFlag(lowerer, info!, lowerer.lowerStaticFieldInits(info!)); + }; // Statics-bearing MIXIN instantiations whose call evaluates in THIS // file: their declaration-time code runs when the call does — the // top-level statement lower-mixins recorded. Insertion order is @@ -3243,13 +3308,7 @@ export function lowerFileInit( // declarations never share a kept statement's start, so `<=` // changes nothing for them. while (at < statics.length && statics[at]!.pos <= ts.tokenStartOfNode(stmt)) { - body.push( - ...withClassInitFlag( - lowerer, - statics[at]!.info, - lowerer.lowerStaticFieldInits(statics[at]!.info), - ), - ); + body.push(...lowerStaticsAt(statics[at]!)); at++; } // A cycle binding's declaration stays at its position: its @@ -3268,13 +3327,7 @@ export function lowerFileInit( body.push(...lowerer.pendingClassExprInits.splice(0), ...stmtIr); } while (at < statics.length) { - body.push( - ...withClassInitFlag( - lowerer, - statics[at]!.info, - lowerer.lowerStaticFieldInits(statics[at]!.info), - ), - ); + body.push(...lowerStaticsAt(statics[at]!)); at++; } for (const { at, g } of unionEntryInits) { diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 84196476a6..09f5baab00 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -40,7 +40,9 @@ import { isJsonRequireDecl } from "./json-require.js"; import { isTimeValueDateDeclaration } from "./date-values.js"; import { bindingSource } from "../binding-source.js"; import { lowerConstructorReturn } from "./constructor-returns.js"; -import type { Lowerer } from "./lowerer.js"; +import type { DynamicSlot, Lowerer } from "./lowerer.js"; +import type { ScrDiagnostic, TypeFailureSlot } from "../../diagnostics/diagnostic.js"; +import { statementGroupKey } from "./type-failure.js"; import { cjsExportValueByName, cjsNamespaceDestructureOf, @@ -415,8 +417,8 @@ function cleanupCall( ): { call: IrExpr; async: boolean } | null { const loc = locOf(node); if (value.type.kind === "object") { - const info = lowerer.classes.get(value.type.className); - if (!info) lowerer.flushDeferredClass(value.type.className); + const info = + lowerer.classes.get(value.type.className) ?? lowerer.flushDeferredClass(value.type.className); const asyncFound = awaited && info ? lowerer.findMethodOn(info, "sym:asyncDispose") : null; const syncFound = info ? lowerer.findMethodOn(info, "sym:dispose") : null; const member = asyncFound ? "sym:asyncDispose" : syncFound ? "sym:dispose" : null; @@ -750,57 +752,38 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir if (!lowerer.suppressStats) lowerer.countStatement(stmt, sf?.fileName); const diagsBefore = lowerer.diags.length; const sinkMark = lowerer.diagSinkMark(); + const touched = new Set(); + lowerer.failureTouches.push(touched); + const pendingSlots: DynamicSlot[] = []; + // A declarator that initializes a forward-captured TDZ box takes it + // out of tdzPredeclared; a retry after a later declarator degrades + // must initialize the same box again. + const tdzBefore = new Map(lowerer.ctx.tdzPredeclared); try { - let lowered: IrStmt | IrStmt[] | null; - let appended = false; - if ( - ts.isVariableStatement(stmt) && - (stmt.declarationList.flags & ts.NodeFlags.Using) !== 0 - ) { - const registrations = lowerUsingStatement(lowerer, stmt); - const accounting: IrStmt[] = []; - for (const registration of registrations) { - out.push(...registration.acquire); - accounting.push(...registration.acquire, ...registration.cleanup); - if (registration.cleanup.length > 0) { - usingMarkers.push({ - start: out.length, - cleanup: registration.cleanup, - loc: registration.loc, - }); - } + for (let attempt = 0; ; attempt++) { + try { + lowerOneStatement(lowerer, stmt, out, usingMarkers, diagsBefore, deferFences); + break; + } catch (e) { + // Localized failure's statement commit point (design §3.3): a + // degradable type failure at a local, loop variable or module + // global degrades that type and the statement lowers again. + // A declaration this statement flushed inside a probe (where + // no retry runs) can retry here too. + if (attempt >= MAX_DEGRADE_RETRIES || deferFences) throw e; + const slot = lowerer.degradeAtCommit(e, STATEMENT_SLOTS); + if (slot === null && !lowerer.reopenFlushedAtCommit(diagsBefore)) throw e; + // A failed attempt adds nothing to `out` but the TDZ boxes + // forward captures declared (they stay declared), so the + // retry appends after them. + lowerer.dropAttemptDiags(diagsBefore); + if (slot !== null) pendingSlots.push(slot); + for (const [symbol, local] of tdzBefore) + if (!lowerer.ctx.tdzPredeclared.has(symbol)) + lowerer.ctx.tdzPredeclared.set(symbol, local); } - lowered = accounting; - appended = true; - } else { - lowered = lowerer.lowerStmt(stmt); - } - // The lib-boundary chokepoint (lib-boundary.ts): checked-dynamic - // arguments that reached builtin-call slots without the checked - // coercion get their dynCheck here, and the uncoercible ones - // fence — inside this statement's poison window, so a JS fence - // becomes the statement's runtimeFence exactly like every other - // deferred rejection. - if (lowered) { - const body: IrStmt[] = []; - if (Array.isArray(lowered)) body.push(...lowered); - else body.push(lowered); - const rewritten = lowerSurplusCalls(lowerer, body); - enforceLibBoundary(lowerer, rewritten); - if (!appended) out.push(...rewritten); - } - // A JS statement that lowered CLEAN can still have flushed - // DEFERRED collection diagnostics (a symbol lookup on an alias - // resolves through flushDeferred — informational here, not a - // blocker of THIS statement): those belong to the runtime-fence - // ledger, not the build — real uses of the broken declaration - // meet their own per-site fences. - if (deferFences && lowerer.diagSink === null && lowerer.diags.length > diagsBefore) { - const flushed = lowerer.diags.splice(diagsBefore); - const ice = flushed.filter((d) => d.code === "SC9001"); - lowerer.diags.push(...ice); - lowerer.runtimeFences.push(...flushed.filter((d) => d.code !== "SC9001")); } + for (const slot of pendingSlots) lowerer.noteDynamicSlot(slot); } catch (e) { // A Go panic inside tsgo, surfaced by the sync channel as a // thrown Error (server intact — the prefetch fence's finding): @@ -813,7 +796,17 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir throw e; } if (!lowerer.suppressStats) lowerer.countStatementFailed(stmt, sf?.fileName); - lowerer.noteBlockedBindings(stmt); + // Localized failure (report-only): the statement's committed + // diagnostics that are not themselves type failures or rejected + // declarations are cascades of the rejected declarations it used. + const committed = + lowerer.diagSink === null && !deferFences ? lowerer.diags.slice(diagsBefore) : []; + if (touched.size > 0) { + const cause = [...touched]; + for (const d of committed) + if (d.typeFailure === undefined && d.rejects === undefined) d.cause ??= cause; + } + lowerer.noteBlockedBindings(stmt, committed); // The runtime-fence conversion (JS sources, direct diagnostics // only — a capture wrapper in force means collection owns the // report). The statement's diagnostics move OFF the build and @@ -837,6 +830,8 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir const fence = lowerer.capturedRuntimeFence(sinkMark, stmt); if (fence) out.push(fence); } + } finally { + lowerer.failureTouches.pop(); } } // Each acquisition protects the remainder of its lexical statement @@ -862,6 +857,81 @@ export function lowerStmts(lowerer: Lowerer, stmts: readonly ts.Statement[]): Ir return out; } +/** Localized failure: retries a degradable statement-level type failure + * at most this many times (each retry degrades one more type). */ +const MAX_DEGRADE_RETRIES = 16; + +/** The slot kinds lowerStmts' commit point degrades: locals, loop + * variables and module globals (stage L1), and the parameters and returns + * of the function expressions, arrows and methods a statement holds + * (stage L2). */ +const STATEMENT_SLOTS: ReadonlySet = new Set([ + "local", + "global", + "loop", + "param", + "return", + "field", +]); + +/** One attempt at lowering a statement of lowerStmts into `out`. */ +function lowerOneStatement( + lowerer: Lowerer, + stmt: ts.Statement, + out: IrStmt[], + usingMarkers: UsingMarker[], + diagsBefore: number, + deferFences: boolean, +): void { + let lowered: IrStmt | IrStmt[] | null; + let appended = false; + if (ts.isVariableStatement(stmt) && (stmt.declarationList.flags & ts.NodeFlags.Using) !== 0) { + const registrations = lowerUsingStatement(lowerer, stmt); + const accounting: IrStmt[] = []; + for (const registration of registrations) { + out.push(...registration.acquire); + accounting.push(...registration.acquire, ...registration.cleanup); + if (registration.cleanup.length > 0) { + usingMarkers.push({ + start: out.length, + cleanup: registration.cleanup, + loc: registration.loc, + }); + } + } + lowered = accounting; + appended = true; + } else { + lowered = lowerer.lowerStmt(stmt); + } + // The lib-boundary chokepoint (lib-boundary.ts): checked-dynamic + // arguments that reached builtin-call slots without the checked + // coercion get their dynCheck here, and the uncoercible ones + // fence — inside this statement's poison window, so a JS fence + // becomes the statement's runtimeFence exactly like every other + // deferred rejection. + if (lowered) { + const body: IrStmt[] = []; + if (Array.isArray(lowered)) body.push(...lowered); + else body.push(lowered); + const rewritten = lowerSurplusCalls(lowerer, body); + enforceLibBoundary(lowerer, rewritten); + if (!appended) out.push(...rewritten); + } + // A JS statement that lowered CLEAN can still have flushed + // DEFERRED collection diagnostics (a symbol lookup on an alias + // resolves through flushDeferred — informational here, not a + // blocker of THIS statement): those belong to the runtime-fence + // ledger, not the build — real uses of the broken declaration + // meet their own per-site fences. + if (deferFences && lowerer.diagSink === null && lowerer.diags.length > diagsBefore) { + const flushed = lowerer.diags.splice(diagsBefore); + const ice = flushed.filter((d) => d.code === "SC9001"); + lowerer.diags.push(...ice); + lowerer.runtimeFences.push(...flushed.filter((d) => d.code !== "SC9001")); + } +} + /** Whether a statement list belongs to a function body (at any block * depth) rather than to a module's top level. */ function enclosedByFunction(stmt: ts.Statement): boolean { @@ -1302,6 +1372,7 @@ function jsDeclarationExpandsObject( inferred !== null && jsOpenObjectType(decl, inferred, lowerer.shapes, lowerer.unions).kind === "dyn") || arms.some((arm) => arm.kind === "record" && jsObjectBindingExpands(lowerer, decl, arm)) || + arms.some((arm) => arm.kind === "record" && jsObjectLiteralFilledLater(lowerer, decl, arm)) || (isJsSourceFile(ts.getSourceFileOfNode(decl)) && !decl.type && !hasJsTypeAnnotation(decl) && @@ -1522,7 +1593,11 @@ function noteEarlyReadVar(lowerer: Lowerer, symbol: ts.Symbol, local: IrLocal): * lowerVarDecl's salvage path declares the local when it can, and * resolution checks locals/globals before the fallthroughs consult * blockedBindings. */ -export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void { +export function noteBlockedBindings( + lowerer: Lowerer, + stmt: ts.Statement, + diags: readonly ScrDiagnostic[] = [], +): void { let list: ts.VariableDeclarationList | null = null; if (ts.isVariableStatement(stmt)) list = stmt.declarationList; else if ( @@ -1533,12 +1608,20 @@ export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void list = stmt.initializer; } if (!list) return; + // Localized failure (report-only): the bindings form one failure group, + // the statement, whose diagnostics are the group's. + const group = statementGroupKey(stmt); + let bound = false; for (const decl of list.declarations.nodes) { for (const nameNode of boundIdentifiersOf(decl.name)) { const symbol = lowerer.checker.getSymbolAtLocation(nameNode); - if (symbol) lowerer.blockedBindings.add(symbol); + if (!symbol) continue; + lowerer.blockedBindings.add(symbol); + if (!lowerer.rejectedGroups.has(symbol)) lowerer.rejectedGroups.set(symbol, group); + bound = true; } } + if (bound) for (const d of diags) d.rejects ??= group; } /** True when a reference/assignment fell through resolution because the @@ -1548,12 +1631,13 @@ export function noteBlockedBindings(lowerer: Lowerer, stmt: ts.Statement): void * flushed by the emit pass). */ export function isBlockedBinding(lowerer: Lowerer, symbol: ts.Symbol | null): boolean { if (!symbol) return false; - return ( + const blocked = lowerer.blockedBindings.has(symbol) || lowerer.deferredDiags.has(symbol) || lowerer.flushedSymbols.has(symbol) || - lowerer.alreadyFlushed.has(symbol) - ); + lowerer.alreadyFlushed.has(symbol); + if (blocked) lowerer.touchRejected(symbol); + return blocked; } /** Lowers a statement in a fresh lexical scope (if/while/for bodies). */ @@ -1878,7 +1962,16 @@ export function lowerStmt(lowerer: Lowerer, stmt: ts.Statement): IrStmt | IrStmt } // Exception-representable values move into the runtime's exception // cell. `throw f()` of a void call has no value. - let value = lowerer.lowerExpr(stmt.expression); + // A thrown object or array literal has no declared type: it is built as + // a dynamic object, so the catch binding's checked casts adopt it + // (lowerer.untypedLiteralContext). + let literal: ts.Expression = stmt.expression; + while (ts.isParenthesizedExpression(literal)) literal = literal.expression; + let value = + (ts.isObjectLiteralExpression(literal) || ts.isArrayLiteralExpression(literal)) && + lowerer.untypedLiteralContext(literal) + ? lowerer.lowerExprExpecting(stmt.expression, DYN) + : lowerer.lowerExpr(stmt.expression); if (value.type.kind === "void") lowerer.badType(stmt.expression, lowerer.typeOf(stmt.expression)); // Preserve primitive reasons and readable record identity across the @@ -3222,8 +3315,8 @@ export function lowerBindingPattern( // chain declares keep named fences (a detached method would lose its // receiver without an error; JS binds the prototype function). if (srcType.kind === "object") { - const info = lowerer.classes.get(srcType.className); - if (!info) lowerer.flushDeferredClass(srcType.className); + const info = + lowerer.classes.get(srcType.className) ?? lowerer.flushDeferredClass(srcType.className); if (info) { for (const el of pattern.elements.nodes) { if (el.name === undefined) continue; @@ -6515,6 +6608,41 @@ function lowerVariableInitializer( } } +/** A JS object literal that leaves out a required field of the binding's + * inferred record: tsc types `var p = {}; p.id = id;` by the members later + * assignments add, so the object only gains them after its initializer, and + * until then it is not a value of that record. Keep it a checked-dynamic + * object, as an object that gains keys outside the record is. */ +function jsObjectLiteralFilledLater( + lowerer: Lowerer, + decl: ts.VariableDeclaration, + type: IrType & { kind: "record" }, +): boolean { + if (decl.type || !decl.initializer || !isJsSourceFile(ts.getSourceFileOfNode(decl))) return false; + let initializer = decl.initializer; + while (ts.isParenthesizedExpression(initializer)) initializer = initializer.expression; + if (!ts.isObjectLiteralExpression(initializer) || hasJsTypeAnnotation(decl)) return false; + const shape = lowerer.shapes.get(type.shapeId); + if (!shape || shape.tuple) return false; + const given = new Set(); + for (const property of initializer.properties.nodes) { + if (ts.isSpreadAssignment(property)) return false; + const name = property.name; + if (name === undefined) return false; + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) + given.add(name.text); + else return false; + } + // A field whose type holds undefined reads as it would before the write. + const holdsUndefined = (t: IrType): boolean => + t.kind === "dyn" || + t.kind === "undefinedT" || + (t.kind === "union" && lowerer.armTag(t.unionId, UNDEFINED_T) >= 0); + return shape.fields.some( + (field) => !given.has(field.name) && !field.name.startsWith("%") && !holdsUndefined(field.type), + ); +} + /** A JS object literal's inferred fields do not close its later storage. * Keep the original native object when writes add keys or delete fields; * converting it to a fixed record would discard that behavior. */ diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index a08b20f88c..f74fa12513 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1,5 +1,6 @@ import { markInferredFactChecks, shapeOverlayFileNames } from "../shape-inference/index.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; +import { unionNarrowFailureMessage } from "./union-narrow.js"; import { compactDeclOrigins, inPackageFile, @@ -12,7 +13,7 @@ import { relieveHeapPressure } from "../../heap-pressure.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; import { keyedHelperName } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; -import { onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; +import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; import { needsGeneralNodeStreams, noteUntypedStreamFlow, @@ -214,7 +215,14 @@ import { isBundledShimForwarder } from "../package-shims.js"; import { erasedTypeOnlyImport } from "../type-only-imports.js"; import { literalValues } from "../literal-values.js"; import { discriminantOwners, literalUnionArm, type UnionLiteral } from "../union-discriminants.js"; -import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; +import type { + ScrDiagnostic, + TypeFailure, + TypeFailureFamily, + TypeFailureSlot, +} from "../../diagnostics/diagnostic.js"; +import { declarationGroupKey, typeFailureOf } from "./type-failure.js"; +import { dynamicSlotDiag, type DynamicSlotEntry } from "../../diagnostics/dynamic-slots.js"; import { unsupportedAnyOperationDiag, blockedBindingUseDiag, @@ -470,6 +478,7 @@ import { genericLocalClassBinding, } from "./generic-functions.js"; import { bindingNeverReassigned } from "./binding-analysis.js"; +import { isMessageListenerCall } from "./expressions/message-listeners.js"; import type { InstantiationPath } from "./instantiation-path.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import { completeArgs, wrappedUndefined, undefinedArgFor } from "./call-arguments.js"; @@ -723,7 +732,42 @@ interface GenericDemandOwner { classDemands: ClassInfo[]; } -export class PoisonError extends Error {} +/** A slot the localized-failure commit points turned into a checked + * dynamic slot (SC6104): where it is, what its declared type is, and the + * diagnostic it would have failed with. */ +export interface DynamicSlot { + loc: SrcLoc; + failure: TypeFailure; + code: string; + reason: string; +} + +/** Localized failure: rounds of the declaration flush retry. */ +const MAX_DECLARATION_RETRIES = 16; + +/** The slot kinds the declaration flush retry degrades (stage L2). */ +const DECLARATION_SLOTS: ReadonlySet = new Set([ + "field", + "param", + "return", + "local", + "global", + "loop", +]); + +export class PoisonError extends Error { + /** Set when a type with no static lowering poisoned a slot (badType and + * the dyn class field policy): what failed, for the localized-failure + * commit points. Report-only in stage L0. */ + readonly failure: TypeFailure | undefined; + /** The checker type that failed (with `failure`). */ + readonly failedType: ts.Type | undefined; + constructor(failure?: TypeFailure, failedType?: ts.Type) { + super(); + this.failure = failure; + this.failedType = failedType; + } +} /** A function-shaped value that exists only in the compiler's static * projection graph. Declared-function aliases point at the original @@ -838,6 +882,9 @@ export interface LowerResult { /** Performance warnings (SC61xx): the likely-hot functions the source * leaves untyped (lowering/dynamic-tier.ts). Never failures. */ performance?: ScrDiagnostic[]; + /** Localized failure: the checked dynamic slots the commit points + * created (each also an SC6104 in `performance`), present when any. */ + dynamicSlots?: DynamicSlotEntry[]; /** Functions lowering placed in the dynamic tier (ir/tiers.ts), present * when there are any. */ dynamicFunctions?: DynamicFunction[]; @@ -949,6 +996,13 @@ export interface LowererMode { /** Symbols whose deferred diagnostics reachable emit already flushed — * the remainder must not report them a second time. */ alreadyFlushed?: ReadonlySet; + /** Localized failure: the types reachable emit degraded (the coverage + * remainder maps them alike, so the declarations reachable emit + * registered with dynamic slots collect there too). */ + degradedTypes?: ReadonlyMap; + /** Localized failure: the declarations reachable emit registered with + * dynamic slots (the coverage remainder registers them too). */ + retriedSymbols?: ReadonlySet; /** LowerOptions.externalTypes, threaded through every lowering pass. */ externalTypes?: ReadonlyMap; /** LowerOptions.externalTypeSpecifiersByFile, shared by every pass. */ @@ -1253,6 +1307,22 @@ export function lowerToIr( timing("dynamic-tier-scan", { found: hotDynamic === null ? 0 : 1 }); if (hotDynamic !== null) result = { ...result, performance: [hotDynamic] }; } + // Localized failure (SC6104): one performance warning per dynamic slot. + if (resultLowerer.dynamicSlots.length > 0) { + const dynamicSlots: DynamicSlotEntry[] = resultLowerer.dynamicSlots.map((slot) => ({ + loc: slot.loc, + name: slot.failure.name, + slot: slot.failure.slot, + declaredType: slot.failure.declaredType, + code: slot.code, + reason: slot.reason, + })); + result = { + ...result, + performance: [...(result.performance ?? []), ...dynamicSlots.map(dynamicSlotDiag)], + dynamicSlots, + }; + } // Contract types (@scriptc/types): written annotations, checked against // the IR by the compile pipeline (contracts/check.ts). // Acyclic object types name the IR unit the program lowered them to; a @@ -1340,6 +1410,8 @@ export function lowerToIr( reachable, remainder: true, alreadyFlushed: resultLowerer.flushedSymbols, + degradedTypes: resultLowerer.degradedTypes, + retriedSymbols: resultLowerer.retriedSymbols, inlineRequireCycleCalls, targetPlatform, ffiImports, @@ -2491,6 +2563,69 @@ export class Lowerer { return fallback ?? null; } + /** An object or array literal in an untyped context: its contextual type + * is `unknown` or `any` from a slot the program declares (a variable, + * field, parameter, return type or `unknown[]` element; a library + * parameter typed `any` only through a type argument), it is thrown, or + * it is a member of such a literal, and it holds data only (no methods, + * accessors or function members). Its value is built as a dynamic + * object: no type the program wrote gives it a native layout, and a later + * checked cast adopts it (dyn.adoptShared), so every holder keeps one + * identity. A library parameter the declarations type `any` or `unknown` + * itself (`console.log`, `JSON.stringify`) only reads the value and keeps + * the native literal. */ + untypedLiteralContext(x: ts.Expression): boolean { + if (isJsSourceFile(ts.getSourceFileOfNode(x))) return false; + // Data only: a literal with methods, accessors or function members is an + // object with behavior, which keeps its native layout (and boxed form). + if ( + ts.isObjectLiteralExpression(x) && + x.properties.nodes.some((p) => { + if (!ts.isPropertyAssignment(p)) + return !ts.isShorthandPropertyAssignment(p) && !ts.isSpreadAssignment(p); + let init: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(init)) init = init.expression; + return ( + ts.isFunctionExpression(init) || ts.isArrowFunction(init) || ts.isClassExpression(init) + ); + }) + ) + return false; + let node: ts.Node = x; + while (node.parent !== undefined && ts.isParenthesizedExpression(node.parent)) + node = node.parent; + const parent = node.parent; + if (parent === undefined) return false; + if (ts.isThrowStatement(parent)) return true; + if (ts.isPropertyAssignment(parent) && parent.initializer === node) { + const literal = parent.parent; + if (ts.isObjectLiteralExpression(literal)) return this.untypedLiteralContext(literal); + } + if (ts.isArrayLiteralExpression(parent) && this.untypedLiteralContext(parent)) return true; + const contextual = this.checker.getContextualType(x); + if (!contextual || !(contextual.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) + return false; + if (ts.isCallExpression(parent) || ts.isNewExpression(parent)) { + const signature = this.checker.getResolvedSignature(parent); + const decl = signature?.declaration; + if (decl && this.isStdlibFile(ts.getSourceFileOfNode(decl))) { + const index = (parent.arguments?.nodes ?? []).indexOf(node as ts.Expression); + const params = (decl as { parameters?: { nodes: readonly ts.ParameterDeclaration[] } }) + .parameters?.nodes; + const param = params?.[Math.min(index, params.length - 1)]; + let annotation = param?.type; + if (annotation && ts.isArrayTypeNode(annotation)) annotation = annotation.elementType; + if ( + !annotation || + annotation.kind === ts.SyntaxKind.AnyKeyword || + annotation.kind === ts.SyntaxKind.UnknownKeyword + ) + return false; + } + } + return true; + } + /** A fresh TypeScript data literal (`{ message: "m" }`) passed to a * checked-dynamic parameter of a callee implemented in JavaScript, whose * contract lets it reshape the object: a call through an untyped value, an @@ -3713,6 +3848,53 @@ export class Lowerer { * the fallthroughs report the inherited-blocker cascade (SC2004) * instead of misattributing the reference. */ readonly blockedBindings = new Set(); + /** Localized failure, report-only: the failure group (type-failure.ts) + * of each declaration whose rejection reached the build — a flushed + * deferred declaration, or a binding of a poisoned declaration + * statement. */ + readonly rejectedGroups = new Map(); + /** One frame per statement lowerStmts is lowering: the failure groups + * its lowering touched. A statement that then poisons attributes its + * diagnostics to them (ScrDiagnostic.cause). */ + readonly failureTouches: Set[] = []; + /** Localized failure: checker types (by id) whose strict mapping failed + * at a committed slot, with their lenient mapping. Filled only at the + * commit points (degradeAtCommit), so a program that compiles today + * never fills it. */ + readonly degradedTypes = new Map(); + /** The dynamic slots the commit points created: one SC6104 each. */ + readonly dynamicSlots: DynamicSlot[] = []; + /** Declarations retryDeferred registered. */ + readonly retriedSymbols = new Set(); + /** LowererMode.retriedSymbols. */ + readonly priorRetriedSymbols: ReadonlySet; + /** Bodies run() skipped because their declaration had not registered. */ + readonly skippedBodies = new Set(); + /** Declarations retryDeferred registered, in order: run() lowers the + * bodies it had skipped. */ + readonly retriedDeclarations: ts.Node[] = []; + /** Diagnostics of lowering that a statement's retry does not repeat (a + * function-local class's member bodies, lowered once and kept): + * dropAttemptDiags keeps them. */ + readonly loweredOnceDiags = new Set(); + /** Deferred classes whose static field initializers' place in %init + * has been lowered (retryDeferredAtDeclaration): a later flush must not + * register them, since their initializers would never run. */ + readonly staticInitsPassed = new Set(); + /** Deferred declarations' collections, for the flush retry + * (retryDeferred). */ + readonly deferredCollectors = new Map< + ts.Symbol, + { symbolOf: () => ts.Symbol | undefined; collect: () => void } + >(); + /** True while a flush retry collects a declaration: the dyn class + * field policy (dynFieldPolicyFence) is lifted for reached classes. */ + liftDynFieldPolicy = false; + /** Collection diagnostics that a flush retry may simply run again (a + * derived class whose base had not registered yet). */ + readonly retriableDiags = new Set(); + /** The checker type behind each type failure badType committed. */ + readonly failedTypes = new Map(); /** True while collectProgram runs: resolution helpers must not flush * deferred diagnostics (collection itself resolves symbols — extends * clauses — and collection order must not decide what reports). */ @@ -3766,12 +3948,12 @@ export class Lowerer { /** Bumps the per-file attribution counter (no-op unless provenance is * active and this is the emit/discovery lane — mirror the CALLER's * suppressStats guard, this method only gates remainder). */ - bumpFileStat(file: string, kind: "total" | "failed"): void { + bumpFileStat(file: string, kind: "total" | "failed", by = 1): void { if (this.remainder || !provenanceActive()) return; let s = this.statsByFile.get(file); if (!s) this.statsByFile.set(file, (s = { total: 0, failed: 0 })); - if (kind === "total") s.total += 1; - else if (kind === "failed") s.failed += 1; + if (kind === "total") s.total += by; + else if (kind === "failed") s.failed += by; } /** The program holds the declaration-typed boundary's projection @@ -3865,6 +4047,9 @@ export class Lowerer { ) { // The mode's options, sorted by name; the values derived from them follow. this.alreadyFlushed = mode.alreadyFlushed ?? new Set(); + if (mode.degradedTypes) + for (const [id, degraded] of mode.degradedTypes) this.degradedTypes.set(id, degraded); + this.priorRetriedSymbols = mode.retriedSymbols ?? new Set(); this.externalTypes = mode.externalTypes ?? new Map(); this.ffiBindingSymbols = mode.ffiBindingSymbols ?? null; this.ffiImports = mode.ffiImports ?? []; @@ -3900,6 +4085,10 @@ export class Lowerer { resolveTypeParam: this.typeParamResolver, resolveTypeParamTs: this.typeParamTsResolver, resolveBoundAliasTs: this.boundAliasTsResolver, + degradedType: (type) => { + const id = (type as { id?: number }).id; + return id === undefined ? null : (this.degradedTypes.get(id)?.lenient ?? null); + }, isErasing: () => this.erasingDepth > 0, genericClassInstance: (decl, ref, mapArgument) => this.genericClassInstanceType(decl, ref, mapArgument), @@ -4932,7 +5121,14 @@ export class Lowerer { } collectProgram(parts: FileParts[]): void { - return collectProgram(this, parts); + collectProgram(this, parts); + // The coverage remainder registers the declarations reachable emit + // registered with dynamic slots (retryDeferred), so it neither reports + // them as unreached nor blocks their uses. + for (const symbol of this.priorRetriedSymbols) { + const diags = this.deferredDiags.get(symbol); + if (diags !== undefined) this.retryDeferred(symbol, diags); + } } /** Names every file's %init and registers the run-once guard globals @@ -5925,6 +6121,22 @@ export class Lowerer { }; if (!stringRead(e.left) && !stringRead(e.right)) return null; if (!primitive(e.left) || !primitive(e.right)) return null; + // A missing read is undefined, and `+` yields a number only when + // neither operand is a string: an operand that is always a string + // (`"x-" + PAD[i]`) makes the result a string. Only literals, + // templates and sums with such an operand are always strings; a + // binding or call can hold a missing read. + const definitelyString = (part: ts.Expression): boolean => { + const p = peel(part); + if (ts.isStringLiteral(p) || ts.isNoSubstitutionTemplateLiteral(p)) return true; + if (ts.isTemplateExpression(p)) return true; + return ( + ts.isBinaryExpression(p) && + p.operatorToken.kind === ts.SyntaxKind.PlusToken && + (definitelyString(p.left) || definitelyString(p.right)) + ); + }; + if (definitelyString(e.left) || definitelyString(e.right)) return null; return { kind: "union", unionId: this.unions.intern([F64, STRING]) }; }; /** The union of primitive result types, or null when any is missing or @@ -8190,9 +8402,12 @@ export class Lowerer { if (sig && !this.wantBody(sig.name)) continue; const fn = this.lowerFunction(decl); if (fn) functions.push(fn); - else if (this.countsSkips()) { - this.countSkippedBody(decl); - if (this.remainder) this.reportBlockedBody(decl); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) { + this.countSkippedBody(decl); + if (this.remainder) this.reportBlockedBody(decl); + } } } for (const decl of fp.classDecls) { @@ -8201,7 +8416,10 @@ export class Lowerer { functions.push( ...this.withClosureRoot(info.def.name, () => this.lowerClassMembers(info)), ); - else if (this.countsSkips()) this.countSkippedBody(decl); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) this.countSkippedBody(decl); + } } } @@ -8236,15 +8454,38 @@ export class Lowerer { let gc = 0; let gi = 0; let es = 0; + let rd = 0; const classDispatch = new ClassDynamicDispatch(); let dispatchChanged = !this.remainder; while ( dispatchChanged || + rd < this.retriedDeclarations.length || ec < this.exprClasses.length || gc < this.genericClassInstances.length || gi < this.instantiationQueue.length || es < this.emitSpecQueue.length ) { + // Localized failure: declarations a flush registered with dynamic + // slots (retryDeferred) after the loops above skipped their bodies. + while (rd < this.retriedDeclarations.length) { + const decl = this.retriedDeclarations[rd++]!; + if (!this.skippedBodies.delete(decl)) continue; + if (this.countsSkips()) this.uncountSkippedBody(decl); + if (ts.isFunctionDeclaration(decl)) { + const fn = this.lowerFunction(decl); + if (fn) functions.push(fn); + else { + this.skippedBodies.add(decl); + if (this.countsSkips()) this.countSkippedBody(decl); + } + } else if (ts.isClassDeclaration(decl)) { + const info = this.classes.get(this.classNamer(decl)); + if (info) + functions.push( + ...this.withClosureRoot(info.def.name, () => this.lowerClassMembers(info)), + ); + } + } while (ec < this.exprClasses.length) { functions.push(...this.lowerClassMembers(this.exprClasses[ec++]!)); } @@ -8257,13 +8498,10 @@ export class Lowerer { // generic method's this/super fence, a fenced parameter // default): the diagnostic is recorded — the instance skips // like a signature-blocked function (lowerFunction's rule). - try { - functions.push( - this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), - ); - } catch (e) { - if (!(e instanceof PoisonError)) throw e; - } + const fn = this.lowerUnitDegrading(() => + this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), + ); + if (fn !== null) functions.push(fn); } // Emit-override specializations queued by the emit sites above (a // body can queue more — the super-forward chain — and generic @@ -8534,6 +8772,22 @@ export class Lowerer { * percentage would overstate how much of the program compiles. Count * every statement the body holds as attempted and failed, in addition to * the skipped-function tally. */ + /** countSkippedBody undone: a flush registered the declaration after + * its body was skipped, and the body lowers now. */ + private uncountSkippedBody(decl: ts.Node): void { + this.stats.functionsSkipped--; + if (this.suppressStats) return; + const n = countBodyStatements(decl); + if (n === 0) return; + this.stats.statementsTotal -= n; + this.stats.statementsFailed -= n; + const file = ts.getSourceFileOfNode(decl).fileName; + for (let i = 0; i < n; i++) { + this.bumpFileStat(file, "total", -1); + this.bumpFileStat(file, "failed", -1); + } + } + private countSkippedBody(decl: ts.Node): void { this.stats.functionsSkipped++; if (this.suppressStats) return; @@ -8857,6 +9111,44 @@ export class Lowerer { register(`%${cName}.set:${prop}`, [], () => this.throwingSetterFn(info, prop)); } }; + // Localized failure: a declaration a flush registered with dynamic + // slots (retryDeferred) registers its units like a class expression. + this.onDeclarationRegistered = (decl: ts.Node): void => { + if (ts.isFunctionDeclaration(decl)) { + if (!decl.body) return; + const symbol = declSymbolOf(this, decl); + const sig = symbol ? this.fnSigsBySymbol.get(symbol) : undefined; + if (!sig || units.has(sig.name)) return; + units.set(sig.name, { + order: unitOrder++, + roots: functionRoots(decl), + lower: () => this.lowerFunction(decl), + }); + if (reachable.has(sig.name)) queue.push(sig.name); + metadataPriority.set(sig.name, [3, expressionMetadataOrder++]); + } else if (ts.isClassDeclaration(decl)) { + const info = this.classes.get(this.classNamer(decl)); + if (!info) return; + if (!info.generic) { + if (!units.has(`%${info.def.name}.constructor`)) this.onExprClassCollected?.(info); + return; + } + // A generic class FAMILY's only units are its statics; its + // instantiations lower in the instance drain. + if (!info.staticMethods) return; + for (const [name, entry] of info.staticMethods) { + const unit = `%${info.def.name}.static:${name}`; + if (units.has(unit)) continue; + units.set(unit, { + order: unitOrder++, + roots: functionRoots(entry.member), + lower: () => lowerStaticMethod(this, info, name), + }); + if (reachable.has(unit)) queue.push(unit); + metadataPriority.set(unit, [3, expressionMetadataOrder++]); + } + } + }; const onEdge = this.onEdge; for (const name of collectionEdges) onEdge(name); // Generic instances queued by the bodies above lower here (an instance @@ -8912,17 +9204,16 @@ export class Lowerer { this.genericDemandPriority.set(inst, ref); // Body-level poisons skip the instance after retaining the // diagnostic and every edge fired before poisoning. - try { - instanceFunctions.push( - this.withGenericDemandOwner(owner, () => - this.shapes.withDeclaredOrderPriority(ref, () => - this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), - ), + const fn = this.lowerUnitDegrading(() => + this.withGenericDemandOwner(owner, () => + this.shapes.withDeclaredOrderPriority(ref, () => + this.withClosureRoot(inst.name, () => this.lowerGenericInstance(info, inst)), ), - ); + ), + ); + if (fn !== null) { + instanceFunctions.push(fn); relieveHeapPressure(); - } catch (e) { - if (!(e instanceof PoisonError)) throw e; } } } @@ -9294,7 +9585,14 @@ export class Lowerer { this.diagSink.push(d); return; } - if (pending) this.commitFlush(pending.symbol, pending.source); + // A probe's copy of a declaration's flushed diagnostics, committed + // after a flush retry registered the declaration: nothing to report. + if (pending && this.retriedSymbols.has(pending.symbol)) return; + if (pending) { + this.commitFlush(pending.symbol, pending.source); + const group = this.rejectedGroups.get(pending.symbol); + if (group !== undefined) d.rejects ??= group; + } // One site, one report: some declarations map a type twice (module // globals pre-register before their initializers lower) — an exact // duplicate (code + span + message) adds noise, not information. @@ -9349,6 +9647,166 @@ export class Lowerer { return (this.typeOf(node).flags & ts.TypeFlags.Any) !== 0; } + /** badType's commit: the diagnostic carries the structured type + * failure (report-only in stage L0), and so does the poison. */ + private typeFail( + node: ts.Node, + type: ts.Type, + family: TypeFailureFamily, + diag: ScrDiagnostic, + ): never { + const failure = typeFailureOf(node, this.checker.typeToString(type), family); + diag.typeFailure = failure; + this.failedTypes.set(failure, type); + this.pushDiag(diag); + throw new PoisonError(failure, type); + } + + /** Localized failure's commit point (design §3.3): `e` poisoned a unit + * whose failure is about to enter the build. When it is a degradable type + * failure at a slot kind in `slots`, the failing type joins + * degradedTypes (its lenient mapping is the checked dynamic, whole slot), + * the failure's own diagnostic leaves the build, and the caller lowers + * the unit again. Returns the slot to record once that retry succeeds, + * or null to commit as today. Never inside a capture buffer (a probe or a + * deferred collection) and never under --tier=static-only. */ + degradeAtCommit(e: unknown, slots: ReadonlySet): DynamicSlot | null { + if (this.tierMode !== "auto" || this.diagSink !== null) return null; + if (!(e instanceof PoisonError)) return null; + const failure = e.failure; + const type = e.failedType; + if (failure === undefined || type === undefined) return null; + if (!failure.degradable || failure.family === "dyn-field-policy") return null; + if (!slots.has(failure.slot)) return null; + const id = (type as { id?: number }).id; + // Already degraded and still failing: no progress, commit as today. + if (id === undefined || this.degradedTypes.has(id)) return null; + let committed: ScrDiagnostic | undefined; + for (let i = this.diags.length - 1; i >= 0; i--) { + if (this.diags[i]!.typeFailure !== failure) continue; + committed = this.diags[i]; + this.diags.splice(i, 1); + } + if (committed === undefined) return null; + this.degradedTypes.set(id, { lenient: DYN, code: committed.code, reason: committed.message }); + return { loc: committed.loc, failure, code: committed.code, reason: committed.message }; + } + + /** A declaration slot whose type maps through degradedTypes is a + * dynamic slot too, though only the first such slot failed: record it + * (outside probes; the lookup is skipped while nothing is degraded). */ + noteDegradedSlot(node: ts.Node, type: ts.Type): void { + if (this.diagSink !== null) return; + const degraded = this.degradedTypes.get((type as { id?: number }).id ?? -1); + if (degraded === undefined) return; + const failure = typeFailureOf(node, this.checker.typeToString(type), "unsupported"); + if (failure.slot === "expression" || failure.slot === "member") return; + this.noteDynamicSlot({ + loc: locOf(node), + failure, + code: degraded.code, + reason: degraded.reason, + }); + } + + /** Localized failure, statement commit point for declarations a probe + * flushed: a statement committing the flushed diagnostics of a deferred + * declaration (inside a capture buffer the flush could not retry) reopens + * it here, outside every probe. When the declaration's diagnostics are + * all retriable, they leave the build, the declaration retries + * (retryDeferred), and on success the caller lowers the statement again. + * Returns whether some declaration registered. */ + reopenFlushedAtCommit(diagsBefore: number): boolean { + if (this.tierMode !== "auto" || this.diagSink !== null) return false; + const groups = new Set(); + for (let i = diagsBefore; i < this.diags.length; i++) { + const group = this.diags[i]!.rejects; + if (group !== undefined && !group.startsWith("statement @")) groups.add(group); + } + if (groups.size === 0) return false; + let registered = false; + for (const [symbol, group] of [...this.rejectedGroups]) { + if (!groups.has(group) || !this.deferredCollectors.has(symbol)) continue; + const committed = this.diags.filter((d) => d.rejects === group); + this.diags.splice(0, this.diags.length, ...this.diags.filter((d) => d.rejects !== group)); + this.flushedSymbols.delete(symbol); + this.rejectedGroups.delete(symbol); + if (this.retryDeferred(symbol, committed)) { + registered = true; + continue; + } + // Not retriable: the declaration's diagnostics commit as before. + const last = this.deferredDiags.get(symbol) ?? committed; + this.deferredDiags.delete(symbol); + this.flushedSymbols.add(symbol); + this.noteRejected(symbol, last); + for (const d of last) this.pushDiag(d); + } + return registered; + } + + /** A failed attempt's own diagnostics leave the build before its unit + * lowers again; flushed declaration diagnostics (`rejects`) stay, since + * the retry will not flush them a second time. */ + dropAttemptDiags(diagsBefore: number): void { + const kept = this.diags + .slice(diagsBefore) + .filter((d) => d.rejects !== undefined || this.loweredOnceDiags.has(d)); + this.diags.length = diagsBefore; + this.diags.push(...kept); + } + + /** Lowers one unit whose body-level poison skips it (a generic + * instance), with localized failure's commit point: a degradable type + * failure at one of its declaration slots degrades the type and the unit + * lowers again (at most MAX_DECLARATION_RETRIES times). Null when it + * still poisons; its diagnostics are then committed as before. */ + lowerUnitDegrading(lower: () => T): T | null { + const slots: DynamicSlot[] = []; + for (let attempt = 0; ; attempt++) { + const mark = this.diags.length; + try { + const result = lower(); + for (const slot of slots) this.noteDynamicSlot(slot); + return result; + } catch (e) { + if (!(e instanceof PoisonError)) throw e; + if (attempt >= MAX_DECLARATION_RETRIES) return null; + const slot = this.degradeAtCommit(e, DECLARATION_SLOTS); + if (slot === null) return null; + slots.push(slot); + this.dropAttemptDiags(mark); + } + } + } + + /** Records a dynamic slot whose unit lowered with it (one per site). */ + noteDynamicSlot(slot: DynamicSlot): void { + const seen = this.dynamicSlots.some( + (s) => + s.loc.file === slot.loc.file && + s.loc.start === slot.loc.start && + s.loc.end === slot.loc.end, + ); + if (!seen) this.dynamicSlots.push(slot); + } + + /** The SC1090 refusal of checked-dynamic class fields in TypeScript + * (classFieldStoresDyn): a policy fence, not a missing representation. + * A flush retry lifts it (liftDynFieldPolicy), so a reached class with + * an 'any' field registers. */ + dynFieldPolicyFence(node: ts.Node, feature: string): never { + const diag = unsupportedDiag("SC1090", locOf(node), feature); + const failure = typeFailureOf( + node, + this.checker.typeToString(this.typeOf(node)), + "dyn-field-policy", + ); + diag.typeFailure = failure; + this.pushDiag(diag); + throw new PoisonError(failure, this.typeOf(node)); + } + badType(node: ts.Node, type: ts.Type): never { const widened = this.checker.getBaseTypeOfLiteralType(type); // Types declared by the ADOPTED @types/node (Buffer, NodeJS.Timeout, @@ -9360,8 +9818,12 @@ export class Lowerer { // applies to node-typed values. const typeSym = widened.getAliasSymbol() ?? widened.getSymbol(); if (this.nodeTypesOnlySymbol(typeSym)) { - this.pushDiag(noLoweringDiag(this.checker.typeToString(type), locOf(node), undefined, true)); - throw new PoisonError(); + this.typeFail( + node, + type, + "node-types", + noLoweringDiag(this.checker.typeToString(type), locOf(node), undefined, true), + ); } // Engine-backed ambient TYPES in a STATIC build report the per-site // SC2012 rather than the generic supported-types recitation. Native @@ -9372,10 +9834,12 @@ export class Lowerer { (FETCH_AMBIENT_TYPES as readonly string[]).includes(ts.symbolName(typeSym)) && this.isStdlibSymbol(typeSym) ) { - this.pushDiag( + this.typeFail( + node, + type, + "fetch-ambient", unsupportedNativeApiDiag(`a value of type '${ts.symbolName(typeSym)}'`, locOf(node)), ); - throw new PoisonError(); } // ENUM OBJECTS (`typeof e` — a numeric enum's type carries the // reverse-map index signature, a string enum's just its members): @@ -9384,14 +9848,16 @@ export class Lowerer { // as the per-site identifier fence — never the index-signature // recitation. if (typeSym && (typeSym.flags & ts.SymbolFlags.Enum) !== 0) { - this.pushDiag( + this.typeFail( + node, + type, + "enum-object", unsupportedDiag( "SC1090", locOf(node), `enum objects as values ('${ts.symbolName(typeSym)}' — member reads like '${ts.symbolName(typeSym)}.X' compile to constants; the object itself has no runtime representation)`, ), ); - throw new PoisonError(); } // Would-be records with INDEX SIGNATURES (`Record`, // `{ [k: string]: T }`) get the index-signature fence (SC2006) naming @@ -9421,19 +9887,26 @@ export class Lowerer { this.isStdlibFile(ts.getSourceFileOfNode(d)), ) ) { - this.pushDiag(noLoweringDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "stdlib-index", + noLoweringDiag(this.checker.typeToString(type), locOf(node)), + ); } - this.pushDiag(indexSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "index-signature", + indexSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // Attribute an unsupported package type to its package. { const pkg = this.npmPackageOf(widened); if (pkg) { - this.pushDiag(unsupportedPackageValueDiag(pkg, locOf(node))); - throw new PoisonError(); + this.typeFail(node, type, "package", unsupportedPackageValueDiag(pkg, locOf(node))); } } // A type that keeps a GENERIC call signature (`(x: T) => T` slots, @@ -9449,8 +9922,12 @@ export class Lowerer { this.checker.getCallSignatures(p).some((s) => s.getTypeParameters().length > 0), ) ) { - this.pushDiag(genericSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "generic-signature", + genericSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } } // STANDARD-LIBRARY nominal provenance, decided once: interface/class @@ -9470,16 +9947,24 @@ export class Lowerer { // generic overload set keeps the monomorphization story: a value of a // multi-signature type has no single compiled signature to hold. if (!stdlibNominal && this.checker.getCallSignatures(widened).length > 1) { - this.pushDiag(overloadedSignatureTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "overloads", + overloadedSignatureTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // INTERSECTIONS that resolved to no lowering (SC2008). The ones that // compile never get here: member intersections intern through the // record path, callable hybrids map to '%call' records, and pinned // mixin instantiations resolve by chain structure. if (widened.isIntersectionType()) { - this.pushDiag(intersectionTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "intersection", + intersectionTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } // SUPPORTED shapes over a component outside its slot (SC2009): the // Map/Set domains, array/tuple elements, union arms, function @@ -9491,8 +9976,12 @@ export class Lowerer { { const detail = describeComponentBlocker(widened, this.typeCtx); if (detail !== null) { - this.pushDiag(componentTypeDiag(this.checker.typeToString(type), detail, locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "component", + componentTypeDiag(this.checker.typeToString(type), detail, locOf(node)), + ); } } // STANDARD-LIBRARY nominal types with no lowering at all: the SC2020 @@ -9522,26 +10011,36 @@ export class Lowerer { TextDecoder: "TextDecoder instances support recognized labels, fatal/ignoreBOM option literals, and streaming UTF-8/UTF-16/single-byte decode; runtime option objects remain unsupported", }; - this.pushDiag( + this.typeFail( + node, + type, + "stdlib-nominal", noLoweringDiag( this.checker.typeToString(type), locOf(node), typeHints[ts.symbolNameOf(stdlibOwnSym) ?? ""], ), ); - throw new PoisonError(); } // USER record shapes blocked by ONE member (SC2009's record arm): // name the member and its type. { const detail = describeRecordMemberBlocker(widened, this.typeCtx); if (detail !== null) { - this.pushDiag(componentTypeDiag(this.checker.typeToString(type), detail, locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "record-member", + componentTypeDiag(this.checker.typeToString(type), detail, locOf(node)), + ); } } - this.pushDiag(unsupportedTypeDiag(this.checker.typeToString(type), locOf(node))); - throw new PoisonError(); + this.typeFail( + node, + type, + "unsupported", + unsupportedTypeDiag(this.checker.typeToString(type), locOf(node)), + ); } /** The lib fence (SC2020): a reached use of standard-library surface @@ -9975,6 +10474,38 @@ export class Lowerer { ); }; + private structuredCloneReadCounts: { workerData: number; messageListeners: number } | null = null; + + /** How often the program reads `workerData` (every expression spelling it, + * counted conservatively) and registers a `"message"` listener: a + * structured clone read at one site only has no other holder + * (narrowing.ts freshDynSource). Computed once per lowering pass. */ + structuredCloneReads(): { workerData: number; messageListeners: number } { + if (this.structuredCloneReadCounts) return this.structuredCloneReadCounts; + const counts = { workerData: 0, messageListeners: 0 }; + const visit = (node: ts.Node): void => { + if (ts.isIdentifier(node) && node.text === "workerData") { + const parent = node.parent; + const declaresName = + parent !== undefined && + (ts.isImportSpecifier(parent) || + ts.isExportSpecifier(parent) || + ((ts.isPropertyAssignment(parent) || + ts.isPropertyDeclaration(parent) || + ts.isPropertySignature(parent) || + ts.isMethodDeclaration(parent)) && + parent.name === node)); + if (!declaresName) counts.workerData++; + } else if (ts.isCallExpression(node) && isMessageListenerCall(node)) { + counts.messageListeners++; + } + ts.forEachChild(node, visit); + }; + for (const sf of this.moduleOrder) ts.forEachChild(sf, visit); + this.structuredCloneReadCounts = counts; + return counts; + } + irTypeOf(node: ts.Node): IrType { const t = this.typeOf(node); // JS inference can leave mutable fields, accessor parameters and @@ -9989,6 +10520,7 @@ export class Lowerer { // f64 — but must not: pre-empt the mapping so the JS fallback below // answers instead. const mapped = neverTaintedJsType(this, node, t) ? null : this.mapTypeOf(t); + if (mapped !== null && this.degradedTypes.size > 0) this.noteDegradedSlot(node, t); if (!mapped) { const argumentsType = jsArgumentsFunctionType(this, t); if (argumentsType !== null) return argumentsType; @@ -11114,6 +11646,26 @@ export class Lowerer { private coerceIntoSlot(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { refuseInferredCallbackDrops(this, node, expr.type, expected); + // A union into one of its record arms extracts that arm, trusting the + // checker to have excluded the others. A built-in object arm the checker + // still types the site with (`c ?? new Map()` into an interface the + // untyped Map satisfies only through `any`) holds a value at run time, + // and no conversion makes a record of it, so the flow is refused like + // the arm's value would be. + if ( + expr.type.kind === "union" && + expected.kind === "record" && + this.armTag(expr.type.unionId, expected) >= 0 + ) { + const site = this.mapTypeOf(this.typeOf(node)); + const siteArms = + site?.kind === "union" ? (this.unions.get(site.unionId)?.arms ?? []) : site ? [site] : []; + for (const arm of this.unions.get(expr.type.unionId)?.arms ?? []) { + if (arm.kind !== "map" && arm.kind !== "set") continue; + if (!siteArms.some((s) => typeEquals(s, arm))) continue; + this.requireExactShape(node, arm, expected); + } + } // An assertion's scalar check whose value goes straight back into a // checked-dynamic slot (`process.loadEnvFile({} as unknown as string)`) // keeps the original value: JavaScript erases assertions, so the check @@ -11276,6 +11828,25 @@ export class Lowerer { } } + // A value the checker types `never` at this site (a read narrowed past + // every member of its union) exists only when an assertion lied about + // its type, so its layout need not fit the slot: it is evaluated, then a + // TypeError is thrown, as checked extraction of an impossible arm does. + if (!typeEquals(e.type, expected) && (this.typeOf(node).flags & ts.TypeFlags.Never) !== 0) { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: e, loc: e.loc }], + result: nodeThrowExpr( + 1, + "", + unionNarrowFailureMessage(e.type, (t) => this.fmt(t)), + expected, + e.loc, + ), + type: expected, + loc: e.loc, + }; + } this.requireExactShape(node, e.type, expected); return e; } @@ -11599,6 +12170,7 @@ export class Lowerer { !this.mapTypeOf(this.typeOf(x)) || this.excessJsContextualProperty(x, parts) || this.reshapeableContextualLiteral(x, parts) || + this.untypedLiteralContext(x) || holdsEvolvingArray(this, x) ) { return lowerDynObjectLiteral(this, x); @@ -12192,6 +12764,7 @@ export class Lowerer { const list = this.deferredDiags.get(symbol) ?? []; list.push(...sink); this.deferredDiags.set(symbol, list); + this.deferredCollectors.set(symbol, { symbolOf, collect }); return symbol; } finally { this.diagSink = previousSink; @@ -12208,20 +12781,48 @@ export class Lowerer { this.reportDeferred(symbol); } + /** A statement's lowering used a declaration: if that declaration was + * rejected, the statement's own failure (if any) is its cascade. */ + touchRejected(symbol: ts.Symbol): void { + const group = this.rejectedGroups.get(symbol); + if (group === undefined) return; + const frame = this.failureTouches[this.failureTouches.length - 1]; + if (frame !== undefined) frame.add(group); + } + + /** Records a declaration whose deferred diagnostics reached the build: + * its failure group, on the group's diagnostics too. */ + private noteRejected(symbol: ts.Symbol, diags: readonly ScrDiagnostic[]): void { + let group = this.rejectedGroups.get(symbol); + if (group === undefined) { + group = declarationGroupKey(symbol); + this.rejectedGroups.set(symbol, group); + } + for (const d of diags) d.rejects ??= group; + } + /** flushDeferred without the collection-pass guard: for declarations * whose diagnostics must report eagerly (class statements with * declaration-time code, extended bases — collectClassShapeDeferring). */ reportDeferred(symbol: ts.Symbol): void { const diags = this.deferredDiags.get(symbol); - if (!diags) return; + if (!diags) { + this.touchRejected(symbol); + return; + } if (this.alreadyFlushed.has(symbol)) { this.deferredDiags.delete(symbol); // the emit pass reported these return; } if (this.diagSink === null) { + if (this.retryDeferred(symbol, diags)) return; + // A failed retry leaves the last attempt's diagnostics deferred. + const committed = this.deferredDiags.get(symbol) ?? diags; this.deferredDiags.delete(symbol); this.flushedSymbols.add(symbol); - for (const d of diags) this.pushDiag(d); + this.noteRejected(symbol, committed); + for (const d of committed) this.pushDiag(d); + this.touchRejected(symbol); return; } // A capture buffer is active (a probe, or another declaration's @@ -12238,6 +12839,7 @@ export class Lowerer { * (the declaration counts as flushed for the coverage remainder). */ private commitFlush(symbol: ts.Symbol, source: ScrDiagnostic): void { this.flushedSymbols.add(symbol); + this.noteRejected(symbol, [source]); const list = this.deferredDiags.get(symbol); if (!list) return; const index = list.indexOf(source); @@ -12245,9 +12847,132 @@ export class Lowerer { if (list.length === 0) this.deferredDiags.delete(symbol); } - flushDeferredClass(className: string): void { + /** Flushes a deferred class known by its IR name, and answers the class + * when the flush registered it (retryDeferred). */ + flushDeferredClass(className: string): ClassInfo | undefined { const symbol = this.deferredClassByName.get(className); if (symbol) this.flushDeferred(symbol); + return this.classes.get(className); + } + + /** Localized failure's declaration commit point (design §3.3): a + * reference is flushing a deferred declaration into the build. When + * every deferred diagnostic is a degradable type failure at one of the + * declaration's slots, the failing types degrade and the collection + * runs again (up to MAX_DECLARATION_RETRIES rounds, each degrading the + * types the previous round still failed on). On success the + * declaration registers, each degraded slot is an SC6104, and nothing + * reports. Otherwise the last round's diagnostics stay deferred for the + * caller to commit. */ + private retryDeferred(symbol: ts.Symbol, diags: readonly ScrDiagnostic[]): boolean { + if (this.tierMode !== "auto") return false; + if (this.staticInitsPassed.has(symbol)) return false; + const collector = this.deferredCollectors.get(symbol); + if (collector === undefined) return false; + const slots: DynamicSlot[] = []; + // The types this retry degraded (not those of retries its collections + // ran), for retryDeferredAtDeclaration to restore when it fails. + const degraded: number[] = []; + try { + return this.retryDeferredRounds(symbol, collector, diags, slots, degraded); + } finally { + this.retryDegraded = degraded; + } + } + + private retryDeferredRounds( + symbol: ts.Symbol, + collector: { symbolOf: () => ts.Symbol | undefined; collect: () => void }, + diags: readonly ScrDiagnostic[], + slots: DynamicSlot[], + degraded: number[], + ): boolean { + let current = diags; + for (let round = 0; round < MAX_DECLARATION_RETRIES; round++) { + const failing: { diag: ScrDiagnostic; failure: TypeFailure; id: number }[] = []; + let retriable = false; + for (const diag of current) { + if (this.retriableDiags.has(diag)) { + retriable = true; + continue; + } + const failure = diag.typeFailure; + // The dyn class field policy lifts during the retry; nothing to + // degrade. + if (failure?.family === "dyn-field-policy") { + retriable = true; + continue; + } + const type = failure === undefined ? undefined : this.failedTypes.get(failure); + const id = (type as { id?: number } | undefined)?.id; + if ( + failure === undefined || + id === undefined || + !failure.degradable || + !DECLARATION_SLOTS.has(failure.slot) || + this.degradedTypes.has(id) + ) + return false; + failing.push({ diag, failure, id }); + } + // A round makes progress by degrading a type, or (once) by running + // a collection that only waited for another declaration. + if (failing.length === 0 && !(retriable && round === 0)) return false; + for (const { diag, failure, id } of failing) { + this.degradedTypes.set(id, { lenient: DYN, code: diag.code, reason: diag.message }); + degraded.push(id); + slots.push({ loc: diag.loc, failure, code: diag.code, reason: diag.message }); + } + this.deferredDiags.delete(symbol); + const lifted = this.liftDynFieldPolicy; + this.liftDynFieldPolicy = true; + let registered: boolean; + try { + registered = this.collectDeferring(collector.symbolOf, collector.collect) === null; + } finally { + this.liftDynFieldPolicy = lifted; + } + if (registered) { + this.retriedSymbols.add(symbol); + this.deferredCollectors.delete(symbol); + for (const [name, deferred] of this.deferredClassByName) + if (deferred === symbol) this.deferredClassByName.delete(name); + for (const slot of slots) this.noteDynamicSlot(slot); + for (const decl of symbol.declarations) + if (ts.isFunctionDeclaration(decl) || ts.isClassDeclaration(decl)) { + this.retriedDeclarations.push(decl); + this.onDeclarationRegistered?.(decl); + } + return true; + } + current = this.deferredDiags.get(symbol) ?? []; + } + return false; + } + + private retryDegraded: number[] = []; + + /** A deferred class with static field initializers is reached where its + * statement evaluates: Node runs the initializers there, referenced or + * not. Its file's %init calls this at that place, which retries the class + * as a flush would, so the initializers lower in place. A failed retry + * reports nothing and leaves the class deferred as it was, and from here + * on no flush registers it (staticInitsPassed): it reports instead. + * Returns whether the class has registered. */ + retryDeferredAtDeclaration(symbol: ts.Symbol): boolean { + const diags = this.deferredDiags.get(symbol); + let registered = !this.deferredCollectors.has(symbol); + if (!registered && diags !== undefined) { + this.retryDegraded = []; + registered = this.retryDeferred(symbol, diags); + if (!registered) { + for (const id of this.retryDegraded) this.degradedTypes.delete(id); + this.deferredDiags.set(symbol, [...diags]); + } + this.retryDegraded = []; + } + this.staticInitsPassed.add(symbol); + return registered; } collectSignature(decl: ts.FunctionDeclaration): void { @@ -12526,6 +13251,9 @@ export class Lowerer { * bodies as worklist units (the units map is otherwise built before * lowering starts). Null in the emit pass. */ onExprClassCollected: ((info: ClassInfo) => void) | null = null; + /** emitReachable's unit registration for a declaration retryDeferred + * registered after collection. */ + onDeclarationRegistered: ((decl: ts.Node) => void) | null = null; /** Mixin functions (`(Base: T) => class extends Base {…}`) by their * function-like node: recognized shape, or null for checked @@ -12704,6 +13432,8 @@ export class Lowerer { /* ── scoping and captures ─────────────────────────────────────────── */ declareLocal(nameNode: ts.Node, name: string, type: IrType, mutable: boolean): IrLocal { + if (type.kind === "dyn" && this.degradedTypes.size > 0) + this.noteDegradedSlot(nameNode, this.typeOf(nameNode)); const symbol = this.checker.getSymbolAtLocation(nameNode); const local = declareContextLocal( this.ctx, @@ -12781,7 +13511,7 @@ export class Lowerer { const fn = decl.parent; const symbol = this.checker.getSymbolAtLocation(nameNode); const isRead = (id: ts.Identifier): boolean => - id !== nameNode && this.checker.getSymbolAtLocation(id) === symbol; + id !== nameNode && bindingSymbolOfRead(this, id) === symbol; const pure = fn !== undefined && symbol !== undefined && onlyForwarded(this, fn, isRead); // A forwarded-only tuple is never built: a tuple with optional // positions (`Parameters<(n?: T) => R>`, whose length varies per call) @@ -13363,8 +14093,8 @@ export class Lowerer { return lowerStmts(this, stmts); } - noteBlockedBindings(stmt: ts.Statement): void { - return noteBlockedBindings(this, stmt); + noteBlockedBindings(stmt: ts.Statement, diags?: readonly ScrDiagnostic[]): void { + return noteBlockedBindings(this, stmt, diags); } isBlockedBinding(symbol: ts.Symbol | null): boolean { diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index cf0428b60f..3520e3aeac 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -2223,8 +2223,8 @@ export const BUILTIN_MODULE_FN_ALIASES: Record< mkdirSync: ["fs.mkdirModeSync", "fs.mkdirRecursiveSync", "fs.mkdirRecursiveModeSync"], // The { recursive, force } and maxRetries/retryDelay option lowerings. rmSync: ["fs.rmOptsSync", "fs.rmRetrySync"], - // The { withFileTypes: true } Dirent form. - readdirSync: ["fs.readdirTypesSync"], + // The { withFileTypes: true } Dirent form and the { recursive: true } walk. + readdirSync: ["fs.readdirTypesSync", "fs.readdirRecursiveSync"], // The { throwIfNoEntry } probe form. statSync: ["fs.statProbeSync"], // realpathSync and realpathSync.native read as function values. diff --git a/packages/compiler/src/frontend/lowering/type-failure.ts b/packages/compiler/src/frontend/lowering/type-failure.ts new file mode 100644 index 0000000000..3502a9bbf7 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/type-failure.ts @@ -0,0 +1,195 @@ +/* Localized failure, stage L0 (report-only). + * + * A type with no static lowering poisons the slot that holds it (a field, + * a parameter, a return, a local), and today the poison rejects the whole + * declaration around the slot: a class, a signature, a declaration + * statement. Every later use of that declaration then fails on its own + * line. The localized-failure design turns such slots into checked + * dynamic slots at the points where the build would otherwise commit the + * error. + * + * This module holds the bookkeeping that stage measures without changing + * any lowering: where a failing type sits (its slot), whether a dynamic + * slot would keep the program's meaning (design §3.7), and the keys that + * tie a cascade diagnostic to the rejected declaration it inherits from. + * The data rides the committed diagnostics as report-only fields + * (ScrDiagnostic.typeFailure, .rejects, .cause); tier-analysis.ts turns it + * into the would-be dynamic slot report of `--explain-dynamic`. */ +import type { + TypeFailure, + TypeFailureFamily, + TypeFailureSlot, +} from "../../diagnostics/diagnostic.js"; +import * as ts from "../ts7/adapter.js"; + +/** Families a checked-dynamic slot can hold (design §3.7). The others + * name values the compiler cannot produce (opaque stdlib and @types/node + * types, engine-backed ambient types), package value types that scope B + * routes to Node islands, enum objects with no runtime value, and generic + * implementations without an erased instance. */ +const DEGRADABLE_FAMILIES: ReadonlySet = new Set([ + "index-signature", + "overloads", + "intersection", + "component", + "record-member", + "unsupported", + "dyn-field-policy", +]); + +function nameText(name: ts.Node | undefined): string | null { + if (name === undefined) return null; + if (ts.isIdentifier(name) || ts.isPrivateIdentifier(name)) return name.text; + if (ts.isStringLiteral(name) || ts.isNumericLiteral(name)) return name.text; + return ts.getTextOfNode(name); +} + +function isParameterProperty(param: ts.ParameterDeclaration): boolean { + return ( + ts.isConstructorDeclaration(param.parent) && + (param.modifiers?.nodes ?? []).some( + (m) => + m.kind === ts.SyntaxKind.PublicKeyword || + m.kind === ts.SyntaxKind.PrivateKeyword || + m.kind === ts.SyntaxKind.ProtectedKeyword || + m.kind === ts.SyntaxKind.ReadonlyKeyword, + ) + ); +} + +function variableSlot(decl: ts.VariableDeclaration): TypeFailureSlot { + const list = decl.parent; + const holder = list?.parent; + if ( + holder !== undefined && + (ts.isForOfStatement(holder) || ts.isForInStatement(holder) || ts.isForStatement(holder)) + ) + return "loop"; + if (holder !== undefined && ts.isVariableStatement(holder) && ts.isSourceFile(holder.parent)) + return "global"; + return "local"; +} + +function functionLabel(fn: ts.Node): string { + if (ts.isConstructorDeclaration(fn)) return "constructor"; + if (ts.isFunctionLikeDeclaration(fn)) { + const own = nameText(fn.name); + if (own !== null) return own; + const parent = fn.parent; + if ( + parent !== undefined && + (ts.isVariableDeclaration(parent) || + ts.isPropertyAssignment(parent) || + ts.isPropertyDeclaration(parent)) + ) { + const bound = nameText(parent.name); + if (bound !== null) return bound; + } + } + return ""; +} + +/** The slot a badType blame node denotes. A declaration's own node, its + * name and its type annotation name the declaration's slot; anything + * reached through an expression is an expression slot (the type of a + * cast, a literal, a temporary). */ +export function typeFailureSlot(node: ts.Node): { slot: TypeFailureSlot; name: string } { + let current: ts.Node = node; + for (let depth = 0; depth < 64; depth++) { + if (ts.isParameter(current)) { + const name = nameText(current.name) ?? ""; + return { slot: isParameterProperty(current) ? "field" : "param", name }; + } + if (ts.isPropertyDeclaration(current)) + return { slot: "field", name: nameText(current.name) ?? "" }; + if (ts.isVariableDeclaration(current)) + return { slot: variableSlot(current), name: nameText(current.name) ?? "" }; + if (ts.isBindingElement(current)) { + let owner: ts.Node | undefined = current.parent; + while (owner !== undefined && !ts.isVariableDeclaration(owner) && !ts.isParameter(owner)) + owner = owner.parent; + const slot: TypeFailureSlot = + owner === undefined + ? "local" + : ts.isParameter(owner) + ? "param" + : variableSlot(owner as ts.VariableDeclaration); + return { slot, name: nameText(current.name) ?? "" }; + } + if (ts.isPropertySignature(current) || ts.isMethodSignatureDeclaration(current)) + return { slot: "member", name: nameText(current.name) ?? "" }; + if (ts.isFunctionLikeDeclaration(current)) + return { slot: "return", name: functionLabel(current) }; + if (ts.isCatchClause(current)) return { slot: "local", name: "" }; + const parent: ts.Node | undefined = current.parent; + if (parent === undefined || ts.isSourceFile(parent)) break; + // Climb from a declaration's name or type annotation, or from inside a + // type node; any other position is an expression's own type. + const declared = + (ts.isParameter(parent) || + ts.isPropertyDeclaration(parent) || + ts.isVariableDeclaration(parent) || + ts.isBindingElement(parent) || + ts.isPropertySignature(parent) || + ts.isMethodSignatureDeclaration(parent) || + ts.isFunctionLikeDeclaration(parent)) && + ((parent as { name?: ts.Node }).name === current || + (parent as { type?: ts.Node }).type === current); + if (!declared && !ts.isTypeNode(current) && !ts.isTypeNode(parent)) break; + current = parent; + } + const text = ts.getTextOfNode(node); + return { slot: "expression", name: text.length > 60 ? `${text.slice(0, 57)}...` : text }; +} + +/** Program code outside node_modules, in TypeScript: the code holes apply + * to (design §3.3, §3.8). JavaScript keeps its dynamic fallback and its + * runtime fences; npm packages keep island routing. */ +function programTypeScript(node: ts.Node): boolean { + const sf = ts.getSourceFileOfNode(node); + const file = sf.fileName; + if (file.includes("/node_modules/")) return false; + if (sf.isDeclarationFile) return false; + return !/\.(?:[cm]?js|jsx)$/.test(file); +} + +export function typeFailureOf( + node: ts.Node, + declaredType: string, + family: TypeFailureFamily, +): TypeFailure { + const { slot, name } = typeFailureSlot(node); + return { + slot, + name, + declaredType, + family, + degradable: DEGRADABLE_FAMILIES.has(family) && programTypeScript(node), + }; +} + +function locKey(node: ts.Node): string { + const sf = ts.getSourceFileOfNode(node); + return `${sf.fileName}:${ts.tokenStartOfNode(node)}`; +} + +/** The failure-group key of a deferred declaration (a class shape or a + * function signature whose collection poisoned). */ +export function declarationGroupKey(symbol: ts.Symbol): string { + const decl = symbol.declarations[0]; + const name = ts.symbolName(symbol); + if (decl === undefined) return `declaration ${name}`; + const kind = + ts.isClassDeclaration(decl) || ts.isClassExpression(decl) + ? "class" + : ts.isFunctionLikeDeclaration(decl) + ? "function" + : "declaration"; + return `${kind} ${name} @${locKey(decl)}`; +} + +/** The failure-group key of a poisoned declaration statement whose + * bindings are blocked. */ +export function statementGroupKey(stmt: ts.Statement): string { + return `statement @${locKey(stmt)}`; +} diff --git a/packages/compiler/src/frontend/lowering/typed-arguments.ts b/packages/compiler/src/frontend/lowering/typed-arguments.ts index e415add7cf..e837d59e6a 100644 --- a/packages/compiler/src/frontend/lowering/typed-arguments.ts +++ b/packages/compiler/src/frontend/lowering/typed-arguments.ts @@ -195,6 +195,15 @@ export function declareTypedArguments( }); } +/** The binding an identifier reads: its symbol, or for the name of a + * shorthand property (`{ xs }`) the variable whose value the property takes. */ +export function bindingSymbolOfRead(lowerer: Lowerer, id: ts.Identifier): ts.Symbol | undefined { + const parent = id.parent; + if (parent && ts.isShorthandPropertyAssignment(parent) && parent.name === id) + return lowerer.checker.getShorthandAssignmentValueSymbol(parent); + return lowerer.checker.getSymbolAtLocation(id); +} + /** Whether every read of a tuple binding in `fn` (identified by `isRoot`) * only forwards it: `f.apply(t, x)`, `g(...x)` into a program function, * `x.length`, or a const alias whose own reads only forward. Reads inside @@ -262,7 +271,7 @@ export function onlyForwarded( const alias = lowerer.checker.getSymbolAtLocation(p.name); const aliasName = p.name; if (alias) { - roots.push((id) => id !== aliasName && lowerer.checker.getSymbolAtLocation(id) === alias); + roots.push((id) => id !== aliasName && bindingSymbolOfRead(lowerer, id) === alias); return undefined; } } diff --git a/packages/compiler/src/frontend/tier-analysis.ts b/packages/compiler/src/frontend/tier-analysis.ts index b75e491a84..0baedd7dd7 100644 --- a/packages/compiler/src/frontend/tier-analysis.ts +++ b/packages/compiler/src/frontend/tier-analysis.ts @@ -29,7 +29,12 @@ * Attribution is syntactic and approximate: the site's innermost node, the * operand it acts on, and that operand's declaration. Stage S6 replaces it * with requests the failing lowering names exactly. */ -import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { + ScrDiagnostic, + TypeFailureFamily, + TypeFailureSlot, +} from "../diagnostics/diagnostic.js"; +import { isWarningCode } from "../diagnostics/render.js"; import type { IrFunction, SrcLoc } from "../ir/ir.js"; import { dynamicOperationSites, type DynamicFunction } from "../ir/tiers.js"; import * as ts from "./ts7/adapter.js"; @@ -85,8 +90,70 @@ export interface TierDynamicFunction extends DynamicFunction { statements: number; } +/** The localized-failure stage whose slot positions include a slot kind + * (design §3.6): locals and globals degrade first, declarations next, + * members of declared types with push-down, expression types last. */ +export type LocalizedFailureStage = "L1" | "L2" | "L3" | "L4"; + +export function slotStage(slot: TypeFailureSlot): LocalizedFailureStage { + switch (slot) { + case "local": + case "global": + case "loop": + return "L1"; + case "field": + case "param": + case "return": + return "L2"; + case "member": + return "L3"; + case "expression": + return "L4"; + } +} + +/** A committed type failure that would become a checked dynamic slot + * (SC6104) instead of an error. */ +export interface WouldBeSlot { + loc: SrcLoc; + slot: TypeFailureSlot; + name: string; + declaredType: string; + family: TypeFailureFamily; + code: string; + message: string; + stage: LocalizedFailureStage; + /** The declaration the failure rejects today (a failure-group key), if + * any: a class, a signature, or a declaration statement. */ + rejects: string | null; + /** Other committed diagnostics that inherit from this slot's group and + * clear once every group they inherit from degrades (with push-down and + * expression slots, the full design). */ + cascade: number; +} + +/** Localized failure, report-only (stage L0): what the committed errors + * would become. Two scenarios: through stage L2 (declaration slots) and + * through L4 (members and expressions too). */ +export interface LocalizedFailureReport { + /** Reached errors (warnings excluded). */ + errors: number; + /** Type failures, by whether a checked dynamic slot keeps meaning. */ + typeFailures: { degradable: number; refused: number }; + slots: WouldBeSlot[]; + /** Diagnostics attributed to a rejected declaration they used. */ + cascades: number; + /** Errors that disappear (degraded slots plus cleared cascades). */ + clearedThroughL2: number; + clearedThroughL4: number; + /** The errors that remain through L4, by code. */ + remainingByCode: Record; +} + export interface TierReport { mode: TierMode; + /** Localized failure (report-only). */ + localizedFailure: LocalizedFailureReport; /** Reached source statements that perform a dynamic operation. */ statementsDynamic: number; dynamicFunctions: TierDynamicFunction[]; @@ -576,6 +643,119 @@ function demotedUnits(requests: readonly DemotionRequest[]): DemotedUnit[] { ); } +/* ── localized failure ───────────────────────────────────────────────── */ + +/** A diagnostic's failure-group bookkeeping (lowerer: badType, deferred + * flushes, blocked statements) turned into the would-be report. A group + * (a rejected declaration or blocked statement) clears in a scenario when + * each of its diagnostics either degrades (a degradable type failure at a + * slot the scenario covers) or is a cascade whose every cause clears. */ +export function localizedFailureReport(reached: readonly ScrDiagnostic[]): LocalizedFailureReport { + const errors = reached.filter((d) => !isWarningCode(d.code)); + const groups = new Map(); + for (const d of errors) { + if (d.rejects === undefined) continue; + const list = groups.get(d.rejects); + if (list === undefined) groups.set(d.rejects, [d]); + else list.push(d); + } + const stages: Record<"L2" | "L4", ReadonlySet> = { + L2: new Set(["L1", "L2"]), + L4: new Set(["L1", "L2", "L3", "L4"]), + }; + const clears = (scenario: "L2" | "L4") => { + const covered = stages[scenario]; + const memo = new Map(); + const degrades = (d: ScrDiagnostic): boolean => + d.typeFailure !== undefined && + d.typeFailure.degradable && + covered.has(slotStage(d.typeFailure.slot)); + const causeClears = (d: ScrDiagnostic): boolean => + d.cause !== undefined && d.cause.length > 0 && d.cause.every((g) => groupClears(g)); + const groupClears = (group: string): boolean => { + const known = memo.get(group); + if (known !== undefined) return known; + memo.set(group, false); // a cycle never clears + const members = groups.get(group) ?? []; + const ok = members.length > 0 && members.every((d) => degrades(d) || causeClears(d)); + memo.set(group, ok); + return ok; + }; + return { degrades, causeClears, groupClears }; + }; + const l2 = clears("L2"); + const l4 = clears("L4"); + const cleared = (c: ReturnType, d: ScrDiagnostic) => + c.degrades(d) || c.causeClears(d); + // Each slot's cascade: diagnostics whose causes (transitively) include + // the slot's group and that clear in the full scenario. + const reachesGroup = (d: ScrDiagnostic, group: string, seen: Set): boolean => { + for (const g of d.cause ?? []) { + if (g === group) return true; + if (seen.has(g)) continue; + seen.add(g); + for (const m of groups.get(g) ?? []) if (reachesGroup(m, group, seen)) return true; + } + return false; + }; + const slots: WouldBeSlot[] = []; + let degradable = 0; + let refused = 0; + for (const d of errors) { + const f = d.typeFailure; + if (f === undefined) continue; + if (!f.degradable) { + refused++; + continue; + } + degradable++; + const group = d.rejects ?? null; + let cascade = 0; + if (group !== null) + for (const e of errors) + if ( + e !== d && + e.typeFailure === undefined && + cleared(l4, e) && + reachesGroup(e, group, new Set()) + ) + cascade++; + slots.push({ + loc: d.loc, + slot: f.slot, + name: f.name, + declaredType: f.declaredType, + family: f.family, + code: d.code, + message: d.message, + stage: slotStage(f.slot), + rejects: group, + cascade, + }); + } + slots.sort( + (a, b) => + b.cascade - a.cascade || a.loc.file.localeCompare(b.loc.file) || a.loc.start - b.loc.start, + ); + const remainingByCode: Record = {}; + let clearedThroughL2 = 0; + let clearedThroughL4 = 0; + for (const d of errors) { + if (cleared(l2, d)) clearedThroughL2++; + if (cleared(l4, d)) clearedThroughL4++; + else remainingByCode[d.code] = (remainingByCode[d.code] ?? 0) + 1; + } + return { + errors: errors.length, + typeFailures: { degradable, refused }, + slots, + cascades: errors.filter((d) => d.cause !== undefined && d.typeFailure === undefined).length, + clearedThroughL2, + clearedThroughL4, + remainingByCode, + }; +} + export interface TierAnalysisInput { mode: TierMode; /** The dynamic-tier IR functions of the reached program. */ @@ -624,6 +804,7 @@ export function analyzeTiers(program: ts.Program, input: TierAnalysisInput): Tie const dynamic = dynamicFunctionReports(program, input.dynamicFunctions); return { mode: input.mode, + localizedFailure: localizedFailureReport(input.reached), statementsDynamic: dynamic.statements, dynamicFunctions: dynamic.functions, requests, diff --git a/packages/compiler/src/frontend/ts7/ast-node.test.ts b/packages/compiler/src/frontend/ts7/ast-node.test.ts index 8b391e6221..b650e7725b 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.test.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.test.ts @@ -473,6 +473,29 @@ test("named child access shares node and list identities without materializing s expect(() => file.namedChild(another, "name")).toThrow("another source file"); }); +test("named child getters and text keep answering the first read", () => { + let materialized = 0; + const file = new AstFile(decoded.get("main.ts")!.bytes, () => { + materialized++; + }); + let index = 1; + while (file.wire.kind(index) !== AstKind.FunctionDeclaration) index++; + const node = file.node(index); + const name = node.name; + expect(name?.index).toBe(file.wire.namedChild(index, "name")); + const after = materialized; + expect(node.name).toBe(name); + expect(node.childNode("name")).toBe(name); + expect(file.node(name!.index)).toBe(name); + expect(node.typeParameters).toBeUndefined(); + expect(materialized).toBe(after); + const text = name!.text; + expect(text).toBe(file.wire.text(name!.index)); + expect(name!.text).toBe(text); + expect(node.text).toBeUndefined(); + expect(node.text).toBeUndefined(); +}); + test("warming a named child preserves lazy failures in later malformed sibling links", () => { const bytes = decoded.get("main.ts")!.bytes; const wire = new AstFile(bytes).wire; diff --git a/packages/compiler/src/frontend/ts7/ast-node.ts b/packages/compiler/src/frontend/ts7/ast-node.ts index 4c7726c004..c49dceffb4 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.ts @@ -83,6 +83,9 @@ export interface AstFileOptions { writeTargets?: boolean; } +const NO_CHILD_INDICES = new Uint32Array(0); +const NO_CHILD_NODES: AstNode[] = []; + export class AstFile { readonly wire: AstWireFile; readonly root: AstNode; @@ -94,7 +97,9 @@ export class AstFile { private readonly structuredNodes = new Map(); private readonly strings = new Map(); private lines: number[] | undefined; - private childIndices: Uint32Array | undefined; + /** Resolved named children, eight slots per node; empty until the first + * named child is read. */ + private childIndices: Uint32Array = NO_CHILD_INDICES; /** The root source file's name (AstNode.fileName). */ private rootFileName: string | undefined; @@ -164,8 +169,8 @@ export class AstFile { const mask = node.data >>> 30 === 0 ? node.data & 0xff : 0xff; if ((mask & (1 << order)) === 0) return 0; const slot = node.index * 8 + order; - const indices = this.childIndices; - if (indices !== undefined) { + let indices = this.childIndices; + if (indices.length !== 0) { const cached = indices[slot]!; if (cached !== 0) return cached; } @@ -173,9 +178,11 @@ export class AstFile { // The pinned wire has at most eight named children per node. One packed // table avoids per-node arrays; allocate it only after successful access // so malformed siblings still fail lazily in the checked wire decoder. - if (this.childIndices === undefined) - this.childIndices = new Uint32Array(this.wire.nodeCount * 8); - this.childIndices[slot] = index; + if (indices.length === 0) { + indices = new Uint32Array(this.wire.nodeCount * 8); + this.childIndices = indices; + } + indices[slot] = index; return index; } @@ -320,6 +327,13 @@ export class AstNode { readonly data: number; private parentResolved = false; private parentCache: AstNode | undefined; + /** The node's text, read once: the frontend asks the same identifiers + * for their text many times. */ + private textResolved = false; + private textCache: string | undefined; + /** Named child nodes by wire slot, filled as they are read; empty until + * the first one is. Slots the child mask leaves empty are never stored. */ + private childNodes: AstNode[] = NO_CHILD_NODES; constructor( readonly file: AstFile, @@ -351,7 +365,11 @@ export class AstNode { return this.parentCache; } get text(): string | undefined { - return this.file.wire.text(this.index); + if (!this.textResolved) { + this.textCache = this.file.wire.text(this.index); + this.textResolved = true; + } + return this.textCache; } get rawText(): string | undefined { return this.file.wire.rawText(this.index); @@ -363,8 +381,21 @@ export class AstNode { } /** childNode by wire slot, as the generated getters select it. */ childNodeAt(order: number): AstNode | undefined { + let nodes = this.childNodes; + if (order >= 0 && order < nodes.length) { + const cached = nodes[order]; + if (cached !== undefined) return cached; + } const index = this.file.namedChildAt(this, order); - return index === 0 ? undefined : this.file.node(index); + if (index === 0) return undefined; + const child = this.file.node(index); + if (nodes.length === 0) { + const mask = this.data >>> 30 === 0 ? this.data & 0xff : 0xff; + nodes = new Array(32 - Math.clz32(mask)); + this.childNodes = nodes; + } + nodes[order] = child; + return child; } childListAt(order: number): AstNodeList | undefined { const index = this.file.namedChildAt(this, order); diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 0635ad6aeb..59ae4887d2 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1282,6 +1282,11 @@ export interface TypeMapperCtx { * checker type its signature instantiates, or null. Consulted only when * the symbolic form does not map. */ resolveBoundAliasTs?: TypeParamTsResolver; + /** Localized failure: the lenient mapping of a type whose strict mapping + * failed at a committed slot (Lowerer.degradedTypes), or null. Consulted + * only after the strict mapping (and its bound-alias retry) returns null, + * at every depth, so every occurrence of a degraded type maps alike. */ + degradedType?: (type: ts.Type) => IrType | null; /** GENERIC program classes (monomorphization by flow): the mapping of a * concrete instantiation reference (`Box`) — the Lowerer * registers/reuses the instantiation (`Box%0`) and answers its object @@ -1856,6 +1861,16 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (mapped !== null) contextResolutions++; } } + // Localized failure: a type that failed at a committed slot maps to its + // lenient form. It counts as a context resolution so neither it nor a + // type containing it is memoized: the strict form may map elsewhere. + if (mapped === null && ctx.degradedType) { + const lenient = ctx.degradedType(type); + if (lenient !== null) { + mapped = lenient; + contextResolutions++; + } + } if ( mapped !== null && memoKey !== null && diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index a71e2c0389..079db8494e 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -55,6 +55,7 @@ export type { import { compilePackedLibrary } from "./backend/library-pack.js"; import { node24FetchCompatProfile } from "./compat/fetch-profile.js"; import { libFnSignatures } from "./ir/builtin-signatures.js"; +import { type FunctionLayout, parseFunctionLayout } from "./backend/llvm/function-layout.js"; import { InternalCompilerError } from "./errors.js"; import { frontendProcessExecArgv } from "./frontend-process-args.js"; import { type ChildProcess, spawn } from "node:child_process"; @@ -64,6 +65,7 @@ import { copyFile, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node: import { tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { gunzipSync } from "node:zlib"; import { clearCcCaches, configuredTargetPlatform, @@ -538,6 +540,14 @@ function performanceWarnings(sink: CompileWarningSink | undefined): { return performance.length === 0 ? {} : { performance }; } +/** The profile match of a --profile-use build's program object, for its + * artifact and its executable cache entry, which a cache hit reports again. */ +function profileMatchField(object: NativeProgramObject | null): { + profileMatch?: NativeProfileMatch; +} { + return object?.profileMatch === undefined ? {} : { profileMatch: object.profileMatch }; +} + function withCompileWarnings( result: T, sink: CompileWarningSink, @@ -802,6 +812,37 @@ async function readNativeProfile( return { use: { path, digest: createHash("sha256").update(bytes).digest("hex") } }; } +/** The digest of the request's function layout file, which keys the + * executable cache: the layout changes the code's order, not its frontend + * input. */ +async function functionLayoutKey(opts: CompileRequestOptions): Promise { + if (opts.functionLayout === undefined) return undefined; + const path = resolve(opts.functionLayout); + let bytes: Buffer; + try { + bytes = await readFile(path); + } catch (err) { + throw new Error( + `cannot read function layout ${path}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + return createHash("sha256").update(bytes).digest("hex"); +} + +/** The request's function layout. */ +async function readFunctionLayout( + opts: CompileRequestOptions, +): Promise<{ functionLayout?: FunctionLayout }> { + if (opts.functionLayout === undefined) return {}; + const path = resolve(opts.functionLayout); + const bytes = await readFile(path); + return { + functionLayout: parseFunctionLayout( + (path.endsWith(".gz") ? gunzipSync(bytes) : bytes).toString("utf8"), + ), + }; +} + function usesPrecompiledRuntimePack(opts: CompileRequestOptions, backend: "llvm"): boolean { if (backend !== "llvm" || opts.sanitize === true) return false; return nativeCodegenTarget() !== null; @@ -875,6 +916,7 @@ async function prepareExecutableInput( llvm = emitLlvmModuleSource(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, + ...(await readFunctionLayout(opts)), pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", runtimeAbiMarker: outputKind === "obj", @@ -949,6 +991,7 @@ async function prepareExecutableInput( llvmSource = emitLlvmModuleSource(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, + ...(await readFunctionLayout(opts)), pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", runtimeAbiMarker: useRuntimePack, @@ -1574,9 +1617,11 @@ async function compileTracked( platform: buildPlatform, }); let profile: NativeProfileOptions | undefined; + let functionLayoutDigest: string | undefined; try { if (profileRefusal !== null) throw new Error(profileRefusal); profile = await readNativeProfile(opts); + functionLayoutDigest = await functionLayoutKey(opts); } catch (err) { return { ok: false, @@ -1612,6 +1657,7 @@ async function compileTracked( ...targetCpuField(opts.targetCpu), ...(opts.cpuDispatch === true ? { cpuDispatch: true as const } : {}), ...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }), + ...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), npmStatic: opts.npmStatic ?? null, @@ -1687,6 +1733,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: earlyHit.llvmPath, backend: earlyHit.native.backend, + ...(earlyHit.profileMatch === undefined ? {} : { profileMatch: earlyHit.profileMatch }), }, binaryPath: opts.outPath, llvmPath: earlyHit.llvmPath, @@ -1764,6 +1811,7 @@ async function compileTracked( executableRestored: true, nativeDependencies: dependencies, frontend: earlyHit.frontend, + ...profileMatchField(nativeProgramObject), }); }, nativeProgramObject?.partitionPaths, @@ -1809,6 +1857,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: earlyHit.llvmPath, backend: earlyHit.native.backend, + ...profileMatchField(nativeProgramObject), }, binaryPath: opts.outPath, llvmPath: earlyHit.llvmPath, @@ -1900,6 +1949,7 @@ async function compileTracked( frontend: frontendInputs.snapshot(), ...(irSourcePath === undefined ? {} : { irPath: irSourcePath }), ...performanceWarnings(warningSink), + ...profileMatchField(nativeProgramObject), }); publishedExecutable = true; }, @@ -1919,6 +1969,7 @@ async function compileTracked( frontend: frontendInputs.snapshot(), ...(irSourcePath === undefined ? {} : { irPath: irSourcePath }), ...performanceWarnings(warningSink), + ...profileMatchField(nativeProgramObject), }).catch(() => undefined); } } catch (err) { @@ -1947,7 +1998,6 @@ async function compileTracked( } await pruneBuildCache(cacheRoot); timing("complete"); - const profileMatch = nativeProgramObject?.profileMatch; return { ok: true, artifact: { @@ -1955,7 +2005,7 @@ async function compileTracked( path: opts.outPath, translationUnitPath: llvmPath, backend, - ...(profileMatch === undefined ? {} : { profileMatch }), + ...profileMatchField(nativeProgramObject), }, binaryPath: opts.outPath, llvmPath, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 423219efaf..709f72d45c 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -765,6 +765,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "fs.rmSync", "fs.rmdirSync", "fs.readdirSync", + "fs.readdirRecursiveSync", "fs.readdirTypesSync", "fs.globSync", "fs.globTypesSync", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index ca7cf9ebbe..0be10a37a6 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -592,6 +592,9 @@ function buildLibFnSignatures() { "fs.rmSync": { argTypes: [STRING], result: VOID }, "fs.rmdirSync": { argTypes: [STRING], result: VOID }, "fs.readdirSync": { argTypes: [STRING], result: arrayOf(STRING) }, + /** `fs.readdirSync(path, { recursive: true })`: every entry below the + * path, relative to it, in Node's queue order. */ + "fs.readdirRecursiveSync": { argTypes: [STRING], result: arrayOf(STRING) }, // Result is the call site's Dirent record array (VOID is the // structure-checked sentinel, the os.networkInterfaces pattern). /** `fs.readdirSync(path, { withFileTypes: true })` — Dirent rows over @@ -3997,6 +4000,12 @@ function buildLibFnSignatures() { * Throws the path TypeError, changing nothing, when a converted object * is not plain data (a prototype, accessors, symbols, frozen state). */ "dyn.declAdopt": { argTypes: [DYN, DYN], result: VOID }, + /** dyn.declAdopt for every other checked cast of a dynamic value to a + * record or array (`value as T`): a shared source object or array + * becomes a live capsule of the value built from it. A node that is not + * plain data, or that the value holds twice, keeps its separate copy. + * Never throws: the conversion already succeeded. */ + "dyn.adoptShared": { argTypes: [DYN, DYN], result: VOID }, "dyn.cloneMissing": { argTypes: [], result: DYN }, /** structuredClone with a NON-EMPTY transfer array of static values: * nothing static is transferable, so the call always throws Node's diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 056449a3cd..a4c1e8c35a 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -967,7 +967,10 @@ const RUNTIME_FEATURE_RULES = { /** The declaration-typed boundary's adoption (scr_decl_adopt.c): its * casts, and closures boxed in a boundary program, which adopt their * arguments. */ - declAdopt: rule({ fns: ["dyn.declAdopt"], initial: (mod) => mod.declBoundary === true }), + declAdopt: rule({ + fns: ["dyn.declAdopt", "dyn.adoptShared"], + initial: (mod) => mod.declBoundary === true, + }), // Checked conversions into package-declared types name the declaration // in their TypeError (scr_type_origin.c). typeOrigins: rule({ initial: (mod) => (mod.declOrigins?.length ?? 0) > 0 }), diff --git a/packages/compiler/src/native/compiler.test.ts b/packages/compiler/src/native/compiler.test.ts index a085356a09..c9636ea76d 100644 --- a/packages/compiler/src/native/compiler.test.ts +++ b/packages/compiler/src/native/compiler.test.ts @@ -107,7 +107,7 @@ test.each([ }); vi.mocked(emitNativeObject).mockImplementation((options) => { writeFileSync(options.outputPath, "object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); const cache = new NativeCache(join(directory, "cache")); const key = contentDigest("executable link"); @@ -250,7 +250,7 @@ test.each([false, true])( expect(emitted).toContain("define i32 @main"); if (fail) throw new Error("codegen failed"); writeFileSync(options.outputPath, "new object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); const target = MACOS_ARM64_TARGET; const toolchain: NativeToolchain = { @@ -341,7 +341,7 @@ test.skipIf(process.platform === "win32").each([false, true])( }); vi.mocked(emitNativeObject).mockImplementation((options) => { writeFileSync(options.outputPath, "object"); - return [options.outputPath]; + return { outputs: [options.outputPath], profileMatch: null }; }); vi.mocked(openNativeExecutableCache).mockReturnValue(null); const links: string[][] = []; @@ -378,3 +378,84 @@ test.skipIf(process.platform === "win32").each([false, true])( if (lldFails) expect(links[1]).not.toContain("-fuse-ld=lld"); }, ); + +test.skipIf(process.platform === "win32")( + "a --profile-use executable reports the helper's profile match", + () => { + const directory = mkdtempSync(join("/tmp", "scriptc-native-profile-")); + directories.push(directory); + const clang = join(directory, "clang"); + writeFileSync(clang, "#!/bin/sh\n"); + chmodSync(clang, 0o755); + const entry = join(directory, "entry.ts"); + const output = join(directory, "program"); + const profile = join(directory, "app.profdata"); + writeFileSync(profile, "indexed profile"); + const module: IrModule = { + irVersion: 17, + sourceFile: entry, + entry: "%main", + functions: [ + { + name: "%main", + params: [], + locals: [], + body: [], + returnType: VOID, + loc: { file: entry, start: 0, end: 0 }, + }, + ], + }; + vi.mocked(runNativeFrontend).mockReturnValue({ + preflight: [], + entryText: () => "", + entryExports: () => new Map(), + entryContract: () => { + throw new Error("unexpected contract request"); + }, + sourceTexts: () => new Map(), + npmStatic: [], + npmImportSites: new Map(), + dispose: () => {}, + lower: () => ({ + module, + diagnostics: [], + runtimeFences: [], + stats: { statementsTotal: 0, statementsFailed: 0, functionsSkipped: 0 }, + }), + }); + const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 }; + vi.mocked(emitNativeObject).mockImplementation((options) => { + expect(options.profile?.use?.path).toBe(profile); + writeFileSync(options.outputPath, "object"); + return { outputs: [options.outputPath], profileMatch: match }; + }); + vi.mocked(openNativeExecutableCache).mockReturnValue(null); + vi.mocked(execFileSync).mockImplementation((_command, args) => { + const list = args as string[]; + writeFileSync(list[list.indexOf("-o") + 1]!, "executable"); + return ""; + }); + const target = LINUX_X64_GNU_TARGET; + const compiler = new NativeCompiler({ + compilerVersion: "0.0.0", + target, + helper: target.helper, + ts7Executable: "unused", + helperExecutable: "unused", + helperPackageRoot: "unused", + runtimePackRoot: "unused", + linker: clang, + linkerArgs: [], + dsymutil: "unused", + }); + const result = compiler.compile(entry, { + outDir: directory, + outPath: output, + backend: "llvm", + optimization: "speed", + profileUse: profile, + }); + expect(result).toMatchObject({ ok: true, artifact: { kind: "exe", profileMatch: match } }); + }, +); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index f2734d2ee0..ea3772db3c 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -63,6 +63,8 @@ import { contentDigest, fileDigest, partsDigest, type NativeCache } from "./cach import { nativeProfileKey, nativeProfileRefusal, + parseNativeProfileMatch, + type NativeProfileMatch, type NativeProfileOptions, } from "../backend/native-profile.js"; import { splitLlvmLibraryProgram } from "../backend/llvm/split.js"; @@ -78,6 +80,18 @@ function partitionCacheKey(key: string, index: number): string { return index === 0 ? key : contentDigest(`${key}:${index}`); } +/** The object cache entry that holds a cached program's profile match. */ +function profileMatchCacheKey(key: string): string { + return contentDigest(`${key}:profile-match`); +} + +/** A program's native objects, and how a --profile-use build's profile + * matched its functions. */ +interface NativeObjects { + paths: string[]; + profileMatch?: NativeProfileMatch; +} + function failure(error: unknown, entry: string, sources: Map): CompileFailure { return { ok: false, @@ -186,7 +200,7 @@ export class NativeCompiler { partitions = 1, profile?: NativeProfileOptions, llvmDigest: () => string = () => fileDigest(input), - ): string[] { + ): NativeObjects { const toolchain = this.toolchain; const outputs = nativePartitionPaths(output, partitions); const helperOptions = { @@ -223,7 +237,20 @@ export class NativeCompiler { verifyNativeHelper(helperOptions); if (nativeFileIdentity(toolchain.helperExecutable) === identity) { cached.forEach((bytes, index) => writeFileSync(outputs[index]!, bytes!)); - return outputs; + if (profile?.use === undefined) return { paths: outputs }; + // Without the cached match, every partition's counts are unknown. + const match = this.cache.read("object", profileMatchCacheKey(key)); + return { + paths: outputs, + profileMatch: (match === null + ? null + : parseNativeProfileMatch(match.toString("utf8"))) ?? { + annotated: 0, + mismatched: 0, + missing: 0, + unknownPartitions: outputs.length, + }, + }; } key = null; } @@ -235,7 +262,7 @@ export class NativeCompiler { this.cache === null || identity === null || partitions < 2 ? null : this.cache.partitionDirectory(identity); - emitNativeObject({ + const { profileMatch } = emitNativeObject({ executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, compilerVersion: toolchain.compilerVersion, @@ -252,15 +279,30 @@ export class NativeCompiler { }); if (this.cache !== null && key !== null) { try { - if (nativeFileIdentity(toolchain.helperExecutable) === identity) + if (nativeFileIdentity(toolchain.helperExecutable) === identity) { + // The match goes in first: a reader that finds every partition finds it too. + if (profileMatch !== null) + this.cache.write( + "object", + profileMatchCacheKey(key), + `${JSON.stringify({ + profile: { + annotated: profileMatch.annotated, + mismatched: profileMatch.mismatched, + missing: profileMatch.missing, + unknown_partitions: profileMatch.unknownPartitions, + }, + })}\n`, + ); outputs.forEach((path, index) => this.cache!.write("object", partitionCacheKey(key!, index), readFileSync(path)), ); + } } catch { /* An updated helper or unavailable cache only prevents reuse. */ } } - return outputs; + return profileMatch === null ? { paths: outputs } : { paths: outputs, profileMatch }; } private emitProgramObject( @@ -272,7 +314,7 @@ export class NativeCompiler { library: boolean, profile?: NativeProfileOptions, llvmDigest?: () => string, - ): string[] { + ): NativeObjects { // Library archives merge dev shards relocatably; executables link the // helper's program partitions directly. const llvm = @@ -303,7 +345,7 @@ export class NativeCompiler { } const merged = localizeNativeLibrary(this.toolchain, stage, objects, [], split.publicSymbols); renameSync(merged, output); - return [output]; + return { paths: [output] }; } compile(entry: string, options: CompileRequestOptions): CompileRequestResult { @@ -427,6 +469,9 @@ export class NativeCompiler { const kept = executableCache?.keep(output) ?? false; const restored = kept || (executableCache?.restore(stagedOutput) ?? false); timing(restored ? "executable-cache-hit" : "executable-cache-miss"); + // The executable cache leaves out --profile-use builds, so this is set + // whenever the build has a profile to report. + let profileMatch: NativeProfileMatch | undefined; if (!restored) { // A missing or invalid runtime pack fails before code generation. if (outputKind === "exe") selectPack(); @@ -440,8 +485,8 @@ export class NativeCompiler { object, ]); requireNativeArtifact(object); - } else if (outputKind === "exe") - programObjects = this.emitProgramObject( + } else if (outputKind === "exe") { + const objects = this.emitProgramObject( llvmInput, object, entry, @@ -451,7 +496,9 @@ export class NativeCompiler { profile, digestLlvm, ); - else + programObjects = objects.paths; + profileMatch = objects.profileMatch; + } else this.emitObject( llvmInput, object, @@ -578,7 +625,13 @@ export class NativeCompiler { rmSync(output + ".dSYM", { recursive: true, force: true }); return { ok: true, - artifact: { kind: "exe", path: output, translationUnitPath: llvmPath, backend: "llvm" }, + artifact: { + kind: "exe", + path: output, + translationUnitPath: llvmPath, + backend: "llvm", + ...(profileMatch === undefined ? {} : { profileMatch }), + }, binaryPath: output, llvmPath, backend: "llvm", diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts new file mode 100644 index 0000000000..ac25924f3a --- /dev/null +++ b/packages/compiler/test/canon-normalize.test.ts @@ -0,0 +1,273 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { canonEqual, canonKey, Normalizer, type Canon } from "../src/frontend/canon.js"; +import { loadProgram } from "../src/frontend/program-node.js"; +import * as ts from "../src/frontend/ts7/adapter.js"; + +let directory: string; +let load: ReturnType; +let checker: ts.TypeChecker; +const aliases = new Map(); +const asTypes = new Map(); +const valueTypes = new Map(); +const typeParams = new Map(); + +beforeAll(() => { + directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-canon-")); + const entry = join(directory, "input.ts"); + writeFileSync( + entry, + ` + // #2228: an intersection that restates an optional array member. + export interface BuildOptions { cwd: string; contextFiles?: Array<{ path: string; content: string }> } + export type Normalized = BuildOptions & { contextFiles: Array<{ path: string; content: string }> }; + export type Same = { x: number[] } & { x: number[] }; + // Unions flatten, drop never, read void as undefined, and dedupe. + export type Flat = string | (number | undefined) | never | void | string; + export type Flag = true | false | 0; + // Branded primitive. + export type Id = string & { readonly brand: "id" }; + // #2241/#2246: a recursive interface with a generic method. + export interface TreeNode { name: string; child?: TreeNode; send(message: T): string } + // #2227: a generic alias spelled over a generic body's own parameters. + export type Catalog = { groups: G[]; provider: P }; + export function cast(x: unknown) { + return x as Catalog; + } + // Narrowed type parameters. + export function present(x: T & {}) { + return x; + } + export const sampleNumber = 1 as number; + export const sampleText = "a" as string; + export const sampleMaybe = undefined as string | undefined; + export class Point { x = 0 } + export type Points = readonly Point[]; + export type Pair = [string, number]; + // Two type parameters named T, in different functions. + export function first(x: unknown) { + return x as T[]; + } + export function second(x: unknown) { + return x as readonly T[]; + } + // Distinct callables, and arrays of them. + export type Calls = ((a: string) => void) & ((a: number) => void); + export type CallArrays = ((a: string) => void)[] | ((a: number) => void)[]; + // A dictionary with a named property. + export type Dict = { [key: string]: number } & { size: number }; + // Recursive array and tuple aliases. + export type Nested = number | Nested[]; + export type Chain = [number, Chain] | null; + // Optional and rest tuple elements. + export type Loose = [string, number?, ...boolean[]]; + // Arrays of two instantiations of one generic class. + export type Maps = Map[] | Map[]; + `, + ); + load = loadProgram(entry); + checker = load.program.getTypeChecker(); + ts.walkPreorder(load.entry, (node) => { + if (ts.isTypeAliasDeclaration(node) || ts.isInterfaceDeclaration(node)) + aliases.set( + node.name.text, + checker.getTypeAtLocation(ts.isTypeAliasDeclaration(node) ? node.type : node.name), + ); + if (ts.isAsExpression(node)) { + let fn: ts.Node | undefined = node.parent; + while (fn !== undefined && !ts.isFunctionDeclaration(fn)) fn = fn.parent; + const params = + fn !== undefined && ts.isFunctionDeclaration(fn) + ? (fn.typeParameters?.nodes ?? []).map((p) => checker.getSymbolAtLocation(p.name)!) + : []; + asTypes.set(ts.getTextOfNode(node.type), { + type: checker.getTypeAtLocation(node.type), + params, + }); + } + if (ts.isTypeParameterDeclaration(node) && ts.isFunctionDeclaration(node.parent)) + typeParams.set( + `${node.parent.name?.text}.${node.name.text}`, + checker.getSymbolAtLocation(node.name)!, + ); + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) + valueTypes.set(node.name.text, checker.getTypeAtLocation(node.name)); + if ( + ts.isParameter(node) && + node.type !== undefined && + ts.isFunctionDeclaration(node.parent) && + node.parent.name?.text === "present" + ) + asTypes.set("present", { + type: checker.getTypeAtLocation(node.type), + params: (node.parent.typeParameters?.nodes ?? []).map((p) => + checker.getSymbolAtLocation(p.name)!, + ), + }); + }); +}); +afterAll(() => { + load?.dispose(); + if (directory) rmSync(directory, { recursive: true, force: true }); +}); + +const key = (c: Canon) => canonKey(c); + +describe("canon", () => { + test("intersections that restate members resolve to one object (#2228)", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Same")!))).toBe("{ x: number[] }"); + expect(key(n.canon(aliases.get("Normalized")!))).toBe( + "{ cwd: string; contextFiles: { path: string; content: string }[] }", + ); + }); + + test("unions are flat, never-free and deduplicated", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Flat")!))).toBe("string | number | undefined"); + expect(key(n.canon(aliases.get("Flag")!))).toBe("boolean | 0"); + }); + + test("a branded primitive is the primitive", () => { + expect(key(new Normalizer(checker).canon(aliases.get("Id")!))).toBe("string"); + }); + + test("recursion closes on the node under construction (#2241, #2246)", () => { + const n = new Normalizer(checker); + const tree = n.canon(aliases.get("TreeNode")!); + expect(tree.k).toBe("obj"); + if (tree.k !== "obj") return; + const child = tree + .props() + .find((p) => p.name === "child")! + .type(); + expect(child.k).toBe("union"); + if (child.k !== "union") return; + expect(child.arms).toContain(tree); + const send = tree + .props() + .find((p) => p.name === "send")! + .type(); + expect(send.k === "obj" && send.call()[0]!.generic).toBe(true); + }); + + test("a generic alias in a generic body substitutes the environment (#2227)", () => { + const n = new Normalizer(checker); + const { type, params } = asTypes.get("Catalog")!; + expect(key(n.canon(type))).toBe("{ groups: param G[]; provider: param P }"); + const bound = n.canon( + type, + new Map([ + [params[0]!, valueTypes.get("sampleNumber")!], + [params[1]!, valueTypes.get("sampleText")!], + ]), + ); + expect(key(bound)).toBe("{ groups: number[]; provider: string }"); + expect(bound.k === "obj" && bound.alias?.name).toBe("Catalog"); + expect(bound.k === "obj" && bound.alias?.tsType).toBe(type); + }); + + test("a narrowed parameter drops its unit arms", () => { + const n = new Normalizer(checker); + const { type, params } = asTypes.get("present")!; + expect(key(n.canon(type, new Map([[params[0]!, valueTypes.get("sampleMaybe")!]])))).toBe( + "string", + ); + }); + + test("type parameters with the same name are distinct in the intern key", () => { + const n = new Normalizer(checker); + const { type } = asTypes.get("T[]")!; + const firstT = typeParams.get("first.T")!; + const secondT = typeParams.get("second.T")!; + const num = valueTypes.get("sampleNumber")!; + const text = valueTypes.get("sampleText")!; + expect( + key( + n.canon( + type, + new Map([ + [firstT, num], + [secondT, text], + ]), + ), + ), + ).toBe("number[]"); + expect( + key( + n.canon( + type, + new Map([ + [firstT, text], + [secondT, num], + ]), + ), + ), + ).toBe("string[]"); + }); + + test("callables and arrays of callables with different parameters stay distinct", () => { + const n = new Normalizer(checker); + const calls = n.canon(aliases.get("Calls")!); + expect(calls.k === "obj" && calls.call().length).toBe(2); + const arms = n.canon(aliases.get("CallArrays")!); + expect(arms.k === "union" && arms.arms.length).toBe(2); + expect(key(arms)).toBe("{ (string) => void }[] | { (number) => void }[]"); + }); + + test("intersecting a dictionary with a named property keeps the index signature", () => { + const dict = new Normalizer(checker).canon(aliases.get("Dict")!); + expect(key(dict)).toBe("{ [string]: number; size: number }"); + }); + + test("recursive array and tuple aliases close on themselves", () => { + const n = new Normalizer(checker); + const nested = n.canon(aliases.get("Nested")!); + expect(nested.k).toBe("union"); + if (nested.k !== "union") return; + // The array closes on itself through the union of its element. + const array = nested.arms.find((a) => a.k === "array"); + const elem = array?.k === "array" ? array.elem : undefined; + expect(elem?.k === "union" && elem.arms.includes(array!)).toBe(true); + const chain = n.canon(aliases.get("Chain")!); + expect(chain.k).toBe("union"); + if (chain.k !== "union") return; + const tuple = chain.arms.find((a) => a.k === "tuple"); + const rest = tuple?.k === "tuple" ? tuple.elems[1]!.type : undefined; + expect(rest?.k === "union" && rest.arms.includes(tuple!)).toBe(true); + }); + + test("a failed comparison withdraws the pairs it assumed equal", () => { + const array = (p: "number" | "string"): Canon => ({ + k: "array", + elem: { k: "prim", p }, + readonly: false, + }); + // number[] fails against string[] before it matches number[]; that + // attempt must not make string[] match later. + const a: Canon = { k: "union", arms: [array("number"), array("number")] }; + const b: Canon = { k: "union", arms: [array("string"), array("number")] }; + expect(canonEqual(a, b)).toBe(false); + expect(canonEqual(b, a)).toBe(false); + }); + + test("instantiations of one generic class stay distinct", () => { + const maps = new Normalizer(checker).canon(aliases.get("Maps")!); + expect(maps.k === "union" && maps.arms.length).toBe(2); + expect(key(maps)).toBe("Map[] | Map[]"); + }); + + test("optional and rest tuple elements keep their flags", () => { + expect(key(new Normalizer(checker).canon(aliases.get("Loose")!))).toBe( + "[string, number | undefined?, ...boolean]", + ); + }); + + test("classes are nominal; arrays and tuples keep their elements", () => { + const n = new Normalizer(checker); + expect(key(n.canon(aliases.get("Points")!))).toBe("readonly Point[]"); + expect(key(n.canon(aliases.get("Pair")!))).toBe("[string, number]"); + }); +}); diff --git a/packages/compiler/test/constant-choice-emission.test.ts b/packages/compiler/test/constant-choice-emission.test.ts new file mode 100644 index 0000000000..bbc54001ae --- /dev/null +++ b/packages/compiler/test/constant-choice-emission.test.ts @@ -0,0 +1,64 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; + +test("a chain of literal stores is one store of a table entry, indexed as on every path", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-constant-choices-")); + try { + const entry = join(dir, "main.ts"); + await writeFile( + entry, + ` +function bases(arr: Uint8Array): void { + for (let i = 0; i < arr.length; ++i) { + const x = arr[i]; + if (x === 97) arr[i] = 0; + else if (x === 116) arr[i] = 1; + else if (x === 99) arr[i] = 2; + else arr[i] = 3; + } +} +function wide(flag: boolean): string { + const a = new Uint8Array(4); + let j = 0; + if (flag) j = 4294967297; + if (j === 0) a[j] = 11; + else if (j === 1) a[j] = 22; + else if (j === 2) a[j] = 33; + else a[j] = 44; + return a.join(","); +} +const sequence = new Uint8Array([97, 116, 99, 103]); +bases(sequence); +console.log(sequence.join(","), wide(true), wide(false)); +`, + ); + const outPath = join(dir, "main.ir.json"); + const lowered = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!lowered.ok) + throw new Error(lowered.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const ll = emitLlvmModule(deserializeModule(await readFile(outPath, "utf8"))); + const body = (name: string): string => + ll.match( + new RegExp(`define internal [^\\n]+ @sc_[bf]+_${name}\\([^\\n]*\\) #0 \\{[\\s\\S]*?\\n\\}`), + )![0]; + + // The cases' span, 97 to 116, with the final else's 3 between the cases. + expect(ll).toMatch( + /@sc_choice_\w+ = private unnamed_addr constant \[20 x i32\] \[i32 0, i32 3, i32 2, (i32 3, ){16}i32 1\]/, + ); + expect(body("bases")).toMatch(/load i32, ptr %\S+\n/); + + // The index is the compared local. Inside the first arm its range is + // [0, 0], but the one store also runs for 4294967297, so the index + // converts as the local's whole range allows (to i64), not to i32. + const wide = body("wide"); + expect(wide).toMatch(/@sc_choice_\w+, i64 0, i64/); + expect(wide).not.toMatch(/fptosi double %\S+ to i32/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/counted-loop-emission.test.ts b/packages/compiler/test/counted-loop-emission.test.ts index d7ebc97fad..e9a79a71ab 100644 --- a/packages/compiler/test/counted-loop-emission.test.ts +++ b/packages/compiler/test/counted-loop-emission.test.ts @@ -244,3 +244,102 @@ test("worker programs poll no loop whose proven ranges bound its runs inside ano await rm(dir, { recursive: true, force: true }); } }); + +test("worker programs read and write shared elements at a strip-mined loop's counter with plain accesses", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-counted-shared-")); + try { + const entry = join(dir, "shared.ts"); + await writeFile( + entry, + [ + 'import { isMainThread, Worker } from "node:worker_threads";', + "export function scale(a: Float64Array, b: Float64Array, n: number): void {", + " for (let i = 0; i < n; i++) b[i] = a[i]! * 2;", + "}", + "export function codes(a: Uint8Array): void {", + " for (let i = 0; i < a.length; ++i) a[i] = a[i] === 97 ? 0 : 3;", + "}", + "export function shift(a: Uint8Array): void {", + " for (let i = 0; i < a.length - 1; i++) a[i] = a[i + 1]!;", + "}", + "export function rows(a: Int32Array, n: number): number {", + " let t = 0;", + " for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) t += a[i]! + a[j]!;", + " return t;", + "}", + "export function waits(a: Int32Array, out: Uint8Array, n: number): number {", + " let spins = 0;", + " for (let i = 0; i < n; i++) {", + " out[i] = 1;", + " while (a[i] === 0) spins++;", + " do spins++; while (a[i] === 1);", + " for (let k = 0; a[i]! > k; k++) spins++;", + " for (let k = 0; k < 3; k += a[i]!) spins++;", + " }", + " return spins;", + "}", + "export function spin(a: Int32Array): number {", + " let n = 0;", + " while (a[0] === 0 && n < 10) n++;", + " return n;", + "}", + "const a = new Float64Array(new SharedArrayBuffer(64)).fill(1);", + "const b = new Float64Array(8);", + "scale(a, b, 8);", + "const c = new Uint8Array(new SharedArrayBuffer(8)).fill(97);", + "codes(c);", + "shift(c);", + "const d = new Int32Array(new SharedArrayBuffer(16)).fill(1);", + "console.log(b[7], c[0], rows(d, 4), spin(d), waits(new Int32Array(new SharedArrayBuffer(16)).fill(2), new Uint8Array(new SharedArrayBuffer(4)), 4));", + "if (!isMainThread) new Worker(new URL(import.meta.url));", + ].join("\n"), + ); + const outPath = join(dir, "shared.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const ll = emitLlvmModule(deserializeModule(await readFile(outPath, "utf8"))); + const bodies = (name: string): string => + [ + ...ll.matchAll( + new RegExp( + `define internal [^\\n]+ @sc_b?f_${name}(?:\\.i32)?\\([^\\n]*\\n([\\s\\S]*?)\\n\\}`, + "g", + ), + ), + ] + .map((match) => match[1]!) + .join("\n"); + // Element loads (the poll's flag read is atomic too, from a global). + const elementLoads = (body: string): number => + body.match(/load atomic \w+, ptr %/g)?.length ?? 0; + // Elements at the counter of a strip-mined loop: plain loads and stores. + for (const name of ["scale", "codes"]) { + const body = bodies(name); + expect(body).toMatch(/^loop\.poll\d+:$/m); + expect(elementLoads(body)).toBe(0); + expect(body).not.toContain("store atomic"); + } + // `a[i + 1]` is not at the counter: its read stays atomic, the store + // at the counter is plain. + const shift = bodies("shift"); + expect(shift).toMatch(/load atomic i8, ptr %/); + expect(shift).not.toContain("store atomic"); + // The outer counter read inside the inner loop reads one element on every + // inner iteration: it stays atomic; the inner counter's read is plain. + const rows = bodies("rows"); + expect(rows).toMatch(/load atomic i32, ptr %/); + // A loop that is not counted keeps atomic reads. + expect(bodies("spin")).toMatch(/load atomic i32, ptr %/); + // A nested loop that tests the outer counter's element in its condition + // or steps by it reads it again on every iteration: each read stays + // atomic, so another thread's write ends the wait. + const waits = bodies("waits"); + expect(waits).toMatch(/^loop\.poll\d+:$/m); + expect(waits.match(/load atomic i32, ptr %/g)?.length).toBeGreaterThanOrEqual(4); + // The outer loop's write at its counter stays plain (speculation counters + // are atomic stores to globals). + expect(waits).not.toMatch(/store atomic i8 \S+, ptr %/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/function-layout.test.ts b/packages/compiler/test/function-layout.test.ts new file mode 100644 index 0000000000..ecf4af7d9c --- /dev/null +++ b/packages/compiler/test/function-layout.test.ts @@ -0,0 +1,183 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { expect, test, vi } from "vitest"; +import { compile } from "../src/index.js"; +import { + functionLayoutKey, + layoutTierOf, + parseFunctionLayout, +} from "../src/backend/llvm/function-layout.js"; + +const source = ` +function first(n: number): number { return n + 1; } +function second(n: number): number { return n * 2; } +function third(n: number): number { return n - 3; } +console.log(first(Number(process.argv.length)), second(4), third(5)); +`; + +async function emitLlvm(layout?: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + let functionLayout: string | undefined; + if (layout !== undefined) { + functionLayout = join(dir, "layout.gz"); + await writeFile(functionLayout, gzipSync(layout)); + } + const result = await compile(entry, { + outDir: dir, + outPath, + outputKind: "llvm", + ...(functionLayout === undefined ? {} : { functionLayout }), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** Each defined function's name with the partition group line it follows. */ +function definitions(llvm: string): { name: string; group: string }[] { + const out: { name: string; group: string }[] = []; + let group = ""; + for (const line of llvm.split("\n")) { + if (line.startsWith("; scriptc-group")) group = line.slice("; scriptc-group".length).trim(); + const name = /^define [^@]*@([\w.$]+)\(/.exec(line)?.[1]; + if (name !== undefined) out.push({ name, group }); + } + return out; +} + +test("a layout writes the functions it lists first, by tier, in their tier's group", async () => { + const plain = definitions(await emitLlvm()).filter((d) => /_(first|second|third)$/.test(d.name)); + expect(plain.map((d) => d.name.replace(/^sc_b?f_/, ""))).toEqual(["first", "second", "third"]); + expect(plain.every((d) => !d.group.startsWith("layout-"))).toBe(true); + + const names = new Map(plain.map((d) => [d.name.replace(/^sc_b?f_/, ""), d.name])); + const laid = definitions(await emitLlvm(`2 ${names.get("first")}\n1 ${names.get("third")}\n`)); + const index = (fn: string) => laid.findIndex((d) => d.name === names.get(fn)); + expect(index("third")).toBeLessThan(index("first")); + expect(index("first")).toBeLessThan(index("second")); + expect(laid[index("third")]!.group).toBe("layout-1"); + expect(laid[index("first")]!.group).toBe("layout-2"); + expect(laid[index("second")]!.group).not.toMatch(/^layout-/); + // Listed functions precede every unlisted definition in the function section. + const firstUnlisted = laid.findIndex( + (d, i) => i > index("third") && !d.group.startsWith("layout-"), + ); + expect(firstUnlisted).toBeGreaterThan(index("first")); +}); + +test("layout keys leave out module numbers", () => { + expect(functionLayoutKey("sc_bf__x25_m447_lowerCall")).toBe( + functionLayoutKey("sc_bf__x25_m12_lowerCall"), + ); + expect(functionLayoutKey("sc_f__x25_init_253")).toBe(functionLayoutKey("sc_f__x25_init_7")); + expect(functionLayoutKey("sc_bf__x25_m3_Lexer_scan")).not.toBe( + functionLayoutKey("sc_bf__x25_m3_Lexer_peek"), + ); +}); + +test("a function listed twice keeps its first tier, and malformed lines are skipped", () => { + const layout = parseFunctionLayout("1 sc_f_a\n2 sc_f_a\n2 sc_f_b\nnot a line\n0 sc_f_c\n\n"); + expect([...layout]).toEqual([ + ["sc_f_a", 1], + ["sc_f_b", 2], + ]); + const text = + "define internal void @sc_f_b() {\n ret void\n}\n\ndefine void @sc_f_a() {\n ret void\n}"; + expect(layoutTierOf(layout, text)).toBe(1); + expect(layoutTierOf(layout, "define void @sc_f_z() {\n ret void\n}")).toBeUndefined(); +}); + +test("an executable built with a layout that lists helpers and functions runs unchanged", async () => { + const program = ` +class Point { constructor(public x: number, public y: number) {} } +function norm(p: Point): number { return Math.sqrt(p.x * p.x + p.y * p.y); } +const points = [new Point(3, 4), new Point(6, 8)]; +console.log(points.map(norm).join(","), points[0]!.x === 3 ? "three" : "other"); +`; + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-exe-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, program); + const llvmPath = join(dir, "main.ll"); + const emitted = await compile(entry, { outDir: dir, outPath: llvmPath, outputKind: "llvm" }); + expect(emitted.ok, JSON.stringify(emitted.diagnostics)).toBe(true); + // The generated helpers written before the functions, and one function, + // in two tiers. + const names = definitions(await readFile(llvmPath, "utf8")).map((d) => d.name); + const helpers = names.slice( + 0, + names.findIndex((name) => /^sc_b?f_/.test(name)), + ); + expect(helpers.length).toBeGreaterThan(0); + const norm = names.find((name) => /^sc_b?f_norm$/.test(name))!; + const layout = join(dir, "layout"); + await writeFile(layout, [`1 ${norm}`, ...helpers.map((name) => `2 ${name}`)].join("\n")); + const laid = join(dir, "laid.ll"); + const laidOut = await compile(entry, { + outDir: dir, + outPath: laid, + outputKind: "llvm", + functionLayout: layout, + }); + expect(laidOut.ok, JSON.stringify(laidOut.diagnostics)).toBe(true); + const order = definitions(await readFile(laid, "utf8")); + expect(order.filter((d) => d.group === "layout-2").map((d) => d.name)).toEqual(helpers); + expect(order.find((d) => d.name === norm)!.group).toBe("layout-1"); + const exe = join(dir, "main"); + const built = await compile(entry, { + outputKind: "exe", + outDir: dir, + outPath: exe, + functionLayout: layout, + }); + expect(built.ok, JSON.stringify(built.diagnostics)).toBe(true); + expect(execFileSync(exe, { encoding: "utf8" })).toBe("5,10 three\n"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("a changed function layout misses the executable cache", async () => { + vi.stubEnv("SCRIPTC_NO_CACHE", undefined); + const dir = await mkdtemp(join(tmpdir(), "scriptc-function-layout-cache-")); + try { + vi.stubEnv("SCRIPTC_CACHE_DIR", join(dir, "cache")); + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + const exe = join(dir, "main"); + const llvm = join(dir, "main.ll"); + const build = async (functionLayout?: string) => { + const built = await compile(entry, { + outputKind: "exe", + outDir: dir, + outPath: exe, + ...(functionLayout === undefined ? {} : { functionLayout }), + }); + expect(built.ok, JSON.stringify(built.diagnostics)).toBe(true); + return definitions(await readFile(llvm, "utf8")); + }; + const plain = await build(); + const third = plain.find((d) => /^sc_b?f_third$/.test(d.name))!.name; + const first = plain.find((d) => /^sc_b?f_first$/.test(d.name))!.name; + const layout = join(dir, "layout"); + await writeFile(layout, `1 ${third}\n`); + expect((await build(layout)).find((d) => d.name === third)!.group).toBe("layout-1"); + // The same path with other contents is another layout. + await writeFile(layout, `1 ${first}\n`); + const edited = await build(layout); + expect(edited.find((d) => d.name === first)!.group).toBe("layout-1"); + expect(edited.find((d) => d.name === third)!.group).not.toMatch(/^layout-/); + } finally { + vi.unstubAllEnvs(); + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/localized-failure-retries.test.ts b/packages/compiler/test/localized-failure-retries.test.ts new file mode 100644 index 0000000000..6bf1b1e23d --- /dev/null +++ b/packages/compiler/test/localized-failure-retries.test.ts @@ -0,0 +1,70 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function compileSource(source: string) { + const directory = mkdtempSync(join(tmpdir(), "scriptc-localized-")); + try { + const entry = join(directory, "main.ts"); + writeFileSync(entry, source); + return await compile(entry, { + outDir: directory, + outPath: join(directory, "program.ir"), + outputKind: "ir", + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +// A statement that lowers again after a later declarator degrades a type +// reuses the member bodies of a function-local class it lowered the first +// time, so the diagnostics of those bodies still report. +test("a degrading retry keeps the diagnostics of a local class lowered before it", async () => { + const result = await compileSource(`type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; +function make(): number { + const K = class { + m(): number { + const a = [1, 2]; + (a as any).foo = 1; + return a.length; + } + }, + size = (cb: Callbacks): number => (cb instanceof Set ? cb.size : 0); + return new K().m() + size(new Set([1])); +} +console.log(make()); +`); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected the local class's diagnostic"); + expect(result.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SC1090", + message: expect.stringContaining("assignment to non-variables"), + }), + ]), + ); +}); + +// Without the failing statement, the same program compiles: the arrow's +// parameter is a dynamic slot. +test("the same program without the local class's failure compiles", async () => { + const result = await compileSource(`type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; +function make(): number { + const K = class { + m(): number { + return 2; + } + }, + size = (cb: Callbacks): number => (cb instanceof Set ? cb.size : 0); + return new K().m() + size(new Set([1])); +} +console.log(make()); +`); + expect(result.ok, JSON.stringify(result.ok ? [] : result.diagnostics)).toBe(true); +}); diff --git a/packages/compiler/test/speculative-loop-emission.test.ts b/packages/compiler/test/speculative-loop-emission.test.ts index d26a0c96cd..dcec6d8af1 100644 --- a/packages/compiler/test/speculative-loop-emission.test.ts +++ b/packages/compiler/test/speculative-loop-emission.test.ts @@ -183,3 +183,47 @@ console.log(key(new Uint8Array([1, 2, 3, 0]), 6)); expect(fast).not.toMatch(/ashr i64 %/); expect(fast).not.toContain("@llvm.smul.with.overflow.i64"); }); + +test("a speculative loop reads its views' header fields once, before its copies", async () => { + // A view held by a const local or by a parameter is the same view in + // every iteration. Its data pointer is read before the branch to the fast + // copy, and no copy (ordinary, fast, or the one a failed check resumes + // in) reads it again. + const llvm = await emitLlvm(` +let last = 42; +function fill(array: ArrayBuffer, count: number): void { + const buf = new Float64Array(array); + let l = last; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + buf[i] = l; + } + last = l; +} +function fillInto(buf: Float64Array, count: number): void { + let l = last; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + buf[i] = l; + } + last = l; +} +const array = new ArrayBuffer(8 * 16); +fill(array, 16); +const view = new Float64Array(8); +fillInto(view, 8); +console.log(new Float64Array(array)[15], view[7], last); +`); + const dataReads = /getelementptr inbounds %ScrBytes, ptr %\w+, i64 0, i32 3$/gm; + for (const name of ["fill", "fillInto"]) { + const fn = + new RegExp(`^define internal void @sc_bf_${name}(?:\\.i32)?\\([^]*?^}`, "m").exec( + llvm, + )?.[0] ?? ""; + const branch = /^ {2}br i1 %\w+, label %spec\.fast\d+, label %spec\.slow\d+$/m.exec(fn); + expect(branch, name).not.toBeNull(); + const copies = fn.slice(branch!.index); + expect(copies.match(dataReads), name).toBeNull(); + expect(fn.slice(0, branch!.index).match(dataReads)?.length, name).toBeGreaterThan(0); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json new file mode 100644 index 0000000000..73b06cc549 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5025-lifetime-region-shared-candidate.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5025-lifetime-region-shared-candidate.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json new file mode 100644 index 0000000000..5e2082ab9a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8008-string-index-paths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8008-string-index-paths.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json new file mode 100644 index 0000000000..7754e8cb8f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8014-foreign-element-callbacks.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8014-foreign-element-callbacks.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json new file mode 100644 index 0000000000..57308033d0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8332-worker-shared-counted-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8332-worker-shared-counted-loops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json new file mode 100644 index 0000000000..059a21156b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/constant-choice-chains.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/constant-choice-chains.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json new file mode 100644 index 0000000000..e1643bcab1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-fresh.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-fresh.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json new file mode 100644 index 0000000000..e1a06ecc52 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-holders.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-holders.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json new file mode 100644 index 0000000000..e8ecb066e1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity-workers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity-workers.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json new file mode 100644 index 0000000000..fdf360d215 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-cast-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-cast-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json new file mode 100644 index 0000000000..41fde2c815 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-literal-identity.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-literal-identity.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json new file mode 100644 index 0000000000..c045eadf11 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/fs-readdir-recursive.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/fs-readdir-recursive.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json new file mode 100644 index 0000000000..7df4fd38f7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-cycles.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-cycles.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json new file mode 100644 index 0000000000..0530e8e135 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-declarations.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-declarations.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json new file mode 100644 index 0000000000..f775701663 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-forward-capture.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-forward-capture.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json new file mode 100644 index 0000000000..88890074df --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-function-slots.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-function-slots.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json new file mode 100644 index 0000000000..2327ce301e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-held-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-held-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json new file mode 100644 index 0000000000..9a45d0613b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-locals.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-locals.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json new file mode 100644 index 0000000000..d1603f0299 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-pi-shapes.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-pi-shapes.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json new file mode 100644 index 0000000000..b3d375ea63 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/localized-failure-registered-statics.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/localized-failure-registered-statics.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json new file mode 100644 index 0000000000..55b7b5d8ac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/never-narrowed-literal-argument.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/never-narrowed-literal-argument.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json new file mode 100644 index 0000000000..ca2d5289c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/rest-tuple-shorthand-property.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/rest-tuple-shorthand-property.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json new file mode 100644 index 0000000000..9fcddc195f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-loop-view-headers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/speculative-loop-view-headers.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json new file mode 100644 index 0000000000..1e78a18024 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/string-plus-element-read-return.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/string-plus-element-read-return.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json new file mode 100644 index 0000000000..e46feaab27 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/symbol-keyed-recursive-record.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/symbol-keyed-recursive-record.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json new file mode 100644 index 0000000000..252e5eb99d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/undeclared-key-optional-chain/main.cjs.json @@ -0,0 +1,18 @@ +{ + "order": [ + "/tests/corpus/undeclared-key-optional-chain/main.cjs" + ], + "diags": [ + { + "code": "SC1010", + "message": "the 'hostilekit' module is not supported yet", + "loc": { + "file": "/tests/corpus/undeclared-key-optional-chain/main.cjs", + "start": 289, + "end": 329 + }, + "milestone": "M4", + "hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported" + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json new file mode 100644 index 0000000000..a29a9d99a4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-arm.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-record-kept-fields-arm.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json new file mode 100644 index 0000000000..f40acc9978 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-kept-fields-no-discriminant.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-record-kept-fields-no-discriminant.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json new file mode 100644 index 0000000000..bf23de474a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-tuple-element-read.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-tuple-element-read.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json new file mode 100644 index 0000000000..26329147fb --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/var-object-filled-later.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/var-object-filled-later.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json new file mode 100644 index 0000000000..2732b9539c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/promise-all-generic-mixed-union.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/promise-all-generic-mixed-union.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json new file mode 100644 index 0000000000..1d39d436fd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/untyped-map-into-interface.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/untyped-map-into-interface.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json new file mode 100644 index 0000000000..5817aa0408 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-keyed-construction/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/types-keyed-construction/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json new file mode 100644 index 0000000000..8c56898f2a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/worker-shared-loop-version-emission.test.ts b/packages/compiler/test/worker-shared-loop-version-emission.test.ts index 7eebcc2e4f..ad7ec1ae97 100644 --- a/packages/compiler/test/worker-shared-loop-version-emission.test.ts +++ b/packages/compiler/test/worker-shared-loop-version-emission.test.ts @@ -56,11 +56,13 @@ if (isMainThread) { // The guard tests the bounds first, then whether the view is shared. expect(fn).toMatch(/^bytes\.version\.owned\d*:/m); expect(fn).toMatch(/^bytes\.version\.fast\.shared\d*:/m); - // The shared copy reads with no bounds test, and shared-safe: a view - // with shared storage loads atomically. + // The shared copy reads with no bounds test. Each copy is a counted loop + // that polls between blocks, so its read at the counter is a plain load + // (plainSharedElement in expr-bytes.ts). const shared = blocks(fn, "bytes.version.fast.shared"); expect(shared).not.toMatch(/icmp (ult|uge|ugt) i64 %\w+, %\w+/); - expect(fn).toMatch(/load atomic double, ptr %\w+ monotonic/); + expect(fn).toMatch(/load double, ptr %\w+/); + expect(fn).not.toMatch(/load atomic double/); }); test("a program without workers keeps two copies of the loop", async () => { diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index a27df378ac..087c926f77 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -2116,6 +2116,99 @@ void scr_heap_stats(double *total, double *used) { *used = (double)(held - (idle < held ? idle : held) + sys_used); } +/* scr_heap_trim: returns the heap pages that hold only freed memory to the + * system after a phase's data died (scr_collect_cycles_phase). The + * size-class slices keep freed blocks on their class's free list, which only + * later blocks of the same class reuse, so a program whose next phase + * allocates other sizes would otherwise grow its resident memory on top of + * the freed heap. */ +#if defined(__GLIBC__) && !defined(SCR_HEAP_SANITIZER) +#include +#endif +#if SCR_SMALL_ALLOC && !defined(SCR_WORKERS) && defined(__linux__) && defined(MADV_DONTNEED) +#include +#include + +/* Returns the pages of class c's slice below its frontier that hold only + * free blocks (MADV_DONTNEED), and relinks the class's other free blocks in + * address order. A block wholly inside a returned page leaves the free list: + * the class carves new blocks past its frontier instead, and the slice's + * reservation is far larger than what a program fills. */ +static void scr_sa_return_free_pages(unsigned c, size_t page) { + size_t sz = scr_sa_class_size(c); + char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift); + char *top = scr_sa.bump[c]; + size_t nblocks = (size_t)(top - start) / sz; + if (!nblocks || !scr_sa.free[c]) return; + size_t words = (nblocks + 63) / 64; + uint64_t *bits = mmap(NULL, words * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (bits == MAP_FAILED) return; + for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) { + size_t k = (size_t)((char *)b - start) / sz; + bits[k / 64] |= (uint64_t)1 << (k % 64); + } +#define SCR_SA_FREE_BIT(k) ((bits[(k) / 64] >> ((k) % 64)) & 1) + /* Mark the returned pages' blocks by clearing their bits after the + * madvise; a page is returned when every block overlapping it is free. */ + uintptr_t first = ((uintptr_t)start + page - 1) & ~(uintptr_t)(page - 1); + uintptr_t last = (uintptr_t)top & ~(uintptr_t)(page - 1); + char *run = NULL; + for (uintptr_t pg = first; pg <= last; pg += page) { + bool empty = false; + if (pg < last) { + size_t k0 = (size_t)((char *)pg - start) / sz; + size_t k1 = (size_t)((char *)pg + page - 1 - start) / sz; + if (k1 >= nblocks) k1 = nblocks - 1; + empty = true; + for (size_t k = k0; k <= k1 && empty; k++) empty = SCR_SA_FREE_BIT(k); + } + if (empty && !run) run = (char *)pg; + if (!empty && run) { + (void)madvise(run, (size_t)((char *)pg - run), MADV_DONTNEED); + /* Blocks wholly inside the returned pages are dropped. */ + size_t k0 = ((size_t)(run - start) + sz - 1) / sz; + size_t k1 = (size_t)((char *)pg - start) / sz; + for (size_t k = k0; k < k1 && (char *)start + (k + 1) * sz <= (char *)pg; k++) + bits[k / 64] &= ~((uint64_t)1 << (k % 64)); + run = NULL; + } + } + /* The kept free blocks, in address order. */ + ScrSaBlock *head = NULL, **tail = &head; + for (size_t w = 0; w < words; w++) + for (uint64_t m = bits[w]; m; m &= m - 1) { + ScrSaBlock *b = (ScrSaBlock *)(start + (w * 64 + (size_t)__builtin_ctzll(m)) * sz); + *tail = b; + tail = &b->next; + } + *tail = NULL; + scr_sa.free[c] = head; +#undef SCR_SA_FREE_BIT + (void)munmap(bits, words * 8); +} +#endif + +void scr_heap_trim(void) { +#ifdef SCR_SA_TRIM + /* A worker runtime returns the calling thread's arena pages as a parking + * thread does; the arena stays the thread's own. */ + if (scr_sa_arena >= 0 && scr_sa_trim_worth()) { + scr_sa_trim(); + return; + } +#elif SCR_SMALL_ALLOC && !defined(SCR_WORKERS) && defined(__linux__) && defined(MADV_DONTNEED) + if (scr_sa.span) { + long page = sysconf(_SC_PAGESIZE); + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) + if (scr_sa.lim[c]) scr_sa_return_free_pages(c, page > 0 ? (size_t)page : 4096); + } +#endif +#if defined(__GLIBC__) && !defined(SCR_HEAP_SANITIZER) + /* Large blocks: the system allocator's free pages too. */ + (void)malloc_trim(0); +#endif +} + /* ── program-lifetime region ──────────────────────────────────────────── * Instances of compact classes (llvm/classes.ts LlClassMeta.compact) in * programs built for Linux without sanitizers (runtimes with the size-class diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 0d9df7106f..d421e8e45a 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -1474,7 +1474,10 @@ static void scr_scan_black(void *obj, uintptr_t depth) { const ScrCycType *type = scr_cyc_type(h); bool acyclic = type->trace == scr_cyc_no_children; if (!scr_cyc_is_buffered(h)) h->buf = acyclic ? SCR_CYC_ACYCLIC : 0; - if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_push_work(&scr_promote, obj); + /* Such a candidate also keeps its generation: the generation names the + * buffer its slot is in (scr_cyc_on_dead), so it is promoted only once a + * pass takes it. */ + if (scr_cyc_gen(h) < SCR_CYC_OLD && !scr_cyc_is_buffered(h)) scr_cyc_push_work(&scr_promote, obj); if (acyclic) return; if (depth == SCR_CYC_WALK_DEPTH) { scr_cyc_push_work(&scr_restore_pending, obj); @@ -2350,7 +2353,9 @@ void scr_collect_cycles(void) { * finding anything, and the death rule's backoff then start over from the * heap the sweep leaves, as after a productive pass. Otherwise the next * phase's garbage would accumulate until the heap grew to that widened - * fraction. */ + * fraction. The freed memory then goes back to the system (scr_heap_trim): + * the next phase allocates other sizes than the dead data held, so it would + * reuse little of it and grow its resident memory on top. */ void scr_collect_cycles_phase(void) { scr_collect_cycles(); for (unsigned gen = SCR_CYC_MATURE; gen < SCR_CYC_NGENS; gen++) { @@ -2358,6 +2363,7 @@ void scr_collect_cycles_phase(void) { scr_cyc_growth_streak[gen] = 0; } scr_cyc_death_shift = 0; + scr_heap_trim(); } /* Worker TLS becomes unreachable when its OS thread returns. Release the diff --git a/packages/runtime/src/scr_decl_adopt.c b/packages/runtime/src/scr_decl_adopt.c index 7372826aba..fc1c4456c9 100644 --- a/packages/runtime/src/scr_decl_adopt.c +++ b/packages/runtime/src/scr_decl_adopt.c @@ -15,7 +15,11 @@ * value has no prototype, accessors, symbol keys or frozen state) and * collects the shared ones; it throws the path TypeError before anything * changes. Phase two converts the shared nodes in place - * (scr_dyn_decl_become, scr_json.c). */ + * (scr_dyn_decl_become, scr_json.c). + * + * Every other checked cast of a dynamic value to a record or array runs the + * same pass leniently (scr_dyn_adopt_shared): a node that cannot become a + * capsule keeps the cast's copy. */ #include "scr_runtime.h" #include @@ -31,6 +35,9 @@ typedef struct { typedef struct { ScrDeclAdopt *items; size_t len, cap; + /* A checked cast outside the boundary (scr_dyn_adopt_shared): a node + * that cannot become a capsule keeps the copy instead of throwing. */ + bool lenient; } ScrDeclAdoptList; static void scr_decl_adopt_push(ScrDeclAdoptList *list, ScrDyn *node, ScrDyn *capsule) { @@ -44,7 +51,7 @@ static void scr_decl_adopt_push(ScrDeclAdoptList *list, ScrDyn *node, ScrDyn *ca list->len++; } -static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path) { +static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path, bool lenient) { const char *want = NULL; if (n->kind == SCR_DYN_OBJ) { if (n->prototype || n->null_proto || n->v.obj.class_name) @@ -72,7 +79,7 @@ static bool scr_decl_adopt_plain(const ScrDyn *n, const ScrDynPath *path) { want = "a plain array (this value is frozen, sealed or not extensible)"; } if (!want) return true; - scr_dyn_check_fail(path, want, n); + if (!lenient) scr_dyn_check_fail(path, want, n); return false; } @@ -82,6 +89,14 @@ static ScrDyn *scr_decl_capsule_ptr_holder(const ScrDeclAdoptList *list, const S return NULL; } +/* Whether the walk follows a child: plain objects and arrays. A boxed + * native record or array is already one object with its native value; a + * cast to another layout copies it (the copy keeps a cached identity per + * capsule, scr_dyn_typed_ref_cache_cast). */ +static bool scr_decl_adopt_follows(const ScrDyn *child) { + return child && (child->kind == SCR_DYN_OBJ || child->kind == SCR_DYN_ARR); +} + /* `capsule` is owned by the call: pushed or released. */ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsule, const ScrDynPath *path, unsigned depth) { @@ -90,15 +105,16 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu scr_dyn_release(capsule); return true; } - if (!scr_decl_adopt_plain(n, path)) { + if (!scr_decl_adopt_plain(n, path, list->lenient)) { scr_dyn_release(capsule); - return false; + return list->lenient; } if (n->rc > 1 && n->rc != SIZE_MAX) { ScrDyn *seen = scr_decl_capsule_ptr_holder(list, n); if (seen) { bool same = seen->v.typed_ref.ptr == capsule->v.typed_ref.ptr; scr_dyn_release(capsule); + if (!same && list->lenient) return true; if (!same) scr_dyn_check_fail(path, "a value that is not also stored elsewhere in this value (scriptc builds a separate native copy for each place)", n); return same; @@ -112,7 +128,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu if (!ops->has(ptr, i)) continue; const ScrStr *name = ops->names[i]; ScrDyn *child = scr_dyn_obj_get(n, name->data, name->len); - if (!child || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, name->data, 0 }; if (!scr_decl_adopt_walk(list, child, ops->get(ptr, i), &p, depth + 1)) { if (!(n->rc > 1 && n->rc != SIZE_MAX)) scr_dyn_release(capsule); @@ -135,7 +151,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu size_t len = n->v.arr.len < view->v.arr.len ? n->v.arr.len : view->v.arr.len; for (size_t i = 0; i < len; i++) { ScrDyn *child = n->v.arr.items[i], *element = view->v.arr.items[i]; - if (!child || !element || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!element || !scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, NULL, i }; if (!scr_decl_adopt_walk(list, child, scr_dyn_retain(element), &p, depth + 1)) { scr_dyn_release(view); @@ -150,7 +166,7 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu const ScrDynTypedArrayOps *ops = capsule->v.typed_ref.array; for (size_t i = 0; i < n->v.arr.len; i++) { ScrDyn *child = n->v.arr.items[i]; - if (!child || (child->kind != SCR_DYN_OBJ && child->kind != SCR_DYN_ARR)) continue; + if (!scr_decl_adopt_follows(child)) continue; ScrDynPath p = { path, NULL, i }; if (!scr_decl_adopt_walk(list, child, ops->get(ptr, (double)i), &p, depth + 1)) { if (!(n->rc > 1 && n->rc != SIZE_MAX)) scr_dyn_release(capsule); @@ -162,8 +178,8 @@ static bool scr_decl_adopt_walk(ScrDeclAdoptList *list, ScrDyn *n, ScrDyn *capsu return true; } -void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { - ScrDeclAdoptList list = { NULL, 0, 0 }; +static void scr_decl_adopt_run(ScrDyn *source, ScrDyn *capsule, bool lenient) { + ScrDeclAdoptList list = { NULL, 0, 0, lenient }; bool ok = scr_decl_adopt_walk(&list, source, scr_dyn_retain(capsule), NULL, 0); for (size_t i = 0; i < list.len; i++) { ScrDeclAdopt *item = &list.items[i]; @@ -172,3 +188,16 @@ void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { } free(list.items); } + +void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule) { + scr_decl_adopt_run(source, capsule, false); +} + +/* Any other checked cast of a dynamic value to a record or array + * (`value as T`): the same adoption, so a source that something else still + * holds keeps one identity with the built value. A node that is not plain + * data, or that the value holds in two places, keeps the separate copy the + * cast built (the conversion already succeeded; nothing throws here). */ +void scr_dyn_adopt_shared(ScrDyn *source, ScrDyn *capsule) { + scr_decl_adopt_run(source, capsule, true); +} diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index b034602df7..11c86a69b4 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -393,6 +393,7 @@ const ScrDynTypedRecordOps *scr_dyn_typed_ref_live_record(const ScrDyn *d); * holds. Both borrowed; a node that is not plain data throws the path * TypeError before anything changes. */ void scr_dyn_decl_adopt(ScrDyn *source, ScrDyn *capsule); +void scr_dyn_adopt_shared(ScrDyn *source, ScrDyn *capsule); /* A checked conversion into a type an npm package declares failed * (scr_type_origin.c, linked by programs with such checks): the pending * TypeError's message gains the declaration and the type patch that would diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index accc9c9e82..283c212344 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -2564,7 +2564,7 @@ static void scr_dyn_record_view_freeze(ScrDyn *view) { void scr_record_freeze(void *record) { size_t *rc = record; - if (*rc != SIZE_MAX) *rc |= SCR_RC_FROZEN; + if (!scr_rc_immortal(record)) *rc |= SCR_RC_FROZEN; } void scr_record_throw_frozen(ScrStr *key, ScrStr *owner, double operation) { diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 4cfda127be..928400da3c 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -6344,6 +6344,56 @@ ScrArr *scr_fs_readdir(ScrStr *path) { return arr; } +/* fs.readdirSync(path, { recursive: true }): Node's readdirSyncRecursive. + * Directories are listed in queue order: the base, then each directory in + * the order its entry was listed. Each entry's path is path.join of its + * directory and name; the result holds it relative to the base + * (path.relative). An entry stat(2) reports as a directory, a symlink to + * one included, joins the queue. */ +ScrArr *scr_fs_readdir_recursive(ScrStr *base) { + if (!scr_fs_path_ok(base, "path")) return NULL; + ScrArr *out = scr_arr_new(SCR_ELEM_STR, 0); + size_t len = 1, cap = 8; + ScrStr **queue = malloc(cap * sizeof *queue); + if (!queue) scr_trap("scriptc: out of memory\n"); + queue[0] = scr_str_retain(base); + bool failed = false; + for (size_t at = 0; at < len && !failed; at++) { + ScrScandir *scan = scr_fs_scandir(queue[at]); + if (!scan) { + failed = true; + break; + } + for (size_t i = 0; i < scr_fs_scandir_count(scan); i++) { + ScrArr *parts = scr_arr_new(SCR_ELEM_STR, 2); + scr_arr_push_ref(parts, scr_str_retain(queue[at])); + scr_arr_push_ref(parts, scr_fs_scandir_name(scan, i)); + ScrStr *full = scr_path_join(parts); + scr_arr_release(parts); + scr_arr_push_ref(out, scr_path_relative(base, full)); + struct stat st; + if (stat(full->data, &st) == 0 && S_ISDIR(st.st_mode)) { + if (len == cap) { + cap *= 2; + queue = realloc(queue, cap * sizeof *queue); + if (!queue) scr_trap("scriptc: out of memory\n"); + } + queue[len++] = full; + } else { + scr_str_release(full); + } + } + scr_fs_scandir_free(scan); + } + for (size_t i = 0; i < len; i++) scr_str_release(queue[i]); + free(queue); + if (failed) { + scr_arr_release(out); + return NULL; + } + return out; +} + /* ── the withFileTypes scandir snapshot ────────────────────────────── * One readdir pass capturing name + entry kind for the emitter's Dirent * assembly (scr_runtime.h has the contract). Kinds are libuv's UV_DIRENT diff --git a/packages/runtime/src/scr_lib.h b/packages/runtime/src/scr_lib.h index 7ba9e2907b..e01bc7b236 100644 --- a/packages/runtime/src/scr_lib.h +++ b/packages/runtime/src/scr_lib.h @@ -293,6 +293,7 @@ void scr_fs_mkdir(ScrStr *path); void scr_fs_rm(ScrStr *path); void scr_fs_rmdir(ScrStr *path); ScrArr *scr_fs_readdir(ScrStr *path); /* names (byte order on POSIX, as libuv sorts), no "."/".." */ +ScrArr *scr_fs_readdir_recursive(ScrStr *path); /* relative paths of every entry below path, Node's queue order */ /* The fs option forms (same throw discipline). mkdir_recursive is Node's * recursive algorithm (existing-dir target is a no-op; errors report diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index afbe4d6af5..8d11d3806e 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -128,8 +128,10 @@ bool scr_pub_mark(ScrPub *ctx, void *obj, bool cyc, int kind) { size_t *rc = obj; if (SCR_RC_PUBLISHED(obj) || ctx->refusal) return false; bool compact = (kind & SCR_PUB_COMPACT) != 0; - /* Both log the low 32 bits: a count, or a compact header word. */ - bool class_id = compact || (kind & SCR_PUB_CLASS_ID) != 0; + /* Both log the low 32 bits: a count, or a compact header word. An + * object with a layout id in its rc word (scr_rc_layout_id) keeps it, as + * a class instance keeps its class id. */ + bool class_id = compact || (kind & SCR_PUB_CLASS_ID) != 0 || scr_rc_layout_id(obj) != 0; kind &= ~(SCR_PUB_CLASS_ID | SCR_PUB_COMPACT); if (ctx->logged == ctx->log_cap) ctx->log = scr_pub_grow(ctx->log, &ctx->log_cap, sizeof *ctx->log, ctx->log_inline); diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 171e420c20..5a705054a5 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -8,13 +8,27 @@ /* ── reference counts ───────────────────────────────────────────────── * Every heap object's first member is its `size_t rc` word. On 64-bit - * targets the reference count is the word's low 32 bits: an emitted - * instance of a class in a program-rooted hierarchy keeps its class id (its - * class's preorder number + 1) in the high 32 bits (llvm/classes.ts - * classIdRcWord), and no other object's count reaches 2^32. A count of SCR_RC_IMMORTAL marks - * an immortal object; for every object but those class instances the whole - * word is then SIZE_MAX. Code that reads or writes the count of an object of - * any kind goes through SCR_RC_COUNT. */ + * targets the word is + * + * bits 0-31 the reference count (SCR_RC_COUNT) + * bit 62 F: a frozen record (SCR_RC_FROZEN) + * bit 63 P: a program-lifetime record (llvm/classes.ts + * programLifetimeRc) + * bits 32-61 the layout id (scr_rc_layout_id) + * + * and no object's count reaches 2^32. The layout id names the object's + * layout in the single object model (scr_layout.h); 0 means the object + * never needed one. An emitted instance of a class in a program-rooted + * hierarchy keeps its class id (its class's preorder number + 1) there + * (llvm/classes.ts classIdRcWord): a class id is the layout id of its + * class's canonical layout. A count of SCR_RC_IMMORTAL marks an immortal + * object. Publication and the runtime's own immortal objects write the + * whole word as SIZE_MAX when the object has no layout id, and write only + * the count when it has one, so an immortal object keeps its layout id + * (scr_rc_immortal tests the count; a high half of all ones reads as no + * layout id). Code that reads or writes the count of an object of any + * kind goes through SCR_RC_COUNT; code that may see an object with a layout + * id tests immortality with scr_rc_immortal, never against SIZE_MAX. */ #if SIZE_MAX > UINT32_MAX typedef uint32_t ScrRcCount; #ifdef __BYTE_ORDER__ @@ -58,25 +72,47 @@ typedef size_t ScrRcCount; #define SCR_RC_FROZEN ((size_t)1 << 30) #define SCR_RC_REFS(obj) (SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL ? SCR_RC_IMMORTAL : SCR_RC_COUNT(obj) & ~SCR_RC_FROZEN) #endif -static inline bool scr_rc_frozen(const void *obj) { - size_t word = *(const size_t *)obj; - return word != SIZE_MAX && (word & SCR_RC_FROZEN) != 0; +/* Whether `obj`, an object whose rc word is a count (not a compact class + * instance), is immortal: its count is SCR_RC_IMMORTAL, whatever its high + * half holds. */ +static inline bool scr_rc_immortal(const void *obj) { + return SCR_RC_COUNT(obj) == SCR_RC_IMMORTAL; } -/* The class id in the high half of `obj`'s rc word, or 0 for an object - * without one: the high half of any other object's word is 0, 0xffffffff - * (immortal), 0x80000000 (program-lifetime) or 0x40000000 (a frozen record), - * and a class id is never one of these. */ -static inline uint32_t scr_rc_class_id(const void *obj) { +static inline bool scr_rc_frozen(const void *obj) { + return !scr_rc_immortal(obj) && (*(const size_t *)obj & SCR_RC_FROZEN) != 0; +} + +/* The high half of the rc word on 64-bit targets: the program-lifetime + * bit, the frozen bit and the layout id (see the layout above). 32-bit + * targets have no high half; their objects keep a layout id elsewhere when + * they need one (scr_layout.h). */ +#define SCR_RC_HIGH_LIFETIME UINT32_C(0x80000000) +#define SCR_RC_HIGH_FROZEN UINT32_C(0x40000000) +#define SCR_RC_LAYOUT_ID_MASK UINT32_C(0x3fffffff) +/* Layout ids are below this bound: 2^30 - 1 is the id bits of an all-ones + * high half, which reads as no id. */ +#define SCR_RC_LAYOUT_ID_LIMIT SCR_RC_LAYOUT_ID_MASK + +/* The layout id in the high half of `obj`'s rc word, or 0 for an object + * without one. The whole-word immortal encoding (SIZE_MAX) has no id. */ +static inline uint32_t scr_rc_layout_id(const void *obj) { #if SIZE_MAX > UINT32_MAX uint32_t high = (uint32_t)(*(const size_t *)obj >> 32); - return high == UINT32_MAX || high == UINT32_C(0x80000000) || high == UINT32_C(0x40000000) ? 0 : high; + return high == UINT32_MAX ? 0 : high & SCR_RC_LAYOUT_ID_MASK; #else (void)obj; return 0; #endif } +/* The class id in the high half of `obj`'s rc word, or 0 for an object + * without one. A class id is the layout id of the class's canonical + * layout, and no object of any other kind has a layout id yet. */ +static inline uint32_t scr_rc_class_id(const void *obj) { + return scr_rc_layout_id(obj); +} + /* ── cycle collection (scr_cycle.c) ─────────────────────────────────── * Reference counting alone cannot free cycles, so every object that can * participate in one — capture boxes, heap closures, unions, promises, and @@ -696,6 +732,10 @@ static inline void scr_cyc_dead(void *obj) { * live heap — do not put it on a per-turn path. */ void scr_collect_cycles(void); void scr_collect_cycles_phase(void); +/* Returns the heap pages that hold only freed memory to the system: the + * calling thread's size-class pages whose blocks are all free, and the + * system allocator's free pages (scr_alloc.c). */ +void scr_heap_trim(void); /* Sets the growth cap SCR_CYCLE_GROWTH_CAP sets (scr_cycle.c), for a program * that knows its own memory profile; the environment variable takes * precedence. Call it before the program allocates much. */ diff --git a/packages/runtime/src/scr_rc.test.c b/packages/runtime/src/scr_rc.test.c new file mode 100644 index 0000000000..159235ce95 --- /dev/null +++ b/packages/runtime/src/scr_rc.test.c @@ -0,0 +1,47 @@ +#undef NDEBUG +#include "scr_rc.h" +#include +#include + +/* The rc word's encodings (scr_rc.h): the count in the low half, and the + * program-lifetime bit, the frozen bit and the layout id in the high half. + * Immortality is a count of all ones; a layout id survives it. */ +static size_t word(uint32_t high, uint32_t count) { + return ((size_t)high << 32) | count; +} + +int main(void) { +#if SIZE_MAX > UINT32_MAX + /* An ordinary object: no layout id, not frozen, not immortal. */ + size_t plain = word(0, 3); + assert(scr_rc_layout_id(&plain) == 0 && !scr_rc_frozen(&plain) && !scr_rc_immortal(&plain)); + /* The whole-word immortal encoding has no layout id and is not frozen. */ + size_t immortal = SIZE_MAX; + assert(scr_rc_layout_id(&immortal) == 0 && scr_rc_immortal(&immortal) && !scr_rc_frozen(&immortal)); + /* A program-lifetime record (llvm/classes.ts programLifetimeRc) and a + * frozen record have no layout id. */ + size_t lifetime = word(SCR_RC_HIGH_LIFETIME, 0); + assert(scr_rc_layout_id(&lifetime) == 0 && !scr_rc_immortal(&lifetime)); + size_t frozen = word(SCR_RC_HIGH_FROZEN, 2); + assert(scr_rc_layout_id(&frozen) == 0 && scr_rc_frozen(&frozen)); + /* A class instance keeps its class id, which is its layout id, beside + * its count, also once immortal, frozen or program-lifetime. */ + size_t instance = word(7, 1); + assert(scr_rc_layout_id(&instance) == 7 && scr_rc_class_id(&instance) == 7); + SCR_RC_COUNT(&instance) = SCR_RC_IMMORTAL; + assert(scr_rc_immortal(&instance) && scr_rc_layout_id(&instance) == 7 && !scr_rc_frozen(&instance)); + size_t frozen_id = word(SCR_RC_HIGH_FROZEN | 9, 1); + assert(scr_rc_layout_id(&frozen_id) == 9 && scr_rc_frozen(&frozen_id)); + size_t lifetime_id = word(SCR_RC_HIGH_LIFETIME | 11, UINT32_C(0x80000000)); + assert(scr_rc_layout_id(&lifetime_id) == 11 && !scr_rc_immortal(&lifetime_id)); + /* The largest layout id. */ + size_t top = word(SCR_RC_LAYOUT_ID_LIMIT - 1, 1); + assert(scr_rc_layout_id(&top) == SCR_RC_LAYOUT_ID_LIMIT - 1); +#else + size_t plain = 3, immortal = SIZE_MAX; + assert(scr_rc_layout_id(&plain) == 0 && !scr_rc_immortal(&plain)); + assert(scr_rc_immortal(&immortal) && !scr_rc_frozen(&immortal)); +#endif + puts("rc word encodings"); + return 0; +} diff --git a/packages/runtime/src/scr_rc.test.ts b/packages/runtime/src/scr_rc.test.ts new file mode 100644 index 0000000000..181d0bebac --- /dev/null +++ b/packages/runtime/src/scr_rc.test.ts @@ -0,0 +1,36 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const exec = promisify(execFile); + +/* The rc word helpers are inline in scr_rc.h, so the test links alone. */ +test.skipIf(process.platform === "win32")( + "the rc word keeps the count, the lifetime and frozen bits and the layout id apart", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-rc-")); + try { + const bin = join(dir, "rc"); + await exec("clang", [ + "-std=c11", + "-O1", + "-g", + "-fsanitize=address,undefined", + join(import.meta.dirname, "scr_rc.test.c"), + "-o", + bin, + ]); + const result = await exec(bin, [], { + timeout: 60000, + env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, + }); + expect(result.stdout).toBe("rc word encodings\n"); + expect(result.stderr).toBe(""); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +); diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 37f94bd656..4296ce791e 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -2469,6 +2469,146 @@ static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len, return (int)count; } +/* Normalization quick check (UAX #15): a string whose every code point is + * inert for the form is already in that form. A code point is inert when + * normalizing it alone leaves it unchanged, its canonical combining class + * is 0, and (for the composing forms) no composition can take it as its + * second element. The last condition holds in the ranges below: none holds + * a code point with combining class 0 that composes with a preceding one + * (those are Hangul jamo vowels and trailing consonants, U+3099/U+309A, + * and vowel signs of Indic and Southeast Asian scripts above U+0900), so a + * code point outside them takes the full algorithm. Inert flags are + * computed through unicode_normalize, 256 code points at a time, when a + * block is first seen. */ +#define SCR_NQC_BLOCKS 0x200 +typedef struct { + uint8_t known[4][SCR_NQC_BLOCKS]; + uint8_t inert[4][SCR_NQC_BLOCKS][32]; + uint8_t starter[4][SCR_NQC_BLOCKS][32]; /* class 0, never a second element */ + struct { + uint32_t cp; + uint8_t len; + char bytes[15]; + } single[4][256]; /* a lone code point's normalized UTF-8 */ +} ScrNormalizeQuick; +static SCR_TL ScrNormalizeQuick *scr_nqc; + +static void scr_nqc_free(void) { + free(scr_nqc); + scr_nqc = NULL; +} + +static bool scr_nqc_in_ranges(uint32_t c) { + return c < 0x900 || (c >= 0x2000 && c < 0x3099) || (c >= 0x309B && c < 0xA000) || + (c >= 0xAC00 && c < 0xD7A4) || (c >= 0xF900 && c < 0x10000) || + (c >= 0x1F000 && c < 0x1FB00); +} + +/* Bit 0: inert. Bit 1: a starter, with class 0 and never the second element + * of a composition, so nothing before it reorders or composes with it. */ +/* Whether c has class 0 and lies in the ranges above. U+0345 is the one + * code point of class 240, which the reordering test cannot tell from + * class 0; a mark of class 1..239 after U+0345 moves before it. */ +static bool scr_nqc_class0(uint32_t c) { + if (c == 0x345 || !scr_nqc_in_ranges(c)) return false; + uint32_t pair[2] = {0x345, c}; + uint32_t *out = NULL; + int n = unicode_normalize(&out, pair, 2, UNICODE_NFD, NULL, scr_normalize_realloc); + bool starter = n >= 2 && out[0] == 0x345; + free(out); + return starter; +} + +static unsigned scr_nqc_compute(uint32_t c, UnicodeNormalizationEnum mode) { + if (!scr_nqc_class0(c)) return 0; + uint32_t *out = NULL; + int n = unicode_normalize(&out, &c, 1, mode, NULL, scr_normalize_realloc); + bool same = n == 1 && out[0] == c; + free(out); + if (same) return 3u; + /* A starter's decomposition must begin with one too (U+FF9E's + * compatibility decomposition is U+3099, which composes with the code + * point before it). */ + UnicodeNormalizationEnum decompose = + mode == UNICODE_NFKC || mode == UNICODE_NFKD ? UNICODE_NFKD : UNICODE_NFD; + out = NULL; + n = unicode_normalize(&out, &c, 1, decompose, NULL, scr_normalize_realloc); + bool leads = n >= 1 && scr_nqc_class0(out[0]); + free(out); + return leads ? 2u : 0u; +} + +/* scr_nqc_compute's bits for c, from its block's flags. */ +static unsigned scr_nqc_class(uint32_t c, UnicodeNormalizationEnum mode) { + if (c >= (uint32_t)SCR_NQC_BLOCKS << 8) return 0; + if (!scr_nqc) { + scr_nqc = calloc(1, sizeof *scr_nqc); + if (!scr_nqc) return 0; + scr_atexit(scr_nqc_free); + } + uint32_t block = c >> 8; + uint8_t *inert = scr_nqc->inert[mode][block]; + uint8_t *starter = scr_nqc->starter[mode][block]; + if (!scr_nqc->known[mode][block]) { + for (uint32_t i = 0; i < 256; i++) { + unsigned bits = scr_nqc_compute((block << 8) | i, mode); + if (bits & 1) inert[i >> 3] |= (uint8_t)(1u << (i & 7)); + if (bits & 2) starter[i >> 3] |= (uint8_t)(1u << (i & 7)); + } + scr_nqc->known[mode][block] = 1; + } + unsigned bit = c & 0xff; + return ((inert[bit >> 3] >> (bit & 7)) & 1) | (((starter[bit >> 3] >> (bit & 7)) & 1) << 1); +} + +/* Normalize the code points of data[from, to) and append them. */ +static void scr_nqc_segment(ScrJsonBuf *b, const char *data, size_t from, size_t to, + UnicodeNormalizationEnum mode) { + size_t first_step; + uint32_t first = scr_case_decode(data + from, &first_step); + if (first_step == to - from && scr_nqc) { + /* One code point (a precomposed letter between inert ones): its + * normalized bytes repeat, so they are kept per form. */ + __typeof__(scr_nqc->single[0][0]) *slot = &scr_nqc->single[mode][first & 0xff]; + if (slot->cp != first || slot->len == 0) { + uint32_t *output = NULL; + int count = unicode_normalize(&output, &first, 1, mode, NULL, scr_normalize_realloc); + if (count < 0) scr_regex_oom(); + ScrJsonBuf one; + scr_jb_init(&one); + for (int i = 0; i < count; i++) scr_case_put_cp(&one, output[i]); + free(output); + ScrStr *bytes = scr_jb_finish(&one); + if (bytes->len == 0 || bytes->len > sizeof slot->bytes) { + scr_jb_write(b, bytes->data, bytes->len); + scr_str_release(bytes); + return; + } + slot->cp = first; + slot->len = (uint8_t)bytes->len; + memcpy(slot->bytes, bytes->data, bytes->len); + scr_str_release(bytes); + } + scr_jb_write(b, slot->bytes, slot->len); + return; + } + uint32_t local[32]; + uint32_t *input = to - from <= 32 ? local : malloc((to - from) * sizeof *input); + if (!input) scr_regex_oom(); + int length = 0; + for (size_t offset = from; offset < to;) { + size_t step; + input[length++] = scr_case_decode(data + offset, &step); + offset += step; + } + uint32_t *output = NULL; + int count = scr_unicode_normalize(&output, input, length, mode); + if (input != local) free(input); + if (count < 0) scr_regex_oom(); + for (int i = 0; i < count; i++) scr_case_put_cp(b, output[i]); + free(output); +} + ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { UnicodeNormalizationEnum mode; if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC; @@ -2483,23 +2623,58 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { /* Every ASCII string is already normalized in all four forms. Validate * the form first: even an empty subject must reject an invalid form. */ if (scr_text_ascii(s)) return scr_str_retain((ScrStr *)s); - if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); - uint32_t *input = malloc((s->len + 1) * sizeof(uint32_t)); - if (!input) scr_regex_oom(); - int length = 0; + bool inert = true; for (size_t offset = 0; offset < s->len;) { + if ((unsigned char)s->data[offset] < 0x80) { + offset++; + continue; + } size_t step; - input[length++] = scr_case_decode(s->data + offset, &step); + uint32_t c = scr_case_decode(s->data + offset, &step); offset += step; + if (!(scr_nqc_class(c, mode) & 1)) { + inert = false; + break; + } } - uint32_t *output = NULL; - int count = scr_unicode_normalize(&output, input, length, mode); - free(input); - if (count < 0) scr_regex_oom(); + if (inert) return scr_str_retain((ScrStr *)s); + if (s->len > INT32_MAX / sizeof(uint32_t)) scr_regex_oom(); + /* An inert code point bounds reordering and composition, and so does any + * other starter (class 0, never a composition's second element), so the + * text normalizes segment by segment. A segment starts at a starter: runs + * of inert code points copy as they are, and a segment with any other + * code point goes through scr_unicode_normalize. */ ScrJsonBuf buffer; scr_jb_init(&buffer); - for (int i = 0; i < count; i++) scr_case_put_cp(&buffer, output[i]); - free(output); + size_t seg = 0, last_inert = 0; + bool plain = true; + for (size_t offset = 0; offset < s->len;) { + unsigned char lead = (unsigned char)s->data[offset]; + size_t step = 1; + unsigned kind = 3; + if (lead >= 0x80) kind = scr_nqc_class(scr_case_decode(s->data + offset, &step), mode); + if (kind & 1) { + if (!plain) { + scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = true; + } + last_inert = offset; + } else if (kind & 2) { + if (plain) scr_jb_write(&buffer, s->data + seg, offset - seg); + else scr_nqc_segment(&buffer, s->data, seg, offset, mode); + seg = offset; + plain = false; + } else if (plain) { + size_t start = last_inert > seg ? last_inert : seg; + scr_jb_write(&buffer, s->data + seg, start - seg); + seg = start; + plain = false; + } + offset += step; + } + if (plain) scr_jb_write(&buffer, s->data + seg, s->len - seg); + else scr_nqc_segment(&buffer, s->data, seg, s->len, mode); return scr_jb_finish(&buffer); } diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 64a4772e5c..71d6299418 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -83,6 +83,12 @@ typedef struct { size_t cu; /* UTF-16 code-unit offset, always a code-point boundary */ size_t cb; /* matching UTF-8 byte offset, never a continuation byte */ } ScrSidxPoint; +/* One allocation: the 32-bit base of every 64th unit follows the deltas. */ +typedef struct { + size_t n; /* the UTF-16 length the offsets describe */ + uint32_t *base; /* byte offset of every 64th unit */ + uint8_t delta[]; /* each unit's byte offset past its block's base */ +} ScrSidxOffsets; typedef struct ScrSidx { const ScrStr *s; /* NULL = empty slot */ size_t u16len; /* SCR_U16_UNKNOWN until the whole current string */ @@ -94,10 +100,12 @@ typedef struct ScrSidx { size_t indexed_cb; bool no_more_points; /* metadata allocation failed/overflowed: fail open */ bool points_complete; /* points cover every stride of indexed prefix */ + uint32_t walks; /* index walks before the offsets exist */ uint16_t *units; /* the UTF-16 code units, once reads warrant them */ - size_t nunits; - size_t reads; /* indexed reads that missed the unit array */ -} ScrSidx; + uint32_t nunits; /* units and offsets need at most 1 << 24 bytes */ + uint32_t reads; /* indexed reads that missed the unit array */ + ScrSidxOffsets *offsets; /* every unit's byte offset, once walks warrant */ +} ScrSidx; /* 128 bytes on 64-bit targets: every lookup clears or copies one */ /* Keep short-string cursor traffic out of the sparse cache. A short-lived * one-byte receiver can be far more common than a large indexed one; sharing * the round-robin slots would otherwise rebuild a warm index every few calls. @@ -136,6 +144,7 @@ static void scr_sidx_clear(ScrSidx *e, unsigned *live) { scr_str_scan.n = 0; } free(e->units); + if (e->offsets) free(e->offsets); /* rare: skip the call on the hot clear */ memset(e, 0, sizeof(*e)); } @@ -337,6 +346,9 @@ static void scr_sidx_concat_append(const ScrStr *s, size_t oldlen) { e->units = NULL; e->nunits = 0; e->reads = 0; + free(e->offsets); /* and so do the offsets */ + e->offsets = NULL; + e->walks = 0; } } } @@ -1576,6 +1588,8 @@ static void scr_sidx_ascii_run(const ScrStr *s, ScrSidx *e) { /* How far past its end a forward access may grow the cached ASCII run, and * how far beyond the access the growth looks ahead. */ #define SCR_SIDX_ASCII_REACH ((size_t)4096) +/* How far from the hot cursor an index walks from it. */ +#define SCR_SIDX_NEAR_UNITS ((size_t)64) #define SCR_SIDX_ASCII_AHEAD ((size_t)256) /* Grow the cached ASCII run forward until it covers byte `want` (plus a @@ -1624,8 +1638,118 @@ static inline size_t scr_u16_to_byte_c(const ScrStr *s, ScrSidx *e, size_t u16, return scr_u16_to_byte_walk(s, e, u16, mid); } +/* Index walks on one long non-ASCII receiver (slices, substrings and + * indexed reads taken across it) each step through bytes from the nearest + * known position. Once those walks would have paid for one pass over the + * receiver (a walk per 32 bytes, at least 64 walks), a receiver from + * SCR_SIDX_MIN_BYTES to SCR_SIDX_OFFSETS_MAX bytes keeps every unit's byte + * offset: a 32-bit base per 64 units and one byte per unit past it (64 units + * span at most 192 bytes). The low half of an astral character has its high + * half's offset. The offsets are freed with the entry, and an append drops + * them (scr_sidx_concat_append). */ +#define SCR_SIDX_OFFSET_BLOCK ((size_t)64) +#define SCR_SIDX_OFFSETS_MAX ((size_t)1 << 24) +static bool scr_sidx_offsets(const ScrStr *s, ScrSidx *e) { + if (e->offsets) return true; + if (s->len < SCR_SIDX_MIN_BYTES || s->len > SCR_SIDX_OFFSETS_MAX) return false; + if (++e->walks < 64 || (size_t)e->walks * 32 < s->len) return false; + size_t n = scr_sidx_len(s, e); + if (s->ascii || e->u16len == s->len) return false; + size_t ndelta = (n + 1 + 3) & ~(size_t)3; + ScrSidxOffsets *o = + malloc(sizeof *o + ndelta + (n / SCR_SIDX_OFFSET_BLOCK + 1) * sizeof(uint32_t)); + if (!o) { + e->walks = 0; + return false; + } + o->n = n; + o->base = (uint32_t *)(o->delta + ndelta); + uint32_t *base = o->base; + uint8_t *delta = o->delta; + size_t cu = 0, cb = 0; + while (cb < s->len && cu < n) { + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + for (size_t w = seq == 4 ? 2 : 1; w > 0 && cu < n; w--, cu++) { + if (cu % SCR_SIDX_OFFSET_BLOCK == 0) base[cu / SCR_SIDX_OFFSET_BLOCK] = (uint32_t)cb; + delta[cu] = (uint8_t)(cb - base[cu / SCR_SIDX_OFFSET_BLOCK]); + } + cb += seq; + } + e->offsets = o; + return true; +} + +static inline size_t scr_sidx_offset_at(const ScrSidx *e, size_t u16) { + return e->offsets->base[u16 / SCR_SIDX_OFFSET_BLOCK] + e->offsets->delta[u16]; +} + +/* Walk from a known code-point boundary (cu, cb) to UTF-16 index u16 and + * leave the hot cursor there. Same contract as scr_u16_to_byte_walk. */ +static size_t scr_sidx_walk_from(const ScrStr *s, ScrSidx *e, size_t cu, size_t cb, size_t u16, + bool *mid, bool refresh_run) { + while (cu > u16) { + /* Eight bytes starting inside a sequence can reach back at most three + * more bytes to its lead. Such a span contains at most nine UTF-16 + * units (seven ASCII bytes plus an astral pair), so it cannot overshoot + * a target at least nine units away. */ + if (cu - u16 >= 9 && cb >= 8) { + size_t start = cb - 8; + while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) + start--; + cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); + cb = start; + } else { + scr_sidx_back(s, &cu, &cb); + } + } + bool m = false; + while (cu < u16 && cb < s->len) { + if (u16 - cu >= 9 && s->len - cb >= 8) { + size_t end = cb + 8; + while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) + end++; + cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); + cb = end; + continue; + } + SCR_SIDX_STEP(); + size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); + size_t w = seq == 4 ? 2 : 1; + if (cu + w > u16) { /* u16 lands between the halves of an astral char */ + m = true; + break; + } + cu += w; + cb += seq; + } + e->cu = cu; + e->cb = cb; + if (refresh_run && cb < s->len && (unsigned char)s->data[cb] < 0x80) + scr_sidx_ascii_run(s, e); + *mid = m; + return cb; +} + static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, ScrSidx *e, size_t u16, bool *mid) { + if (scr_sidx_offsets(s, e)) { + *mid = false; + if (u16 >= e->offsets->n) return s->len; + size_t cb = scr_sidx_offset_at(e, u16); + size_t cu = u16; + if (u16 > 0 && scr_sidx_offset_at(e, u16 - 1) == cb) { + *mid = true; /* the low half of an astral character */ + cu = u16 - 1; + } + e->cu = cu; + e->cb = cb; + return cb; + } + /* Within a short distance of the hot cursor (a scanner + * taking slices near its position), walk from that cursor: no index + * search, extension or ASCII-run growth is cheaper. */ + if (e->cb <= s->len && scr_sidx_abs_diff(e->cu, u16) <= SCR_SIDX_NEAR_UNITS) + return scr_sidx_walk_from(s, e, e->cu, e->cb, u16, mid, false); /* A short step forward from the hot cursor (a sequential scan of * non-ASCII text) walks from the cursor itself. */ if (u16 >= e->cu && u16 - e->cu <= 8) { @@ -1665,48 +1789,7 @@ static __attribute__((noinline)) size_t scr_u16_to_byte_walk(const ScrStr *s, Sc } scr_sidx_materialize_identity_prefix(s, e); ScrSidxPoint near = scr_sidx_near_u16(s, e, u16); - size_t cu = near.cu, cb = near.cb; - while (cu > u16) { - /* Eight bytes starting inside a sequence can reach back at most three - * more bytes to its lead. Such a span contains at most nine UTF-16 - * units (seven ASCII bytes plus an astral pair), so it cannot overshoot - * a target at least nine units away. */ - if (cu - u16 >= 9 && cb >= 8) { - size_t start = cb - 8; - while (start > 0 && ((unsigned char)s->data[start] & 0xC0) == 0x80) - start--; - cu -= scr_utf16_units_span(s->data + start, cb - start, NULL); - cb = start; - } else { - scr_sidx_back(s, &cu, &cb); - } - } - bool m = false; - while (cu < u16 && cb < s->len) { - if (u16 - cu >= 9 && s->len - cb >= 8) { - size_t end = cb + 8; - while (end < s->len && ((unsigned char)s->data[end] & 0xC0) == 0x80) - end++; - cu += scr_utf16_units_span(s->data + cb, end - cb, NULL); - cb = end; - continue; - } - SCR_SIDX_STEP(); - size_t seq = scr_utf8_seq_len((unsigned char)s->data[cb]); - size_t w = seq == 4 ? 2 : 1; - if (cu + w > u16) { /* u16 lands between the halves of an astral char */ - m = true; - break; - } - cu += w; - cb += seq; - } - e->cu = cu; - e->cb = cb; - if (cb < s->len && (unsigned char)s->data[cb] < 0x80) - scr_sidx_ascii_run(s, e); - *mid = m; - return cb; + return scr_sidx_walk_from(s, e, near.cu, near.cb, u16, mid, true); } static size_t scr_byte_to_u16_walk(const ScrStr *s, ScrSidx *e, size_t byte_off); @@ -1875,10 +1958,17 @@ static uint16_t *scr_str_units(const ScrStr *s, size_t *n) { * megabytes, which scanners read in full. The array is freed with the * entry, and an append drops it (scr_sidx_concat_append). */ #define SCR_SIDX_UNITS_MAX ((size_t)1 << 24) +static void scr_sidx_take_units(const ScrStr *s, ScrSidx *e) { + size_t n = 0; + e->units = scr_str_units(s, &n); + e->nunits = (uint32_t)n; +} + static bool scr_sidx_units(const ScrStr *s, ScrSidx *e) { if (!e->units) { - if (++e->reads < 32 || e->reads * 16 < s->len || s->len > SCR_SIDX_UNITS_MAX) return false; - e->units = scr_str_units(s, &e->nunits); + if (s->len > SCR_SIDX_UNITS_MAX || ++e->reads < 32 || (size_t)e->reads * 16 < s->len) + return false; + scr_sidx_take_units(s, e); } /* Publish the scan's receiver for emitted reads (scr_runtime.h). */ if (scr_str_scan.s != s) { @@ -1905,7 +1995,7 @@ void scr_str_scan_units_for(ScrStr *s, double from, double to) { (void)scr_sidx_units(s, e); return; } - e->units = scr_str_units(s, &e->nunits); + scr_sidx_take_units(s, e); } if (scr_str_scan.s != s) { scr_str_scan.s = s; @@ -1980,8 +2070,9 @@ double scr_str_last_index_of_from(ScrStr *s, ScrStr *needle, double position) { size_t start_byte = scr_u16_to_byte_c(s, e, start16, &mid); size_t max_start = s->len - needle->len; if (start_byte < max_start) max_start = start_byte; + char first = needle->data[0]; for (size_t i = max_start + 1; i-- > 0;) { - if (memcmp(s->data + i, needle->data, needle->len) == 0) { + if (s->data[i] == first && memcmp(s->data + i, needle->data, needle->len) == 0) { return (double)scr_byte_to_u16_c(s, e, i); } } diff --git a/packages/runtime/test/heap-trim.test.ts b/packages/runtime/test/heap-trim.test.ts new file mode 100644 index 0000000000..f2ec405feb --- /dev/null +++ b/packages/runtime/test/heap-trim.test.ts @@ -0,0 +1,43 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const testDir = import.meta.dirname; + +// scr_heap_trim returns the size-class pages that hold only free blocks, in +// the single-threaded allocator and in the per-thread arenas of worker +// runtimes while another thread allocates. +test.runIf(process.platform === "linux")( + "heap trim returns free pages and keeps live blocks", + async () => { + const buildDir = join(testDir, "build"); + await mkdir(buildDir, { recursive: true }); + for (const [name, flags] of [ + ["test_heap_trim", []], + ["test_heap_trim_workers", ["-DSCR_WORKERS", "-pthread"]], + ] as const) { + const bin = join(buildDir, name); + await execFileAsync("clang", [ + "-std=c11", + "-D_GNU_SOURCE", + "-O1", + "-Wall", + "-Wextra", + "-fsanitize=undefined", + ...flags, + "-o", + bin, + join(testDir, "test_heap_trim.c"), + join(testDir, "../src/scr_alloc.c"), + ]); + const run = await execFileAsync(bin, [], { + env: { ...process.env, UBSAN_OPTIONS: "halt_on_error=1" }, + }); + expect(run.stdout).toBe("heap trim checks passed\n"); + } + }, + 120_000, +); diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index dc48abd263..08eaecad41 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -1314,6 +1314,44 @@ static void check_limited_pass(void) { check(freed - before == 4 * RING, "the live ring was not collected once dead"); } +/* A limited pass that reaches a live candidate it did not take leaves the + * candidate in its slot and in its generation, which names the slot's + * buffer: when the candidate dies afterwards, it leaves that buffer, and no + * later pass meets its freed block. */ +static void check_limited_pass_keeps_untaken_generation(void) { + enum { RING = 16 }; + if (configured_nursery_threshold() < 8) return; + scr_collect_cycles(); + size_t before = freed; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + Node *shared = scr_cyc_alloc(sizeof(*shared), &cyc_type); + shared->rc = 1; /* its owner */ + /* A dead ring the filter takes points at it. */ + Node *dead = make_ring(RING); + dead->other = shared; + shared->rc++; + /* It loses a reference and becomes a candidate the filter refuses. */ + shared->rc++; + release_live(shared); + release_live(dead); + ScrCycHdr *h = scr_cyc_hdr(shared); + unsigned gen = scr_cyc_gen(h); + check(scr_cyc_is_buffered(h), "the shared node is not a candidate"); + take(dead); + check(scr_cyc_collect_set(&taken, SCR_CYC_NURSERY) == RING, + "limited pass did not free the taken ring"); + check(scr_cyc_is_buffered(h) && scr_cyc_gen(h) == gen, + "limited pass moved a candidate it did not take out of its generation"); + memset(taken_slots, 0, sizeof taken_slots); + /* Its owner lets go: it dies directly, out of its buffer. */ + check(shared->rc == 1, "the shared node kept a reference from the freed ring"); + shared->rc--; + scr_cyc_on_dead(shared); + node_free(shared); + scr_collect_cycles(); + check(freed - before == RING + 1, "the passes after the limited one freed something else"); +} + /* A full pass whose fullest candidate buffer is the old generation's (more * entries than a pass keeps between passes), with nursery candidates * beside it: every dead ring in either buffer is reclaimed, the live rings @@ -1593,6 +1631,7 @@ int main(void) { check_allocation_growth_runs_passes(); check_publication_takes_candidates(); check_limited_pass(); + check_limited_pass_keeps_untaken_generation(); check_large_old_buffer_with_nursery_candidates(); check_unjudged_backlog_follows_growth(); check_acyclic_objects(); diff --git a/packages/runtime/test/test_heap_trim.c b/packages/runtime/test/test_heap_trim.c new file mode 100644 index 0000000000..afd1404301 --- /dev/null +++ b/packages/runtime/test/test_heap_trim.c @@ -0,0 +1,145 @@ +/* scr_heap_trim: after most blocks of a class are freed, the pages holding + * only free blocks go back to the system, the blocks still allocated keep + * their contents, and the class serves new blocks that overlap none of + * them. Built with and without SCR_WORKERS; with it, another thread + * allocates and frees its own blocks throughout, and frees blocks of the + * trimming thread before the trim (remote frees). */ +#include "../src/scr_runtime.h" + +#include +#include +#include +#include +#include +#ifdef SCR_WORKERS +#include +#include +#endif + +#define COUNT 400000 +#define SIZE 48 + +_Noreturn void scr_trap(const char *msg) { + fputs(msg, stderr); + exit(EXIT_FAILURE); +} + +/* One block kept per KEEP_PAGES pages of blocks, whatever the page size: a + * kept block pins its page, so the others hold only free blocks. */ +#define KEEP_PAGES 4 +static size_t keep_every; + +static void check(bool condition, const char *message) { + if (condition) return; + fprintf(stderr, "heap trim test failed: %s\n", message); + exit(EXIT_FAILURE); +} + +/* Resident bytes of this process. */ +static size_t resident(void) { + FILE *f = fopen("/proc/self/statm", "r"); + check(f != NULL, "open /proc/self/statm"); + unsigned long size = 0, pages = 0; + check(fscanf(f, "%lu %lu", &size, &pages) == 2, "read /proc/self/statm"); + fclose(f); + return (size_t)pages * (size_t)sysconf(_SC_PAGESIZE); +} + +#ifdef SCR_WORKERS +static atomic_bool churning = true; + +/* Allocates, fills and frees blocks of several sizes until told to stop. */ +static void *churn(void *arg) { + (void)arg; + void *held[256] = {0}; + for (size_t round = 0; atomic_load(&churning); round++) { + size_t slot = round % 256, size = 16 + (round % 7) * 24; + if (held[slot]) scr_mem_free(held[slot]); + held[slot] = scr_mem_alloc(size); + check(held[slot] != NULL, "allocate on the other thread"); + memset(held[slot], 0x33, size); + } + for (size_t i = 0; i < 256; i++) + if (held[i]) scr_mem_free(held[i]); + return NULL; +} + +/* Frees `count` blocks starting at `first` from another thread. */ +typedef struct { + unsigned char **blocks; + size_t first, count; +} RemoteFree; + +static void *free_remotely(void *arg) { + RemoteFree *r = arg; + for (size_t i = r->first; i < r->first + r->count; i++) + if (i % keep_every != 0) { + scr_mem_free(r->blocks[i]); + r->blocks[i] = NULL; + } + return NULL; +} +#endif + +static int compare(const void *a, const void *b) { + uintptr_t x = *(const uintptr_t *)a, y = *(const uintptr_t *)b; + return x < y ? -1 : x > y; +} + +int main(void) { + keep_every = KEEP_PAGES * ((size_t)sysconf(_SC_PAGESIZE) / SIZE + 1); +#ifdef SCR_WORKERS + pthread_t churner; + check(pthread_create(&churner, NULL, churn, NULL) == 0, "start the other thread"); +#endif + unsigned char **blocks = malloc(COUNT * sizeof *blocks); + check(blocks != NULL, "block table"); + for (size_t i = 0; i < COUNT; i++) { + blocks[i] = scr_mem_alloc(SIZE); + check(blocks[i] != NULL, "allocate"); + memset(blocks[i], (int)(i & 0xff), SIZE); + } + size_t kept = 0; + uintptr_t *keep = malloc((COUNT / keep_every + 1) * sizeof *keep); + check(keep != NULL, "keep table"); +#ifdef SCR_WORKERS + RemoteFree remote = {blocks, 0, COUNT / 4}; + pthread_t freer; + check(pthread_create(&freer, NULL, free_remotely, &remote) == 0, "start the freeing thread"); + check(pthread_join(freer, NULL) == 0, "join the freeing thread"); +#endif + for (size_t i = 0; i < COUNT; i++) { + if (i % keep_every == 0) { + keep[kept++] = (uintptr_t)blocks[i]; + continue; + } + if (blocks[i]) scr_mem_free(blocks[i]); + } + size_t before = resident(); + scr_heap_trim(); + size_t after = resident(); + check(before > after && before - after >= (size_t)8 << 20, "the free pages go back to the system"); + for (size_t i = 0; i < COUNT; i += keep_every) + for (size_t j = 0; j < SIZE; j++) check(blocks[i][j] == (unsigned char)(i & 0xff), "kept blocks keep their contents"); + qsort(keep, kept, sizeof *keep, compare); + for (size_t i = 0; i < COUNT; i++) { + if (i % keep_every == 0) continue; + unsigned char *p = scr_mem_alloc(SIZE); + check(p != NULL, "allocate after the trim"); + uintptr_t key = (uintptr_t)p; + check(bsearch(&key, keep, kept, sizeof *keep, compare) == NULL, "a new block is not a kept one"); + memset(p, 0x5a, SIZE); + blocks[i] = p; + } + for (size_t i = 0; i < COUNT; i += keep_every) + for (size_t j = 0; j < SIZE; j++) check(blocks[i][j] == (unsigned char)(i & 0xff), "kept blocks survive new allocations"); + for (size_t i = 0; i < COUNT; i++) scr_mem_free(blocks[i]); +#ifdef SCR_WORKERS + atomic_store(&churning, false); + check(pthread_join(churner, NULL) == 0, "join the other thread"); +#endif + free(keep); + free(blocks); + printf("heap trim checks passed\n"); + return 0; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5fc32620ad..f4a645778e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,8 +110,8 @@ importers: specifier: workspace:* version: link:../runtime tsc-ts: - specifier: 0.0.9 - version: 0.0.9 + specifier: 0.0.10 + version: 0.0.10 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1360,8 +1360,8 @@ packages: cpu: [x64] os: [win32] - '@scriptc/threads@0.4.2': - resolution: {integrity: sha512-nR9FUxpOpgtTmJu6dejttElBL+JJTAZzJ2DZM+eNoujmLx9VpdlAVdgj3cu/MDsTUg/1sIUBq1ME1csC/Aa6yg==} + '@scriptc/threads@0.4.5': + resolution: {integrity: sha512-wHYrCItM68xYAwjnvM6VrRLDWd9+lAEQNjKh6PyLA8RdE7D8ts/VMoTFLmXRu2uvRywJuY18WPSCoEFRf3qfiA==} engines: {node: '>=24.0.0'} '@sinclair/typebox@0.25.24': @@ -1377,42 +1377,42 @@ packages: '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} - '@tsc-ts/darwin-arm64@0.0.9': - resolution: {integrity: sha512-aIjSulef3DIEX7F3RRWLXyHWFgjXiVffBkrfE83ay/tpbsQaniu16cFxYGYNaqqVGft/oivecpdTDBbpPlsb8Q==} + '@tsc-ts/darwin-arm64@0.0.10': + resolution: {integrity: sha512-mRctKijCzWRxJgxiVAdsk+x9WB7kZz9GBbJ4eVP95bJFkIz9D11uw3xC0m6jWYNoaShF6zn48eLMTYOyJNxBdA==} cpu: [arm64] os: [darwin] - '@tsc-ts/darwin-x64@0.0.9': - resolution: {integrity: sha512-LHE74tzIEDtrEN8ENvCH/ejnnZjT0D2Vaz2HmggcvJudgr0W300CcwGf2Kdfw8MVBBk6JN0HPF9+a5AUUtFZGg==} + '@tsc-ts/darwin-x64@0.0.10': + resolution: {integrity: sha512-oV78/mi3ot8CG3BA9QxrsamvNXutpschgae+qV5skU4gAqimmBsBHy/MWTf9SCssvtWeqrX4uZvcwE0h/AJ+MQ==} cpu: [x64] os: [darwin] - '@tsc-ts/linux-arm64-gnu@0.0.9': - resolution: {integrity: sha512-dNiQIKApLcKIJPzb3pk2gfGrrgmOWa+q3/qjrx5ia/SzIWz0+lENlVvHiIn9ukqDacDE30FW8xHzby8Kyd+10A==} + '@tsc-ts/linux-arm64-gnu@0.0.10': + resolution: {integrity: sha512-O5msxFydn9r64QVm9WDWG8CKkXLjXWo12XeJWDncHlQYK80SoM4T8/C1UTDLtb6A2P+NQ5a+Xq4AaHmNb8VIqg==} cpu: [arm64] os: [linux] libc: [glibc] - '@tsc-ts/linux-arm64-musl@0.0.9': - resolution: {integrity: sha512-DolAqHtr1GUyG03Ch8+sJzpX1G+xzazAmj18kABzuOPre/6njrSWnmqJmtXQWvhwLBBEjajIkUJ4V+t5qigsrg==} + '@tsc-ts/linux-arm64-musl@0.0.10': + resolution: {integrity: sha512-v9N5HuqcEW97MzwzLqPNAN5tfAcXIAlLRtp7BSJJd5VKfBu6DplqNZzwCH+//NuAQOtWGSr/Dq565JFPmDRt5w==} cpu: [arm64] os: [linux] libc: [musl] - '@tsc-ts/linux-x64-gnu@0.0.9': - resolution: {integrity: sha512-leANVZ4tKz3myBSVogb173ttbAurPOOGjy1WeBlRhkVJ7zOPMiyzJi3cPOAK/4O8RcxXOsCiC2AfKPKDYM8bEw==} + '@tsc-ts/linux-x64-gnu@0.0.10': + resolution: {integrity: sha512-sRTPB/50uAJSpMjlA3sc9GTAySnLwIlf1txR8eHlcVpKE6q0QgWvEkKqCSGtFXkn8IQ8z8O68jrAk9WjmRTaYQ==} cpu: [x64] os: [linux] libc: [glibc] - '@tsc-ts/linux-x64-musl@0.0.9': - resolution: {integrity: sha512-a4FW16QXkrwNHAr4Qo/qDA5w9LxCJNrYbpqiz3TJ1sePTqvW/JTJc14tjx6avhW4mIaQp8AB52m7TqNAuDMm4Q==} + '@tsc-ts/linux-x64-musl@0.0.10': + resolution: {integrity: sha512-U4qZwP0JzrR/l4kzhh+w+y6o2guEEinC/wminfXXDhez2DtjNTOrM1GxMPfZ7JkCIcKEruL7iCC2telapFk5LQ==} cpu: [x64] os: [linux] libc: [musl] - '@tsc-ts/win32-x64-msvc@0.0.9': - resolution: {integrity: sha512-phThYnmOinlvY4dS4DEl1L9t/mFLiHR1zL5plSUfMB5NizA/jESfgMGRu+HW3uwOpCETSxgsuTNAXO7UuEIHIQ==} + '@tsc-ts/win32-x64-msvc@0.0.10': + resolution: {integrity: sha512-TsiK6k0ljFW3pNR5lvF7SL0nv3aGxr31ZbBCfIXcBrqAvpumINOy/G0ESfTvZWx8llwOQt3PxBS414gF/J0tFA==} cpu: [x64] os: [win32] @@ -2577,8 +2577,8 @@ packages: ts-toolbelt@6.15.5: resolution: {integrity: sha512-FZIXf1ksVyLcfr7M317jbB67XFJhOO1YqdTcuGaq9q5jLUoTikukZ+98TPjKiP2jC5CgmYdWWYs0s2nLSU0/1A==} - tsc-ts@0.0.9: - resolution: {integrity: sha512-7fDfC/o1YoITG6sSXj4ZznijQYg8AfoCagmYvwG1BFLR/hz4Lwnq7jqDwcZfoDuPYTkrs8jHUTAVG1turniJHA==} + tsc-ts@0.0.10: + resolution: {integrity: sha512-oF5hZbeNgJXwJ868adnvBAihxYUwra1rj6Sa2+UhuhW477t5MmGnOsZgGTSGz511uTfuUxDz6e/gLIzvvScbGw==} engines: {node: '>=20'} hasBin: true @@ -3439,7 +3439,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.62.2': optional: true - '@scriptc/threads@0.4.2': {} + '@scriptc/threads@0.4.5': {} '@sinclair/typebox@0.25.24': {} @@ -3454,25 +3454,25 @@ snapshots: mkdirp: 1.0.4 path-browserify: 1.0.1 - '@tsc-ts/darwin-arm64@0.0.9': + '@tsc-ts/darwin-arm64@0.0.10': optional: true - '@tsc-ts/darwin-x64@0.0.9': + '@tsc-ts/darwin-x64@0.0.10': optional: true - '@tsc-ts/linux-arm64-gnu@0.0.9': + '@tsc-ts/linux-arm64-gnu@0.0.10': optional: true - '@tsc-ts/linux-arm64-musl@0.0.9': + '@tsc-ts/linux-arm64-musl@0.0.10': optional: true - '@tsc-ts/linux-x64-gnu@0.0.9': + '@tsc-ts/linux-x64-gnu@0.0.10': optional: true - '@tsc-ts/linux-x64-musl@0.0.9': + '@tsc-ts/linux-x64-musl@0.0.10': optional: true - '@tsc-ts/win32-x64-msvc@0.0.9': + '@tsc-ts/win32-x64-msvc@0.0.10': optional: true '@tybys/wasm-util@0.10.3': @@ -4813,17 +4813,17 @@ snapshots: ts-toolbelt@6.15.5: {} - tsc-ts@0.0.9: + tsc-ts@0.0.10: dependencies: - '@scriptc/threads': 0.4.2 + '@scriptc/threads': 0.4.5 optionalDependencies: - '@tsc-ts/darwin-arm64': 0.0.9 - '@tsc-ts/darwin-x64': 0.0.9 - '@tsc-ts/linux-arm64-gnu': 0.0.9 - '@tsc-ts/linux-arm64-musl': 0.0.9 - '@tsc-ts/linux-x64-gnu': 0.0.9 - '@tsc-ts/linux-x64-musl': 0.0.9 - '@tsc-ts/win32-x64-msvc': 0.0.9 + '@tsc-ts/darwin-arm64': 0.0.10 + '@tsc-ts/darwin-x64': 0.0.10 + '@tsc-ts/linux-arm64-gnu': 0.0.10 + '@tsc-ts/linux-arm64-musl': 0.0.10 + '@tsc-ts/linux-x64-gnu': 0.0.10 + '@tsc-ts/linux-x64-musl': 0.0.10 + '@tsc-ts/win32-x64-msvc': 0.0.10 tslib@2.8.1: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c9f873c4ee..b678a87301 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,15 +7,15 @@ minimumReleaseAgeExclude: - "effect@4.0.0" - "@effect/platform-node@4.0.0" - "@effect/platform-node-shared@4.0.0" - - "@tsc-ts/darwin-arm64@0.0.9" - - "@tsc-ts/darwin-x64@0.0.9" - - "@tsc-ts/linux-arm64-gnu@0.0.9" - - "@tsc-ts/linux-arm64-musl@0.0.9" - - "@tsc-ts/linux-x64-gnu@0.0.9" - - "@tsc-ts/linux-x64-musl@0.0.9" - - "@tsc-ts/win32-x64-msvc@0.0.9" - - "tsc-ts@0.0.9" - - "@scriptc/threads@0.4.2" + - "@tsc-ts/darwin-arm64@0.0.10" + - "@tsc-ts/darwin-x64@0.0.10" + - "@tsc-ts/linux-arm64-gnu@0.0.10" + - "@tsc-ts/linux-arm64-musl@0.0.10" + - "@tsc-ts/linux-x64-gnu@0.0.10" + - "@tsc-ts/linux-x64-musl@0.0.10" + - "@tsc-ts/win32-x64-msvc@0.0.10" + - "tsc-ts@0.0.10" + - "@scriptc/threads@0.4.5" allowBuilds: esbuild: true overrides: diff --git a/scripts/bootstrap-seed.mjs b/scripts/bootstrap-seed.mjs index f0993f1260..6bce075f5e 100644 --- a/scripts/bootstrap-seed.mjs +++ b/scripts/bootstrap-seed.mjs @@ -25,6 +25,19 @@ export function verifyBootstrapSeed(directory, revision, sanitize) { } } +/** The environment a bootstrap seed builds with, over the inherited one. The + * bootstrap compares the seed's LLVM with the native compiler's own emission, + * which has no function layout. The layout only orders the functions, so the + * seed is built without it. */ +export function seedBuildEnvironment(sanitize) { + return { + SCRIPTC_SAN: sanitize ? "1" : "", + SCRIPTC_NATIVE_EMIT_IR: "1", + SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", + SCRIPTC_NATIVE_LAYOUT: "", + }; +} + async function main() { const directory = process.env.SCRIPTC_BOOTSTRAP_SEED_DIRECTORY; const revision = process.env.GITHUB_SHA; @@ -47,11 +60,7 @@ async function main() { { cwd: root, stdio: "inherit", - env: { - ...process.env, - SCRIPTC_NATIVE_EMIT_IR: "1", - SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", - }, + env: { ...process.env, ...seedBuildEnvironment(sanitize) }, }, ); writeFileSync( diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 05f36cf689..e760658808 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -37,6 +37,15 @@ const useProfile = !sanitize && requestedProfile !== "" && (requestedProfile !== undefined || existsSync(committedProfile)); +// Builds lay out the compiler's functions with the committed layout +// (scripts/refresh-cli-layout.mjs), or the one SCRIPTC_NATIVE_LAYOUT names; +// an empty SCRIPTC_NATIVE_LAYOUT, and instrumented builds, use none. +const committedLayout = join(root, "packages/compiler/native/cli.layout.gz"); +const requestedLayout = process.env["SCRIPTC_NATIVE_LAYOUT"]; +const functionLayout = + profileGenerate !== undefined || requestedLayout === "" + ? undefined + : (requestedLayout ?? (existsSync(committedLayout) ? committedLayout : undefined)); let target = nativeCodegenTarget(); if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); @@ -126,6 +135,7 @@ const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), sanitize, ...(profileGenerate === undefined ? {} : { profileGenerate }), ...(profileUse === undefined ? {} : { profileUse }), + ...(functionLayout === undefined ? {} : { functionLayout }), emitIr: process.env["SCRIPTC_NATIVE_EMIT_IR"] === "1", }); if (!result.ok) diff --git a/scripts/refresh-cli-layout.mjs b/scripts/refresh-cli-layout.mjs new file mode 100644 index 0000000000..cb396015fc --- /dev/null +++ b/scripts/refresh-cli-layout.mjs @@ -0,0 +1,275 @@ +#!/usr/bin/env node +/* Refreshes packages/compiler/native/cli.layout.gz, the function layout that + * builds of the native compiler use (scripts/build-native-cli.mts). The + * layout lists the functions the compiler runs, in tiers: tier 1 for + * `--version` and rebuilds of unchanged programs, then the functions of the + * other builds by the share of training builds that run them (COLD_TIERS). + * The emitter writes the listed functions first, by tier, so the code a build + * runs sits together in the compiler's executable and a run maps fewer of its + * pages: a function nearly every build runs sits with the others that do, not + * among functions only a few programs need. + * + * Builds an instrumented native compiler for this Linux host (or reuses one + * with --instrumented), builds each training program cold and then again + * unchanged, and reads the functions each kind of run executed from the raw + * profiles. The training programs are the runtime benchmark workloads at + * `speed` and every seventh corpus program at the default optimization. A + * refresh takes about 15 minutes on 16 cores, most of it for the instrumented + * build. A stale layout lists some functions that no longer exist and misses + * new ones, which then keep their usual order. + * + * node scripts/refresh-cli-layout.mjs [--jobs 8] [--instrumented ] + */ +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { constants, gzipSync, inflateSync } from "node:zlib"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const { values } = parseArgs({ + options: { + jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) }, + output: { type: "string", default: join(root, "packages/compiler/native/cli.layout.gz") }, + work: { type: "string" }, + instrumented: { type: "string" }, + help: { type: "boolean", default: false }, + }, +}); +if (values.help) { + console.log( + "Usage: node scripts/refresh-cli-layout.mjs [--jobs N] [--output ] [--work ] [--instrumented ]", + ); + process.exit(0); +} +if (process.platform !== "linux" || process.arch !== "x64") + throw new Error("the compiler layout is collected on Linux x64"); +const jobs = Number(values.jobs); +const output = resolve(values.output); +mkdirSync(dirname(output), { recursive: true }); +const work = resolve(values.work ?? mkdtempSync(join(tmpdir(), "scriptc-cli-layout-"))); +const raw = join(work, "raw"); +const builds = join(work, "builds"); +rmSync(raw, { recursive: true, force: true }); +for (const dir of [raw, builds]) mkdirSync(dir, { recursive: true }); +const log = join(work, "training.log"); + +const cli = resolve(values.instrumented ?? join(work, "cli")); +if (values.instrumented === undefined) { + console.log(`building the instrumented compiler in ${cli}`); + const built = spawnSync( + process.execPath, + [ + "--max-old-space-size=8192", + "--import", + "tsx", + join(root, "scripts/build-native-cli.mts"), + cli, + ], + { + cwd: root, + stdio: "inherit", + env: { ...process.env, SCRIPTC_NATIVE_PROFILE_GENERATE: join(raw, "build-%4m.profraw") }, + }, + ); + if (built.status !== 0) throw new Error("the instrumented compiler did not build"); +} +const scriptc = join(cli, "bin/scriptc"); +// Phase timing shows whether a rebuild hit both caches. It is on for every +// run, so the rebuild's environment matches the first build's. +const baseEnv = { ...process.env, SCRIPTC_TIMING: "1" }; +delete baseEnv["SCRIPTC_LLVM_HELPER"]; + +/** Runs the instrumented compiler, writing its profile to `options.profile`. */ +function run(args, options) { + return new Promise((done) => { + const child = spawn(scriptc, args, { + cwd: options.cwd, + stdio: ["ignore", "pipe", "pipe"], + env: { + ...baseEnv, + SCRIPTC_CACHE_DIR: options.cache, + LLVM_PROFILE_FILE: options.profile, + }, + }); + let text = ""; + child.stdout.on("data", (chunk) => (text += chunk)); + child.stderr.on("data", (chunk) => (text += chunk)); + child.on("close", (code) => { + appendFileSync(log, `$ scriptc ${args.join(" ")}\n${text}exit ${code}\n`); + done({ ok: code === 0, text }); + }); + }); +} + +async function pool(tasks) { + let next = 0; + let passed = 0; + await Promise.all( + Array.from({ length: jobs }, async () => { + while (next < tasks.length) if (await tasks[next++]()) passed++; + }), + ); + return passed; +} + +const programNumber = (name) => Number.parseInt(name, 10); +const training = [ + ...readdirSync(join(root, "benchmarks/runtime/workloads")) + .filter((name) => name.endsWith(".ts")) + .map((name) => ({ + file: join(root, "benchmarks/runtime/workloads", name), + optimization: "speed", + })), + ...readdirSync(join(root, "tests/corpus")) + .filter((name) => /^\d+-.*\.ts$/.test(name) && programNumber(name) % 7 === 0) + .map((name) => ({ file: join(root, "tests/corpus", name), optimization: "release" })), +]; + +// Tier 1 collects each run that took the warm path from its own profile; a +// rebuild that missed a cache (it shows no `frontend-cache-hit` or +// `executable-cache-hit` phase) is left out. The first builds each write +// their own profile (cold--*), merged across the build's processes. +const warm = new Set(); +let skipped = 0; +async function warmRun(args, cache, name, cacheHits) { + const profile = join(raw, `warm-${name}.profraw`); + const { ok, text } = await run(args, { cwd: root, cache, profile }); + if (ok && cacheHits.every((phase) => text.includes(`"phase":"${phase}"`))) + for (const fn of executedFunctions(profile)) warm.add(fn); + else skipped++; + rmSync(profile, { force: true }); +} + +const started = Date.now(); +await warmRun(["--version"], join(work, "cache-version"), "version", []); +console.log(`training: ${training.length} programs, cold then unchanged, ${jobs} at a time`); +const passed = await pool( + training.map((program, index) => async () => { + const cache = join(work, `cache-${index}`); + const out = join(builds, `program-${index}`); + const args = ["build", program.file, `--optimization=${program.optimization}`, "-o", out]; + const { ok } = await run(args, { + cwd: root, + cache, + profile: join(raw, `cold-${index}-%1m.profraw`), + }); + if (ok) + await warmRun(args, cache, String(index), ["frontend-cache-hit", "executable-cache-hit"]); + rmSync(cache, { recursive: true, force: true }); + rmSync(out, { force: true }); + return ok; + }), +); +console.log( + `trained in ${Math.round((Date.now() - started) / 1000)} s; ${passed} of ${training.length} programs built, ${skipped} rebuilds left out of tier 1 (see ${log})`, +); + +/** Functions a raw profile (LLVM's format version 10) records as run: the + * names whose counters are not all zero, without the file prefix of + * internal functions. */ +function executedFunctions(path) { + const buf = readFileSync(path); + const u64 = (offset) => Number(buf.readBigUInt64LE(offset)); + if (buf.readBigUInt64LE(0) !== 0xff6c70726f667281n) + throw new Error(`${path} is not a 64-bit raw profile`); + const version = Number(buf.readBigUInt64LE(8) & 0xffffffffn); + if (version !== 10) throw new Error(`${path} has raw profile version ${version}, expected 10`); + const [binaryIdsBytes, records, beforeCounters, counters, afterCounters] = [2, 3, 4, 5, 6].map( + (field) => u64(field * 8), + ); + const [bitmapBytes, afterBitmap, namesBytes] = [7, 8, 9].map((field) => u64(field * 8)); + const valueKinds = u64(15 * 8) + 1; + const recordBytes = Math.ceil((6 * 8 + 4 + 2 * valueKinds + 4) / 8) * 8; + const dataStart = 16 * 8 + binaryIdsBytes; + const countersStart = dataStart + records * recordBytes + beforeCounters; + const namesStart = countersStart + counters * 8 + afterCounters + bitmapBytes + afterBitmap; + const countersDelta = buf.readBigInt64LE(10 * 8); + const names = new Map(); + let at = namesStart; + const uleb = () => { + let value = 0; + let scale = 1; + for (;;) { + const byte = buf[at++]; + value += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) return value; + scale *= 128; + } + }; + while (at < namesStart + namesBytes) { + const plain = uleb(); + const packed = uleb(); + if (plain === 0) break; + const bytes = + packed === 0 ? buf.subarray(at, at + plain) : inflateSync(buf.subarray(at, at + packed)); + at += packed === 0 ? plain : packed; + for (const name of bytes.toString("latin1").split("\x01")) + names.set(createHash("md5").update(name, "latin1").digest().readBigUInt64LE(0), name); + } + const ran = []; + for (let index = 0; index < records; index++) { + const record = dataStart + index * recordBytes; + const counterOffset = Number( + buf.readBigInt64LE(record + 16) - (countersDelta - BigInt(index * recordBytes)), + ); + const count = buf.readUInt32LE(record + 48); + for (let c = 0; c < count; c++) { + if (buf.readBigUInt64LE(countersStart + counterOffset + c * 8) !== 0n) { + const name = names.get(buf.readBigUInt64LE(record)); + if (name !== undefined) ran.push(name.slice(name.indexOf(";") + 1)); + break; + } + } + } + return ran; +} + +// The cold tiers, by the least share of training builds that run a function: +// tier 2 for functions at least 90% of the builds run, tier 3 for 60%, and so +// on; tier 7 holds the functions fewer than 3% of the builds run. +const COLD_TIERS = [0.9, 0.6, 0.3, 0.1, 0.03]; +const coldRuns = new Map(); +for (const name of readdirSync(raw)) { + const program = /^cold-(\d+)-/.exec(name)?.[1]; + if (program === undefined) continue; + const ran = coldRuns.get(program) ?? new Set(); + for (const fn of executedFunctions(join(raw, name))) ran.add(fn); + coldRuns.set(program, ran); +} +const runs = new Map(); +for (const ran of coldRuns.values()) for (const fn of ran) runs.set(fn, (runs.get(fn) ?? 0) + 1); +const coldTier = (count) => { + const share = count / coldRuns.size; + const index = COLD_TIERS.findIndex((least) => share >= least); + return 2 + (index < 0 ? COLD_TIERS.length : index); +}; +const lines = [ + ...[...warm].sort().map((name) => `1 ${name}`), + ...[...runs] + .filter(([name]) => !warm.has(name)) + .map(([name, count]) => [coldTier(count), name]) + .sort((a, b) => a[0] - b[0] || (a[1] < b[1] ? -1 : a[1] > b[1] ? 1 : 0)) + .map(([tier, name]) => `${tier} ${name}`), +]; +writeFileSync(output, gzipSync(`${lines.join("\n")}\n`, { level: constants.Z_BEST_COMPRESSION })); +const perTier = new Map(); +for (const line of lines) + perTier.set(line.split(" ")[0], (perTier.get(line.split(" ")[0]) ?? 0) + 1); +console.log( + `wrote ${output}: ${[...perTier].map(([tier, count]) => `tier ${tier}: ${count}`).join(", ")} functions, from ${coldRuns.size} builds (${statSync(output).size} bytes)`, +); +rmSync(builds, { recursive: true, force: true }); +if (values.work === undefined) rmSync(work, { recursive: true, force: true }); diff --git a/tests/corpus/5025-lifetime-region-shared-candidate.ts b/tests/corpus/5025-lifetime-region-shared-candidate.ts new file mode 100644 index 0000000000..7b5084a056 --- /dev/null +++ b/tests/corpus/5025-lifetime-region-shared-candidate.ts @@ -0,0 +1,85 @@ +// LifetimeRegion (@scriptc/experimental): freeing a region whose objects +// own a cycle runs a limited cycle pass over what the region owned. That +// pass also reaches a shared object the region pointed at, which is a +// candidate of its own (it lost a reference) and stays referenced from +// outside. The shared object must stay a valid candidate after the pass: +// it dies afterwards, later passes run, and the program prints the same +// as under Node.js. +import type { ProgramLifetime } from "@scriptc/types"; +import { LifetimeRegion } from "@scriptc/experimental"; + +class Shared { + readonly label: string; + peers: Shared[] = []; + constructor(label: string) { + this.label = label; + } +} + +// Ordinary objects the region's instance owns: a cycle that points back +// into the region and at the shared object. +class Owned { + holder: Holder | undefined = undefined; + shared: Shared | undefined = undefined; + partner: Owned | undefined = undefined; +} + +class Holder implements ProgramLifetime { + owned: Owned | undefined = undefined; + readonly id: number; + constructor(id: number) { + this.id = id; + } +} + +// Cyclic garbage, so the collector runs passes of its own. +class Ring { + next: Ring | undefined = undefined; + readonly value: number; + constructor(value: number) { + this.value = value; + } +} + +function churn(n: number): number { + let total = 0; + for (let i = 0; i < n; i++) { + const a = new Ring(i); + const b = new Ring(i + 1); + a.next = b; + b.next = a; + total += a.value + b.next.value; + } + return total; +} + +function round(k: number): string { + let shared: Shared | undefined = new Shared(`shared ${k}`); + // A second reference, dropped below: the shared object becomes a candidate. + let keep: Shared[] = [shared]; + shared.peers.push(new Shared(`peer ${k}`)); + const region = new LifetimeRegion(); + region.run(() => { + const holder = new Holder(k); + const owned = new Owned(); + const partner = new Owned(); + owned.partner = partner; + partner.partner = owned; + owned.holder = holder; + owned.shared = shared; + holder.owned = owned; + }); + keep = []; + region.release(); + const label = shared.label + " / " + shared.peers[0]!.label + " / " + keep.length; + // The shared object dies, and the passes that follow read the candidate buffers. + shared = undefined; + churn(2000); + return label; +} + +for (let k = 0; k < 20; k++) { + const label = round(k); + if (k % 5 === 0) console.log(label); +} +console.log("churn", churn(10000)); diff --git a/tests/corpus/7515-net-socket-backpressure.ts b/tests/corpus/7515-net-socket-backpressure.ts index 260726a153..6524e6d545 100644 --- a/tests/corpus/7515-net-socket-backpressure.ts +++ b/tests/corpus/7515-net-socket-backpressure.ts @@ -7,6 +7,11 @@ // untyped code reads the same properties, and a write whose send fails // after the peer reset the connection answers false. An echo client that // waits for 'drain' when write answers false moves every byte. +// +// The server picks each connection's handling from `mode` when it accepts +// it. The lookup client's bytes fit in the kernel's buffers, so its 'drain' +// can come before the server accepts that connection; the program waits for +// the accept before it changes `mode`. import { connect, createServer, Socket } from "node:net"; import type { LookupAddress, LookupAllOptions, LookupOneOptions } from "node:dns"; @@ -113,6 +118,15 @@ function afterReset(port: number): Promise { let paused: Socket | null = null; let mode = "echo"; +let accepted: (() => void) | null = null; + +/** Resolves when the server accepts its next connection. */ +function nextConnection(): Promise { + return new Promise((resolve) => { + accepted = resolve; + }); +} + const server = createServer((socket: Socket) => { if (mode === "echo") socket.pipe(socket); else if (mode === "drain") socket.resume(); @@ -120,6 +134,11 @@ const server = createServer((socket: Socket) => { socket.pause(); paused = socket; } else socket.destroy(); + if (accepted !== null) { + const resolve = accepted; + accepted = null; + resolve(); + } }); server.listen(0, "127.0.0.1", async () => { @@ -127,7 +146,9 @@ server.listen(0, "127.0.0.1", async () => { const port = typeof addr === "object" && addr !== null ? addr.port : 0; await echo(port); mode = "drain"; + const lookupAccepted = nextConnection(); await lookupPending(port); + await lookupAccepted; mode = "pause"; await largeWrite(port); mode = "end"; diff --git a/tests/corpus/8008-string-index-paths.ts b/tests/corpus/8008-string-index-paths.ts new file mode 100644 index 0000000000..a9a748ad86 --- /dev/null +++ b/tests/corpus/8008-string-index-paths.ts @@ -0,0 +1,102 @@ +// UTF-16 index operations on long non-ASCII strings: slices and substrings +// walking forward and backward, interleaved on two strings, at astral +// characters and their halves, with negative and out-of-range indexes; +// lastIndexOf, startsWith and endsWith with positions; and normalize on +// text already in each form, composed and decomposed text, Hangul, +// compatibility characters and combining marks. + +const unit = "Ünïcödé tëxt with ß and 日本語 and émoji 😀 mixed in, "; +const big = unit.repeat(3000); +const other = "ascii words then 中文 and 🎉🎉 and ü, ".repeat(4000); + +function digest(text: string): number { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0; + return h; +} + +let forward = 0; +for (let i = 0; i + 30 < big.length; i += 7) { + forward = (forward * 33 + digest(big.slice(i, i + 20)) + digest(big.substring(i + 3, i + 9))) >>> 0; +} +let backward = 0; +for (let i = big.length - 1; i > 30; i -= 11) { + backward = (backward * 33 + digest(big.slice(i - 25, i)) + big.substring(i, i - 5).length) >>> 0; +} +let interleaved = 0; +for (let i = 0; i + 40 < other.length && i + 40 < big.length; i += 13) { + interleaved = (interleaved * 33 + digest(big.slice(i, i + 9)) + digest(other.slice(i + 1, i + 40))) >>> 0; +} +console.log(big.length, forward, backward, interleaved); + +// Indexes at astral characters and their halves. +const emojiAt = big.indexOf("😀", 100000); +console.log( + emojiAt, + JSON.stringify(big.slice(emojiAt, emojiAt + 1)), + JSON.stringify(big.slice(emojiAt + 1, emojiAt + 3)), + JSON.stringify(big.substring(emojiAt - 1, emojiAt + 1)), + big.charCodeAt(emojiAt + 1), + JSON.stringify(big.at(emojiAt + 1)), + JSON.stringify(big.charAt(emojiAt)), +); +console.log( + JSON.stringify(big.slice(-5)), + JSON.stringify(big.slice(-30, -20)), + big.slice(big.length - 3, big.length + 10).length, + JSON.stringify(big.substring(-10, 4)), + big.slice(5, 2).length, + JSON.stringify(big.slice(NaN, 3)), +); +for (const at of [0, 1, 52, 99999, 150000, big.length - 1, big.length, -1, -2, -big.length]) { + console.log(at, JSON.stringify(big.at(at)), JSON.stringify(big.charAt(at)), big.charCodeAt(at)); +} + +// lastIndexOf, startsWith and endsWith with positions. +console.log( + big.lastIndexOf("日本"), + big.lastIndexOf("日本", 100000), + big.lastIndexOf("😀", 50), + big.lastIndexOf("absent"), + big.lastIndexOf("é", 0), + big.lastIndexOf("Ü"), + big.lastIndexOf(""), + big.lastIndexOf("", 10), +); +let starts = 0; +let ends = 0; +for (let i = 0; i < big.length; i += 17) { + if (big.startsWith("tëxt", i)) starts++; + if (big.endsWith("😀", i)) ends++; +} +console.log(starts, ends, big.startsWith("Ü"), big.endsWith("in, "), big.startsWith("ünï"), big.endsWith("in, ", big.length - 1)); + +// normalize. +const samples = [ + "plain ascii", + "précomposé café naïve", + "décomposé café", + "日本語のテキスト 한국어 문장", + "각 각", + "ｆｕｌｌｗｉｄｔｈ ｶﾀｶﾅ fi ① ℌ ²", + "ậ ậ ͅ", + "Ω Å Å Ω K", + "が が か゚", + "emoji 😀 and \ud800 lone", + unit, +]; +for (const sample of samples) { + const forms = ["NFC", "NFD", "NFKC", "NFKD"].map((form) => sample.normalize(form)); + console.log( + forms.map((text) => `${text.length}:${digest(text)}`).join(" "), + forms.map((text) => text === sample).join(","), + ); +} +const longNfc = big.normalize(); +const longNfd = big.normalize("NFD"); +console.log(longNfc === big, longNfc.length, longNfd.length, digest(longNfd.slice(0, 100)), longNfd.normalize("NFC") === big); +try { + "x".normalize("NFX"); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/corpus/8014-foreign-element-callbacks.ts b/tests/corpus/8014-foreign-element-callbacks.ts new file mode 100644 index 0000000000..188a3119de --- /dev/null +++ b/tests/corpus/8014-foreign-element-callbacks.ts @@ -0,0 +1,118 @@ +// Callbacks over a number[] or string[] that holds elements written +// through an unknown[] view, when the callback never reads the element: +// it declares no element parameter, or never references the one it +// declares. The methods visit every element as in Node, and filter keeps +// a written element as it is. + +const numbers: number[] = [1, 2]; +const numbersView: unknown[] = numbers; +numbersView.push("x", undefined, { k: 1 }); +numbers.push(5); + +// Inline callbacks (fused loops). +console.log(numbers.filter(() => true)); +console.log(numbers.filter((_, i) => i % 2 === 0)); +console.log(numbers.map(() => 0)); +console.log(numbers.map((_, i) => i * 10)); +console.log(numbers.some(() => false), numbers.every((_v) => true)); +console.log(numbers.findIndex((_, i) => i === 2), numbers.findLastIndex((_, i) => i < 3)); +console.log(numbers.find(() => false), numbers.findLast((_, i) => i === 0)); +console.log(numbers.reduce((s) => s + 1, 0), numbers.reduce((s, v) => s + 1, 100)); +console.log(numbers.reduceRight((s, _v, i) => s + i, 0)); +let visits = 0; +numbers.forEach((_, i) => { + visits += i; +}); +console.log(visits); +console.log(numbers.filter((_, i) => i > 1).map((_, i) => i)); +console.log(numbers.flatMap(() => [7]), numbers.flatMap((_, i) => [i, -i])); +console.log(numbers.filter(() => true).length, typeof (numbers.filter(() => true) as unknown[])[2]); + +// Callback values (shared helpers). +const always = (): boolean => true; +const never = (): boolean => false; +const count = (s: number): number => s + 1; +console.log(numbers.filter(always)); +console.log(numbers.some(never), numbers.every(always), numbers.find(never)); +console.log(numbers.findIndex(never), numbers.reduce(count, 0)); +console.log(numbers.map(always)); + +// A kept written element is the same value in the result. +const kept = numbers.filter(always) as unknown[]; +console.log(kept[2] === "x", kept[3] === undefined, 3 in kept, kept[4], kept.length); + +// string[] the same way. +const words: string[] = ["a", "b"]; +const wordsView: unknown[] = words; +wordsView.push(7, null); +console.log(words.filter(() => true), words.map((_, i) => `#${i}`)); +console.log(words.filter((_w, i) => i !== 1), words.reduce((s) => s + 1, 0)); + +// Holes stay holes and are skipped as in Node. +const sparse: number[] = [1]; +const sparseView: unknown[] = sparse; +sparseView[3] = "z"; +console.log(sparse.filter(() => true), sparse.map((_, i) => i), sparse.findIndex((_, i) => i === 1)); + +// A parameter used only through a shorthand property is still read. +const plain = [1, 2, 3]; +console.log(plain.map((v) => ({ v })), plain.reduce((acc, v) => ({ sum: acc.sum + v, v }), { sum: 0, v: 0 })); +console.log(plain.filter((v) => [{ v }].length > 0 && v > 1)); + +// A callback that ignores its element still receives the element as it was +// before the callback ran: filter keeps it, and find returns it, even when +// the callback overwrites or removes the slot. +const changing: number[] = [1, 2, 3]; +const changingView: unknown[] = changing; +changingView.push("x"); +let at = 0; +console.log( + changing.filter(() => { + changingView[at] = at === 1 ? "w" : 100 + at; + at++; + return true; + }), + changing, +); +const found: number[] = [1, 2]; +(found as unknown[]).push("y"); +let step = 0; +console.log( + found.find(() => { + found[step] = 50; + step++; + return step === 2; + }), + found, +); +const shrinking: number[] = [4, 5]; +(shrinking as unknown[]).push("z", 6); +console.log( + shrinking.findLast(() => { + shrinking.length = 1; + return true; + }), + shrinking.length, +); +const cut: number[] = [7, 8, 9]; +(cut as unknown[]).push("q"); +console.log( + cut.filter((_v, i) => { + if (i === 1) cut.length = 1; + return true; + }), + cut.length, +); + +// Arrays proven to hold no undefined keep their elements in filter results +// when the predicate ignores them. +const dense = [1, 2, 3, 4]; +let calls = 0; +console.log( + dense.filter((_v, i) => { + calls++; + return i % 2 === 0; + }), + calls, + dense.filter((_v, i) => i > 1), +); diff --git a/tests/corpus/8332-worker-shared-counted-loops.ts b/tests/corpus/8332-worker-shared-counted-loops.ts new file mode 100644 index 0000000000..010531c139 --- /dev/null +++ b/tests/corpus/8332-worker-shared-counted-loops.ts @@ -0,0 +1,128 @@ +// Counted loops over typed arrays backed by a SharedArrayBuffer in a program +// with workers: loops to a length and to a bound, ascending, descending and +// by two, reads and writes at the counter and next to it, nested loops, and +// each element type. The worker and the main thread run the same loops over +// the same shared memory, one after the other. +import { isMainThread, parentPort, Worker, workerData } from "node:worker_threads"; + +interface Shared { + bytes: Uint8Array; + shorts: Int16Array; + words: Uint32Array; + ints: Int32Array; + floats: Float32Array; + doubles: Float64Array; +} + +function codes(a: Uint8Array): void { + for (let i = 0; i < a.length; ++i) { + const x = a[i]; + if (x === 97) a[i] = 0; + else if (x === 116) a[i] = 1; + else if (x === 99) a[i] = 2; + else a[i] = 3; + } +} + +function checksum(a: Uint8Array): number { + let t = 0; + for (let i = 0; i < a.length; i++) t = (t * 31 + a[i]!) % 1000003; + return t; +} + +function shiftLeft(a: Uint8Array): void { + for (let i = 0; i < a.length - 1; i++) a[i] = a[i + 1]!; +} + +function descend(a: Int16Array, n: number): number { + let t = 0; + for (let i = n - 1; i >= 0; i--) { + a[i] = a[i]! - i * 3; + t += a[i]!; + } + return t; +} + +function everyOther(a: Uint32Array): number { + let t = 0; + for (let i = 0; i < a.length; i += 2) { + a[i] = a[i]! * 2 + 4000000000; + t += a[i]! % 1000; + } + return t; +} + +function grid(a: Int32Array, n: number): number { + let t = 0; + for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) t += a[i]! * a[j]! - j; + return t; +} + +function mix(f: Float32Array, d: Float64Array, n: number): number { + for (let i = 0; i < n; i++) { + f[i] = f[i]! * 0.5 + i; + d[i] = d[i]! + f[i]! / 3; + } + let t = 0; + for (let i = 0; i < n; i++) t += d[i]!; + return Math.round(t * 1000) / 1000; +} + +function fill(s: Shared): void { + const letters = [97, 116, 99, 103]; + let seed = 7; + for (let i = 0; i < s.bytes.length; i++) { + seed = (seed * 1103515245 + 12345) % 2147483648; + s.bytes[i] = letters[(seed >> 16) % 4]!; + } + for (let i = 0; i < s.shorts.length; i++) s.shorts[i] = (i * 37) % 2000 - 1000; + for (let i = 0; i < s.words.length; i++) s.words[i] = i * 2654435761; + for (let i = 0; i < s.ints.length; i++) s.ints[i] = (i % 7) - 3; + for (let i = 0; i < s.floats.length; i++) s.floats[i] = i / 4; + for (let i = 0; i < s.doubles.length; i++) s.doubles[i] = i * 1.25; +} + +function run(where: string, s: Shared): string { + codes(s.bytes); + const coded = checksum(s.bytes); + shiftLeft(s.bytes); + const shifted = checksum(s.bytes); + const down = descend(s.shorts, s.shorts.length); + const other = everyOther(s.words); + const cells = grid(s.ints, 40); + const mixed = mix(s.floats, s.doubles, s.floats.length); + return `${where}: ${coded} ${shifted} ${down} ${other} ${cells} ${mixed} ${s.bytes[0]} ${s.words[2]}`; +} + +function views(buffers: SharedArrayBuffer[]): Shared { + return { + bytes: new Uint8Array(buffers[0]!), + shorts: new Int16Array(buffers[1]!), + words: new Uint32Array(buffers[2]!), + ints: new Int32Array(buffers[3]!), + floats: new Float32Array(buffers[4]!), + doubles: new Float64Array(buffers[5]!), + }; +} + +if (isMainThread) { + const buffers = [ + new SharedArrayBuffer(10007), + new SharedArrayBuffer(2 * 515), + new SharedArrayBuffer(4 * 333), + new SharedArrayBuffer(4 * 40), + new SharedArrayBuffer(4 * 129), + new SharedArrayBuffer(8 * 129), + ]; + const shared = views(buffers); + fill(shared); + const worker = new Worker(new URL(import.meta.url), { workerData: buffers }); + worker.on("message", (line: string) => { + console.log(line); + console.log(run("main", shared)); + fill(shared); + console.log(run("main again", shared)); + }); +} else { + parentPort!.postMessage(run("worker", views(workerData as SharedArrayBuffer[]))); +} diff --git a/tests/corpus/constant-choice-chains.ts b/tests/corpus/constant-choice-chains.ts new file mode 100644 index 0000000000..02757d3e5b --- /dev/null +++ b/tests/corpus/constant-choice-chains.ts @@ -0,0 +1,128 @@ +// if/else-if chains that store one of several literals to one typed-array +// element, chosen by comparing one number with integer literals: the +// subject's values that no case names, repeated and negative cases, +// results that do not fit the element, and chains that keep their arms. + +function bases(arr: Uint8Array): void { + for (let i = 0; i < arr.length; ++i) { + const x = arr[i]; + if (x === 97) arr[i] = 0; + else if (x === 116) arr[i] = 1; + else if (x === 99) arr[i] = 2; + else arr[i] = 3; + } +} + +const sequence = new Uint8Array(64); +for (let i = 0; i < sequence.length; i++) sequence[i] = "actgACTGnx"[i % 10]!.charCodeAt(0); +bases(sequence); +console.log("bases", sequence.join("")); + +// A number subject: NaN, -0, fractions, infinities and values far outside the cases. +const subjects = [0, -0, 1, 2, 3, 4, -1, 0.5, 2.0000001, NaN, Infinity, -Infinity, 2 ** 40, -(2 ** 40), 1e300]; +function classify(out: Int16Array, at: number, v: number): void { + if (v === 0) out[at] = 10; + else if (v === 1) out[at] = -20; + else if (v === 2) out[at] = 30; + else if (v === 1) out[at] = 99; // never reached + else if (v === -1) out[at] = 40; + else out[at] = -50; +} +const classes = new Int16Array(subjects.length); +for (let i = 0; i < subjects.length; i++) classify(classes, i, subjects[i]!); +console.log("classes", classes.join(" ")); + +// The literal on the left, and results the element converts: 300 wraps in a +// Uint8Array, -1 is 255, 1.5 truncates, and a clamped array rounds and clamps. +function convert(u8: Uint8Array, c: Uint8ClampedArray, f: Float64Array, i: Int8Array, v: number): void { + if (5 === v) u8[0] = 300; + else if (6 === v) u8[0] = -1; + else if (7 === v) u8[0] = 1.5; + else u8[0] = 8589934599; // 2 ** 33 + 7 + if (v === 5) c[0] = 300; + else if (v === 6) c[0] = -1; + else if (v === 7) c[0] = 2.5; + else c[0] = 3.5; + if (v === 5) f[0] = 0.25; + else if (v === 6) f[0] = -0; + else if (v === 7) f[0] = 1e300; + else f[0] = NaN; + if (v === 5) i[0] = 128; + else if (v === 6) i[0] = -129; + else if (v === 7) i[0] = -128; + else i[0] = 127; +} +for (const v of [5, 6, 7, 8, NaN]) { + const u8 = new Uint8Array(1); + const c = new Uint8ClampedArray(1); + const f = new Float64Array(1); + const i = new Int8Array(1); + convert(u8, c, f, i, v); + console.log("convert", v, u8[0], c[0], Object.is(f[0], -0) ? "-0" : f[0], i[0]); +} + +// An index past the end ignores the store, whatever the chain chooses. +const short = new Uint8Array(2); +for (let k = 0; k < 4; k++) { + const v = k + 1; + if (v === 1) short[k] = 11; + else if (v === 2) short[k] = 22; + else if (v === 3) short[k] = 33; + else short[k] = 44; +} +console.log("short", short.join(",")); + +// The index is the compared local. Inside each arm the range analysis +// proves its value there; on the final else's path it is past the end, also +// past 2^32 and below -2^32, and the store is ignored. +function indexed(flag: boolean): string { + const a = new Uint8Array(4); + let j = 0; + if (flag) j = 4294967297; + if (j === 0) a[j] = 11; + else if (j === 1) a[j] = 22; + else if (j === 2) a[j] = 33; + else a[j] = 44; + return a.join(","); +} +function indexedLoop(): string { + const a = new Uint8Array(8); + for (let k = 0; k < 4; k++) { + let j = 1; + if (k === 3) j = 5; + else if (k === 2) j = 4294967298; + else if (k === 1) j = -4294967294; + if (j === 1) a[j] = 11; + else if (j === 2) a[j] = 22; + else if (j === 3) a[j] = 33; + else a[j] = 44; + } + return a.join(","); +} +console.log("indexed", indexed(false), indexed(true), indexedLoop()); + +// Chains that keep their arms: an arm with a second statement, no final +// else, a different index, and a subject that is not a local. +let calls = 0; +function count(): number { + calls++; + return 9; +} +const kept = new Uint8Array(8); +for (let k = 0; k < 8; k++) { + const v = k % 5; + if (v === 0) kept[k] = 1; + else if (v === 1) { + kept[k] = 2; + calls += 100; + } else if (v === 2) kept[k] = 3; + else kept[k] = 4; + if (v === 0) kept[0] += 1; + else if (v === 1) kept[0] += 2; + else if (v === 2) kept[0] += 3; + if (count() === v) kept[k] = 7; + else if (v === 3) kept[(k + 1) % 8] = 5; + else if (v === 4) kept[k] = 6; + else kept[k] = kept[k]!; +} +console.log("kept", kept.join(","), calls); diff --git a/tests/corpus/dyn-cast-identity-fresh.ts b/tests/corpus/dyn-cast-identity-fresh.ts new file mode 100644 index 0000000000..f0ef0fd53c --- /dev/null +++ b/tests/corpus/dyn-cast-identity-fresh.ts @@ -0,0 +1,21 @@ +// Casts of values nothing else holds yet: a parse result and a response +// body. They convert outright; the results are ordinary records. + +type Point = { x: number; y: number }; +type Doc = { title: string; points: Point[] }; + +const parsed = JSON.parse('{"title":"t","points":[{"x":1,"y":2},{"x":3,"y":4}]}') as Doc; +parsed.points[0]!.x = 10; +parsed.points.push({ x: 5, y: 6 }); +console.log("parse", parsed.title, parsed.points.length, JSON.stringify(parsed.points[0])); + +const again = JSON.parse('{"x":1,"y":2}') as Point; +const other = JSON.parse('{"x":1,"y":2}') as Point; +console.log("separate parses", again === other, again.x === other.x); + +async function main(): Promise { + const body = (await new Response('{"title":"r","points":[{"x":7,"y":8}]}').json()) as Doc; + body.points[0]!.y = 80; + console.log("response", body.title, JSON.stringify(body.points)); +} +main(); diff --git a/tests/corpus/dyn-cast-identity-holders.ts b/tests/corpus/dyn-cast-identity-holders.ts new file mode 100644 index 0000000000..55b311462f --- /dev/null +++ b/tests/corpus/dyn-cast-identity-holders.ts @@ -0,0 +1,62 @@ +// Checked casts of dynamic values held by parameters, fields, closures and +// containers keep one identity with the holder's value. + +type Point = { x: number; y: number }; + +// A parameter: the caller's value and the cast result are one object. +function asPoint(value: unknown): Point { + return value as Point; +} +const shared: unknown = JSON.parse('{"x":1,"y":2}'); +const fromParam = asPoint(shared); +fromParam.x = 5; +console.log("parameter", (shared as Point).x, asPoint(shared) === fromParam); + +// A field of a class instance. +class Holder { + data: unknown; + constructor(data: unknown) { + this.data = data; + } +} +const holder = new Holder(JSON.parse('{"x":3,"y":4}')); +const fromField = holder.data as Point; +fromField.y = 40; +console.log("field", JSON.stringify(holder.data), (holder.data as Point) === fromField); + +// A value captured by a closure. +const captured: unknown = JSON.parse('{"x":0,"y":0}'); +const bump = (): void => { + (captured as Point).x++; +}; +bump(); +bump(); +console.log("closure", (captured as Point).x); + +// Elements of an unknown[] container. +const list: unknown[] = [JSON.parse('{"x":1,"y":1}'), JSON.parse('{"x":2,"y":2}')]; +const second = list[1] as Point; +second.x = 20; +console.log("container", JSON.stringify(list), (list[1] as Point) === second); + +// A union target with a record arm. +const maybe: unknown = JSON.parse('{"x":7,"y":8}'); +const viaUnion = maybe as Point | undefined; +if (viaUnion) viaUnion.x = 70; +console.log("union", (maybe as Point).x); + +// A record whose field is an array, mutated through both casts. +type Bag = { items: number[] }; +const bagRaw: unknown = JSON.parse('{"items":[1]}'); +const bag = bagRaw as Bag; +const items = (bagRaw as { items: unknown }).items as number[]; +items.push(2); +bag.items.push(3); +console.log("record array", bag.items.join(","), items.length, bag.items === items); + +// A native value boxed as unknown and cast back to its own type. +const nativeList: number[] = [1, 2]; +const boxedList: unknown = nativeList; +const backList = boxedList as number[]; +backList.push(3); +console.log("native array", nativeList.length, backList === nativeList); diff --git a/tests/corpus/dyn-cast-identity-workers.ts b/tests/corpus/dyn-cast-identity-workers.ts new file mode 100644 index 0000000000..9ee2e42b94 --- /dev/null +++ b/tests/corpus/dyn-cast-identity-workers.ts @@ -0,0 +1,23 @@ +// Structured clones a worker receives: workerData and message payloads. +// Two casts of workerData are one object; a message payload converts once. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +type Job = { name: string; point: { x: number; y: number }; tags: string[] }; + +if (isMainThread) { + const worker = new Worker(new URL(import.meta.url), { + workerData: { name: "job", point: { x: 1, y: 2 }, tags: ["a"] }, + }); + worker.on("message", (message: unknown) => { + const reply = message as Job; + reply.tags.push("main"); + console.log("reply", reply.name, reply.point.x, reply.tags.join(",")); + }); +} else { + const job = workerData as Job; + const same = workerData as Job; + job.point.x = 10; + same.tags.push("worker"); + console.log("worker", same.point.x, job.tags.join(","), job === same); + parentPort!.postMessage(job); +} diff --git a/tests/corpus/dyn-cast-identity.ts b/tests/corpus/dyn-cast-identity.ts new file mode 100644 index 0000000000..5ac30c52f3 --- /dev/null +++ b/tests/corpus/dyn-cast-identity.ts @@ -0,0 +1,71 @@ +// A checked cast of a dynamic value to a record or array (`value as T`) +// keeps the value's identity: every cast of the same source, and the source +// itself, read and write one object. + +type Point = { x: number; y: number }; + +// Two casts of one parsed object alias each other and the source. +const raw: unknown = JSON.parse('{"x":1,"y":2}'); +const a = raw as Point; +const b = raw as Point; +a.x = 10; +b.y = 20; +console.log("aliasing", a.x, a.y, b.x, b.y); +console.log("identity", a === b, (raw as Point) === a); +console.log("source", JSON.stringify(raw)); + +// Nested objects keep their identity through the outer and inner casts. +type Outer = { name: string; inner: { v: number } }; +const nestedRaw: unknown = JSON.parse('{"name":"o","inner":{"v":1}}'); +const outer = nestedRaw as Outer; +const inner = (nestedRaw as { inner: unknown }).inner as { v: number }; +inner.v = 2; +console.log("nested", outer.inner.v, outer.inner === inner); +outer.inner.v = 3; +console.log("nested back", inner.v); + +// Arrays: pushes and element writes show through every cast. +const listRaw: unknown = JSON.parse("[1,2]"); +const first = listRaw as number[]; +const second = listRaw as number[]; +first.push(3); +second[0] = 9; +console.log("array", first.length, second.length, first[0], first === second, JSON.stringify(listRaw)); + +// Arrays of records. +type Item = { id: number; tags: string[] }; +const itemsRaw: unknown = JSON.parse('[{"id":1,"tags":["a"]},{"id":2,"tags":[]}]'); +const items = itemsRaw as Item[]; +const again = itemsRaw as Item[]; +items[0]!.tags.push("b"); +again[1]!.id = 7; +console.log("array of records", again[0]!.tags.join(","), items[1]!.id, items[0] === again[0]); + +// The source stays an ordinary object for untyped reads and writes. +const bagRaw: unknown = JSON.parse('{"y":2,"x":1,"extra":"e"}'); +const point = bagRaw as Point; +const bag = bagRaw as Record; +bag.z = 3; +console.log("added", JSON.stringify(bagRaw), Object.keys(bag).join(",")); +delete bag.extra; +console.log("deleted", JSON.stringify(bagRaw), JSON.stringify(point)); +for (const k in bag) console.log("key", k, bag[k]); +point.x = 5; +console.log("typed write", bag.x); + +// A native record boxed as unknown comes back as itself. +const native: Point = { x: 1, y: 2 }; +const boxed: unknown = native; +const unboxed = boxed as Point; +unboxed.x = 10; +console.log("native", native.x, unboxed === native); + +// A fresh parse result has no other holder. +const fresh = JSON.parse('{"x":4,"y":5}') as Point; +console.log("fresh", fresh.x + fresh.y); + +// A frozen source keeps working as a copy. +const frozen: unknown = Object.freeze(JSON.parse('{"x":1,"y":2}')); +const held = [frozen]; +const thawed = frozen as Point; +console.log("frozen", thawed.x, held.length, Object.isFrozen(frozen)); diff --git a/tests/corpus/dyn-literal-identity.ts b/tests/corpus/dyn-literal-identity.ts new file mode 100644 index 0000000000..915b8f73eb --- /dev/null +++ b/tests/corpus/dyn-literal-identity.ts @@ -0,0 +1,83 @@ +// Object and array literals in untyped contexts (an `unknown` or `any` +// slot, an `unknown[]` element, a thrown value, an `unknown`/`any` +// parameter) keep one identity through later checked casts. + +type Counter = { count: number }; +type Item = { name: string; tags: string[] }; + +// The original example: a write through the cast shows through the holder. +const holder: any = { count: 1 }; +const item = holder as Counter; +item.count++; +console.log("any holder", holder.count, item === holder); + +const box: unknown = { count: 10 }; +const viewA = box as Counter; +const viewB = box as Counter; +viewA.count += 5; +console.log("unknown holder", viewB.count, viewA === viewB, JSON.stringify(box)); + +// Elements of an unknown[] array literal. +const values: unknown[] = [ + { name: "a", tags: ["x"] }, + { name: "b", tags: [] }, +]; +const second = values[1] as Item; +second.tags.push("y"); +console.log("unknown[] element", JSON.stringify(values), (values[1] as Item) === second); + +// Nested literals inside an untyped literal. +const tree: unknown = { name: "root", children: [{ name: "leaf", tags: ["t"] }] }; +const root = tree as { name: string; children: Item[] }; +const leaf = (tree as { children: unknown[] }).children[0] as Item; +leaf.tags.push("u"); +console.log("nested", root.children[0]!.tags.join(","), root.children[0] === leaf); + +// A thrown literal. +try { + throw { code: "E_BUSY", attempts: [1] }; +} catch (error) { + const failure = error as { code: string; attempts: number[] }; + failure.attempts.push(2); + console.log("thrown", failure.code, (error as { attempts: number[] }).attempts.length, failure === error); +} + +// Literals passed to unknown and any parameters. +function remember(value: unknown): Counter { + const counter = value as Counter; + counter.count *= 2; + return value as Counter; +} +console.log("unknown parameter", remember({ count: 4 }).count); + +function relabel(value: any): Item { + const target = value as Item; + target.name = target.name.toUpperCase(); + return value; +} +const relabeled = relabel({ name: "pkg", tags: [] }); +console.log("any parameter", relabeled.name, relabeled.tags.length); + +// A literal returned as unknown, and one stored in an unknown field. +function make(): unknown { + return { count: 0 }; +} +const made = make(); +(made as Counter).count = 7; +console.log("returned", (made as Counter).count); + +class Slot { + value: unknown = { name: "slot", tags: ["s"] }; +} +const slot = new Slot(); +(slot.value as Item).tags.push("t"); +console.log("field", JSON.stringify(slot.value)); + +// Untyped literals keep JavaScript's own key order and extra keys. +const extra: unknown = { name: "e", z: 1, tags: [], a: 2 }; +const typed = extra as Item; +typed.tags.push("k"); +console.log("extra keys", JSON.stringify(extra), Object.keys(typed).join(",")); + +// Literals handed to library calls are unchanged. +console.log(JSON.stringify({ count: 3 }), [{ n: 1 }].length); diff --git a/tests/corpus/fs-readdir-recursive.ts b/tests/corpus/fs-readdir-recursive.ts new file mode 100644 index 0000000000..39fd7f5ccd --- /dev/null +++ b/tests/corpus/fs-readdir-recursive.ts @@ -0,0 +1,52 @@ +// readdirSync(path, { recursive: true }) lists every entry below the path, +// relative to it: the path's own entries first, then each directory's in the +// order its entry was listed. A symlink to a directory is followed. + +import { mkdirSync, mkdtempSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = mkdtempSync(join(tmpdir(), "readdir-recursive-")); +writeFileSync(join(root, "a.txt"), "a"); +mkdirSync(join(root, "b", "d"), { recursive: true }); +writeFileSync(join(root, "b", "c.txt"), "c"); +writeFileSync(join(root, "b", "d", "e.txt"), "e"); +mkdirSync(join(root, "empty")); +mkdirSync(join(root, "z")); +writeFileSync(join(root, "z", "y.txt"), "y"); +symlinkSync(join(root, "b", "d"), join(root, "link")); + +console.log(readdirSync(root, { recursive: true }).join(" ")); +console.log(readdirSync(root, { recursive: false }).join(" ")); +console.log(readdirSync(`${root}/b/`, { recursive: true }).join(" ")); + +let deep = false; +for (const flag of [false, true]) { + deep = flag; + console.log(deep, readdirSync(join(root, "z"), { recursive: deep }).length); +} +console.log(readdirSync(join(root, "empty"), { recursive: true }).length); + +// A flag expression typed `true` or `false` still evaluates, once. +let probes = 0; +function yes(): true { + probes++; + return true; +} +function no(): false { + probes++; + return false; +} +console.log( + readdirSync(join(root, "b"), { recursive: yes() }).length, + readdirSync(join(root, "b"), { recursive: no() }).length, + probes, +); + +try { + readdirSync(join(root, "missing"), { recursive: true }); +} catch (e) { + console.log((e as NodeJS.ErrnoException).code); +} + +rmSync(root, { recursive: true }); diff --git a/tests/corpus/localized-failure-cycles.ts b/tests/corpus/localized-failure-cycles.ts new file mode 100644 index 0000000000..2db499707f --- /dev/null +++ b/tests/corpus/localized-failure-cycles.ts @@ -0,0 +1,29 @@ +// Localized failure, stage L2: reference cycles through dynamic slots are +// collected (instance -> dynamic field -> capsule -> instance). +type Peer = Link | Map string> | AbortSignal | (() => number); + +class Link { + id: number; + peer: Peer | undefined; + constructor(id: number) { + this.id = id; + this.peer = undefined; + } + peerId(): number { + const p = this.peer; + return p instanceof Link ? p.id : -1; + } +} + +let total = 0; +for (let i = 0; i < 20000; i++) { + const a = new Link(i); + const b = new Link(i + 1); + a.peer = b; + b.peer = a; + total += a.peerId() - b.peerId(); +} +const keep = new Link(7); +keep.peer = () => keep.id * 2; +const f = keep.peer; +console.log(total, typeof f === "function" ? f() : 0, keep.peerId()); diff --git a/tests/corpus/localized-failure-declarations.ts b/tests/corpus/localized-failure-declarations.ts new file mode 100644 index 0000000000..c4acdfbaa7 --- /dev/null +++ b/tests/corpus/localized-failure-declarations.ts @@ -0,0 +1,64 @@ +// Localized failure, stage L2: fields, parameters and returns whose +// declared type has no static lowering become checked dynamic slots, so +// their class or function still compiles. +interface ToolDefinition { + name: string; + execute(id: string, params: any): string; +} +type Progress = (done: number, total: number) => void; +type ProgressOrSignal = Progress | AbortSignal | Map string> | Set; + +class Session { + // A field: the class registers, the field holds the array by reference. + private tools: ToolDefinition[] = []; + state: any = { count: 0 }; + readonly label: string; + constructor(label: string) { + this.label = label; + } + add(tool: ToolDefinition): void { + this.tools.push(tool); + } + run(id: string, params: unknown): string[] { + const out: string[] = []; + for (const tool of this.tools) out.push(tool.execute(id, params)); + this.state.count = this.state.count + 1; + return out; + } + count(): number { + return this.tools.length; + } + // A return slot. + first(): ToolDefinition | ProgressOrSignal | undefined { + return this.tools[0]; + } +} + +// A parameter: the function registers and is called from typed code. +function listAll(dir: string, onProgress?: ProgressOrSignal): number { + if (typeof onProgress === "function") onProgress(1, dir.length); + else if (onProgress instanceof Set) return onProgress.size; + return dir.length; +} + +// A base class with a dynamic parameter, and an override. +class Base { + notify(target: ProgressOrSignal | undefined): string { + return target === undefined ? "base:none" : "base:" + typeof target; + } +} +class Derived extends Base { + override notify(target: ProgressOrSignal | undefined): string { + return "derived/" + super.notify(target); + } +} + +const echo: ToolDefinition = { name: "echo", execute: (id, params) => `${id}:${JSON.stringify(params)}` }; +const s = new Session("main"); +s.add(echo); +s.add({ name: "len", execute: (id) => String(id.length) }); +console.log(s.label, s.count(), s.run("abc", { k: 1 }).join(" | "), s.state.count); +console.log("same tool", s.first() === echo); +console.log(listAll("hello"), listAll("hello", (d, t) => console.log("progress", d, t)), listAll("x", new Set([1, 2]))); +const objs: Base[] = [new Base(), new Derived()]; +for (const o of objs) console.log(o.notify(undefined), o.notify((a, b) => a + b)); diff --git a/tests/corpus/localized-failure-forward-capture.ts b/tests/corpus/localized-failure-forward-capture.ts new file mode 100644 index 0000000000..ba43a037d7 --- /dev/null +++ b/tests/corpus/localized-failure-forward-capture.ts @@ -0,0 +1,19 @@ +// Localized failure, stage L1: a statement that declares a binding an +// earlier closure captures, and then a binding whose declared type has no +// static lowering, still initializes the captured binding. +type Callbacks = ((done: number) => void) | AbortSignal | Map string> | Set; + +function run(): void { + const read = (): number => total; + const total = 3, + hook: Callbacks = new Set([1, 2]); + console.log(read(), hook instanceof Set, hook instanceof Set && hook.size); + let late = (): string => label; + let label = "first", + other: Callbacks = new Set([3]); + console.log(late(), other instanceof Set); + label = "second"; + late = (): string => label + "!"; + console.log(late()); +} +run(); diff --git a/tests/corpus/localized-failure-function-slots.ts b/tests/corpus/localized-failure-function-slots.ts new file mode 100644 index 0000000000..0a7fdf641f --- /dev/null +++ b/tests/corpus/localized-failure-function-slots.ts @@ -0,0 +1,28 @@ +// Localized failure, stage L2: parameters and returns of arrows, function +// expressions and generic instances become checked dynamic slots. +type Hook = Map string> | AbortSignal | (() => number) | Set; + +interface Model { + id: T; + hook: Hook; +} + +function retry(label: string, request: () => Promise | Hook): T | string { + const r = request(); + return typeof r === "function" ? label + ":" + String(r()) : label + ":other"; +} + +function describe(model: Model): string { + return `${String(model.id)}/${model.hook instanceof Set ? model.hook.size : typeof model.hook}`; +} + +const run = (hook: Hook, n: number): string => (typeof hook === "function" ? String(hook() + n) : "set"); +const make = function (size: number): Hook { + return new Set(Array.from({ length: size }, (_, i) => `k${i}`)); +}; + +console.log(run(() => 40, 2), run(new Set(["a"]), 1)); +const h = make(3); +console.log(h instanceof Set ? h.size : -1); +console.log(retry("a", () => () => 7), retry("b", () => new Set())); +console.log(describe({ id: 5, hook: new Set(["x", "y"]) }), describe({ id: "s", hook: () => 1 })); diff --git a/tests/corpus/localized-failure-held-values.ts b/tests/corpus/localized-failure-held-values.ts new file mode 100644 index 0000000000..ee52498c1a --- /dev/null +++ b/tests/corpus/localized-failure-held-values.ts @@ -0,0 +1,70 @@ +// Localized failure, stage L1: values of every kind held in a local or +// module global whose declared type has no static lowering keep their +// identity and behavior. +class Point { + x: number; + y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } + norm(): number { + return Math.abs(this.x) + Math.abs(this.y); + } +} +interface Cfg { + name: string; + depth: number; +} +type Anything = + | Point + | Cfg + | number[] + | Map string> + | AbortSignal + | (() => number) + | Promise; + +const p = new Point(3, -4); +const cfg: Cfg = { name: "a", depth: 1 }; +const xs = [1, 2, 3]; +let counter = 0; +const tick = (): number => ++counter; + +// A module global holding a class instance. +let held: Anything = p; +if (held instanceof Point) { + console.log("point", held.norm(), held === p); + held.x = 10; +} +console.log("typed alias", p.x, p.norm()); + +// A record: mutation through the hole is visible in the typed alias. +held = cfg; +if (!(held instanceof Point) && !Array.isArray(held) && typeof held === "object" && "depth" in held) { + held.depth = 5; + console.log("record", held.name, held === cfg); +} +console.log("typed alias", cfg.depth, JSON.stringify(cfg), Object.keys(cfg).join(",")); + +// An array: pushes through the hole show in typed code. +held = xs; +if (Array.isArray(held)) { + held.push(4); + console.log("array", held.length, held === xs); +} +console.log("typed alias", xs.length, xs.join("+")); + +function main(): void { + // A local holding a closure, called through the hole. + let local: Anything = tick; + if (typeof local === "function") { + console.log("closure", local(), local(), counter, local === tick); + } + // A promise, awaited through the hole. + local = Promise.resolve(42); + if (local instanceof Promise) { + local.then((v) => console.log("promise", v)); + } +} +main(); diff --git a/tests/corpus/localized-failure-locals.ts b/tests/corpus/localized-failure-locals.ts new file mode 100644 index 0000000000..418b86d877 --- /dev/null +++ b/tests/corpus/localized-failure-locals.ts @@ -0,0 +1,43 @@ +// Localized failure, stage L1: locals, loop variables and module globals +// whose declared type has no static lowering hold checked-dynamic values. +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +const report: Progress = (done, total) => console.log(`progress ${done}/${total}`); +const seen = new Set([1, 2, 3]); + +// A module global, written with typed values of several arms. +let current: Callbacks | undefined = report; + +function describe(label: string): void { + // A local of the same declared type. + const local: Callbacks | undefined = current; + if (local === undefined) { + console.log(label, "none"); + } else if (typeof local === "function") { + local(1, 2); + console.log(label, "function", local === report); + } else if (local instanceof Set) { + console.log(label, "set", local.size, local.has(2), local === seen); + local.add(4); + } else { + console.log(label, "other"); + } +} + +describe("first"); +current = seen; +describe("second"); +console.log("typed alias sees the write", seen.size, seen.has(4)); +current = undefined; +describe("third"); + +// A loop variable over an array whose element type does not compile. +const all: (Callbacks | number)[] = [report, seen, 7]; +for (const item of all) { + if (typeof item === "number") console.log("number", item + 1); + else if (typeof item === "function") item(3, 4); + else console.log("object", item instanceof Set); +} +all.push(9); +console.log(all.length); diff --git a/tests/corpus/localized-failure-pi-shapes.ts b/tests/corpus/localized-failure-pi-shapes.ts new file mode 100644 index 0000000000..9879606eb3 --- /dev/null +++ b/tests/corpus/localized-failure-pi-shapes.ts @@ -0,0 +1,37 @@ +// Localized failure, stage L2: pi-shaped declarations. +interface Token { + text: string; +} +interface AgentMessage { + role: "user" | "assistant"; + content: string[]; +} +type CreateRuntime = (cwd: string) => Promise<{ session: Session; extra: Map string> | AbortSignal }>; + +class Session { + messages: AgentMessage[] = []; + // A field whose union mentions a stdlib type with no lowering: the class + // registers; constructing a WeakRef would still be refused. + cachedTokens?: WeakRef<{ source: string; tokens: Token[] }> | undefined; + rendererState: any = {}; + effectState: Record = {}; + private readonly createRuntime: CreateRuntime | undefined; + constructor(createRuntime: CreateRuntime | undefined) { + this.createRuntime = createRuntime; + } + hasRuntime(): boolean { + return this.createRuntime !== undefined; + } + clear(): AgentMessage[] { + // An inferred return over a spread clone. + return this.messages.map((message) => ({ ...message, content: [] })); + } +} + +const s = new Session(undefined); +const withRuntime = new Session(async (cwd) => ({ session: s, extra: new Map() })); +s.messages.push({ role: "user", content: ["hi"] }, { role: "assistant", content: ["yo", "!"] }); +s.rendererState.mode = "fast"; +s.effectState["k"] = 3; +console.log(s.hasRuntime(), s.cachedTokens === undefined, s.rendererState.mode, s.effectState["k"]); +console.log(JSON.stringify(s.clear()), s.messages[1]!.content.length, withRuntime.hasRuntime()); diff --git a/tests/corpus/localized-failure-registered-statics.ts b/tests/corpus/localized-failure-registered-statics.ts new file mode 100644 index 0000000000..9ca49d4f26 --- /dev/null +++ b/tests/corpus/localized-failure-registered-statics.ts @@ -0,0 +1,46 @@ +// Localized failure, stage L2: a class that registers with dynamic slots +// runs its static field initializers where its statement evaluates, and a +// generic class's static methods compile once it registers. +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +function sideEffect(label: string): number { + console.log("static init", label); + return label.length; +} + +// Registers at its statement: the initializers run before the next line. +class Registry { + state: any = { n: 1 }; + static count = sideEffect("registry"); + static names: string[] = ["a", "b"]; + read(): number { + return this.state.n; + } +} + +console.log("after the class statement"); + +// Never referenced: its initializer runs all the same. +class Unused { + slot: any = null; + static ready = sideEffect("unused"); +} + +// A generic class whose static method has a dynamic parameter. +class Box { + value: T; + constructor(v: T) { + this.value = v; + } + static describe(target: Callbacks | undefined): string { + return target === undefined ? "none" : typeof target; + } + static label(): string { + return "box"; + } +} + +const r = new Registry(); +console.log(r.read(), Registry.count, Registry.names.join(",")); +console.log(Box.label(), Box.describe(undefined), Box.describe(new Set([1])), new Box(3).value); diff --git a/tests/corpus/never-narrowed-literal-argument.ts b/tests/corpus/never-narrowed-literal-argument.ts new file mode 100644 index 0000000000..a9bb5489fc --- /dev/null +++ b/tests/corpus/never-narrowed-literal-argument.ts @@ -0,0 +1,46 @@ +// A string-literal union narrowed to `never` passes to a `never` parameter: +// the exhaustiveness-check idiom over if-chains and switches compiles, and the +// unreachable call never runs. + +function unreachable(_: never): never { + throw new Error("unreachable"); +} + +type Kind = "a" | "b"; +function viaIf(kind: Kind): number { + if (kind === "a") return 1; + if (kind === "b") return 2; + return unreachable(kind); +} + +type Shape = "circle" | "square" | "triangle"; +function corners(shape: Shape): number { + switch (shape) { + case "circle": + return 0; + case "square": + return 4; + case "triangle": + return 3; + default: + return unreachable(shape); + } +} + +function asserted(kind: Kind): string { + if (kind === "a" || kind === "b") return kind.toUpperCase(); + return unreachable(kind as never); +} + +type Level = 1 | 2; +function viaNumbers(level: Level): string { + if (level === 1) return "one"; + if (level === 2) return "two"; + return unreachable(level); +} + +const kinds: Kind[] = ["a", "b"]; +console.log(kinds.map(viaIf).join(","), viaIf(Date.now() > 0 ? "a" : "b")); +const shapes: Shape[] = ["circle", "square", "triangle"]; +console.log(shapes.map(corners).join(",")); +console.log(kinds.map(asserted).join(""), viaNumbers(1), viaNumbers(2)); diff --git a/tests/corpus/rest-tuple-shorthand-property.ts b/tests/corpus/rest-tuple-shorthand-property.ts new file mode 100644 index 0000000000..d67afa443f --- /dev/null +++ b/tests/corpus/rest-tuple-shorthand-property.ts @@ -0,0 +1,23 @@ +// A rest parameter typed as a fixed tuple, read through a shorthand property +// (`{ xs }`), directly or through a const alias, is the tuple of the call's +// arguments. + +function wrap(...xs: [number]) { + return { xs }; +} +function pair(...args: [string, number]) { + const kept = args; + return { kept, size: args.length }; +} +function add(a: number, b: number): number { + return a + b; +} +function forward(...args: [number, number]): number { + return add(...args); +} + +const w = wrap(1); +console.log(w.xs[0], w.xs.length, JSON.stringify(w)); +const p = pair("a", 2); +console.log(p.kept[0], p.kept[1], p.size, JSON.stringify(p)); +console.log(forward(3, 4)); diff --git a/tests/corpus/speculative-loop-view-headers.ts b/tests/corpus/speculative-loop-view-headers.ts new file mode 100644 index 0000000000..a1b92c1d96 --- /dev/null +++ b/tests/corpus/speculative-loop-view-headers.ts @@ -0,0 +1,36 @@ +// Speculative integer loops over typed-array views held by a const local +// and by a parameter. The fast copy, the copy a failed check resumes in and +// the ordinary copy all read and write the same views: a product that +// leaves the safe integers fails the fast copy's check partway through, +// and writes past the end are ignored. +function fill(buffer: ArrayBuffer, count: number, seed: number): number { + const view = new Float64Array(buffer); + let l = seed; + for (let i = 0; i < count; i++) { + l = (l * 3877 + 29573) % 139968; + view[i] = l; + } + return l; +} + +function grow(out: Int32Array, count: number, start: number): number { + let x = start; + for (let i = 0; i < count; i++) { + x = x * 3 + 1; + out[i] = x % 1000; + } + return x; +} + +function sum(view: Float64Array | Int32Array): number { + let s = 0; + for (const v of view) s += v; + return s; +} + +const buffer = new ArrayBuffer(8 * 64); +console.log(fill(buffer, 64, 42), sum(new Float64Array(buffer))); +console.log(fill(buffer, 100, 7), new Float64Array(buffer)[63]); +const out = new Int32Array(40); +console.log(grow(out, 60, 1), sum(out), out[0], out[39]); +console.log(grow(out, 10, 5), out.join(",")); diff --git a/tests/corpus/string-plus-element-read-return.ts b/tests/corpus/string-plus-element-read-return.ts new file mode 100644 index 0000000000..ec07fa3d50 --- /dev/null +++ b/tests/corpus/string-plus-element-read-return.ts @@ -0,0 +1,42 @@ +// A function declared to return a string concatenates a string with an +// element of a readonly string array. A missing element reads as undefined, +// which `+` turns into "undefined" next to a string, so the result is always +// a string and its call can be used inline. + +const PAD: readonly string[] = ["00", "01", "02"]; + +function label(i: number): string { + return "x-" + PAD[i]; +} +function date(y: number, m: number, d: number): string { + return y + "-" + PAD[m] + "-" + PAD[d]; +} +function suffix(i: number): string { + return PAD[i] + "!"; +} + +console.log(label(1).length, label(1), label(7)); +console.log(date(2026, 1, 2).length, date(2026, 2, 9)); +console.log(suffix(0).toUpperCase(), suffix(5)); + +// Two possibly missing reads add to NaN when both are missing. +function pair(i: number, j: number): string { + return PAD[i] + PAD[j]; +} +const both = pair(8, 9); +console.log(pair(0, 1), typeof both, String(both)); + +// A missing element held in a binding or returned by a function is undefined, +// so two of them still add to NaN. +function at(i: number): string { + return PAD[i]; +} +function viaBinding(i: number, j: number): string { + const head = PAD[i]; + return head + PAD[j]; +} +function viaCall(i: number, j: number): string { + return at(i) + PAD[j]; +} +console.log(viaBinding(0, 1), String(viaBinding(5, 6)), typeof viaBinding(5, 6)); +console.log(viaCall(0, 2), String(viaCall(5, 6)), viaBinding(0, 9)); diff --git a/tests/corpus/symbol-keyed-recursive-record.ts b/tests/corpus/symbol-keyed-recursive-record.ts new file mode 100644 index 0000000000..c26d7282e5 --- /dev/null +++ b/tests/corpus/symbol-keyed-recursive-record.ts @@ -0,0 +1,35 @@ +// A recursive record keyed by unique symbols, built from literals whose +// child arrays start empty, holds the nodes later pushed into those arrays. + +const kValue: unique symbol = Symbol("value"); +const kChildren: unique symbol = Symbol("children"); + +interface TreeNode { + [kChildren]: TreeNode[]; + [kValue]: number; +} + +function node(value: number): TreeNode { + return { [kValue]: value, [kChildren]: [] }; +} + +const root = node(0); +root[kChildren].push(node(1)); +root[kChildren].push(node(2)); +root[kChildren][0]![kChildren].push(node(3)); +console.log(root[kChildren].length, root[kChildren][0]![kValue], root[kChildren][1]![kValue]); + +function sum(n: TreeNode): number { + let total = n[kValue]; + for (const child of n[kChildren]) total += sum(child); + return total; +} +function depth(n: TreeNode): number { + return 1 + Math.max(0, ...n[kChildren].map(depth)); +} +console.log(sum(root), depth(root)); + +// A literal that starts with children. +const seeded: TreeNode = { [kValue]: 10, [kChildren]: [node(11)] }; +seeded[kChildren].push(node(12)); +console.log(seeded[kChildren].map((c) => c[kValue]).join(","), sum(seeded)); diff --git a/tests/corpus/undeclared-key-optional-chain/main.cjs b/tests/corpus/undeclared-key-optional-chain/main.cjs new file mode 100644 index 0000000000..de7b3351a1 --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/main.cjs @@ -0,0 +1,11 @@ +// A package typed from its JavaScript returns profiles without a key its +// declarations promise ("a;b"). An optional read of that key answers +// undefined for a profile, and for a missing one (the chain reads the +// nullish receiver as undefined), in presence tests and in values. +const { loadProfiles } = require("hostilekit"); +const profiles = loadProfiles(); +const present = profiles[0]; +const missing = profiles[5]; +console.log(present?.["a;b"] === undefined, present?.["a;b"] ?? "none", typeof present?.["a;b"]); +console.log(missing?.["a;b"] === undefined, missing?.["a;b"] ?? "none", typeof missing?.["a;b"]); +console.log(present["a;b"] === undefined, present.id); diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts new file mode 100644 index 0000000000..4bf22dfea0 --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.d.ts @@ -0,0 +1,2 @@ +export type Profile = { id: number; "a;b": string }; +export declare function loadProfiles(): Profile[]; diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js new file mode 100644 index 0000000000..3165012566 --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/index.js @@ -0,0 +1,5 @@ +"use strict"; +function loadProfiles() { + return [{ id: 1 }]; +} +exports.loadProfiles = loadProfiles; diff --git a/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json new file mode 100644 index 0000000000..2fdbc5b41e --- /dev/null +++ b/tests/corpus/undeclared-key-optional-chain/node_modules/hostilekit/package.json @@ -0,0 +1 @@ +{ "name": "hostilekit", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/corpus/union-record-kept-fields-arm.ts b/tests/corpus/union-record-kept-fields-arm.ts new file mode 100644 index 0000000000..69887cf49e --- /dev/null +++ b/tests/corpus/union-record-kept-fields-arm.ts @@ -0,0 +1,56 @@ +// A record flowing into a discriminated union where one record arm holds it +// by dropping fields and another keeps every field but declares an optional +// field the record lacks. The record's discriminant selects the arm that keeps +// its fields. + +type Loc = { file: string; start: number }; + +type Expr = + | { kind: "ref"; type: number; loc: Loc } + | { + kind: "closure"; + fnName: string; + captures: string[]; + adapts?: true; + type: number; + loc: Loc; + } + | { kind: "call"; callee: string; args: Expr[]; type: number; loc: Loc }; + +function closureOf(name: string, loc: Loc): Expr { + const closure = { + kind: "closure" as const, + fnName: name, + captures: [], + type: 1, + loc, + }; + return closure; +} + +function describe(expr: Expr): string { + if (expr.kind === "closure") + return `closure ${expr.fnName}/${expr.captures.length} ${expr.adapts ?? "-"}`; + if (expr.kind === "call") + return `call ${expr.callee}(${expr.args.map(describe).join(", ")})`; + return `ref ${expr.type}`; +} + +const loc: Loc = { file: "a.ts", start: 3 }; +const first = closureOf("pick", loc); +console.log(first.kind, describe(first), JSON.stringify(first)); + +const exprs: Expr[] = [first]; +const captured = { + kind: "closure" as const, + fnName: "inner", + captures: ["x"], + type: 2, + loc, +}; +exprs.push(captured); +exprs.push({ kind: "ref", type: 7, loc }); +console.log( + describe({ kind: "call", callee: "run", args: exprs, type: 0, loc }), +); +console.log(exprs.map((e) => e.kind).join(",")); diff --git a/tests/corpus/union-record-kept-fields-no-discriminant.ts b/tests/corpus/union-record-kept-fields-no-discriminant.ts new file mode 100644 index 0000000000..45370f152b --- /dev/null +++ b/tests/corpus/union-record-kept-fields-no-discriminant.ts @@ -0,0 +1,39 @@ +// A record flowing into a union without a discriminant where one record arm +// holds it by dropping fields and another keeps every field but declares an +// optional field the record lacks. The record converts into the arm that +// keeps its fields, and the optional field stays absent. + +type Loc = { file: string; start: number }; + +type Slot = + | { type: number; loc: Loc } + | { + fnName: string; + captures: string[]; + adapts?: true; + type: number; + loc: Loc; + }; + +const loc: Loc = { file: "a.ts", start: 3 }; + +function slotOf(name: string): Slot { + const slot = { fnName: name, captures: [], type: 1, loc }; + return slot; +} + +const slot = slotOf("stored"); +console.log("fnName" in slot, "adapts" in slot, JSON.stringify(slot)); +if ("fnName" in slot) + console.log(slot.fnName, slot.adapts, slot.captures.length); + +function describe(value: Slot): string { + return "fnName" in value + ? `${value.fnName}/${value.captures.length}` + : `type ${value.type}`; +} + +const inner = { fnName: "inner", captures: ["x"], type: 2, loc }; +const slots: Slot[] = [slot, { type: 7, loc }]; +slots.push(inner); +console.log(slots.map(describe).join(", "), JSON.stringify(slots[2])); diff --git a/tests/corpus/union-tuple-element-read.ts b/tests/corpus/union-tuple-element-read.ts new file mode 100644 index 0000000000..141609d2b6 --- /dev/null +++ b/tests/corpus/union-tuple-element-read.ts @@ -0,0 +1,45 @@ +// Literal-index reads from a union of tuples whose element types differ: +// each tuple's own element is read, and the checker's narrowing of the +// tuple (or of the element) applies as usual. + +type Ok = readonly [true, T]; +type Err = readonly [false, E]; +type Result = Ok | Err; + +function pick(n: number): Result { + return n > 0 ? [true, n] : [false, "neg"]; +} + +for (const n of [1, -1, 5]) { + const r = pick(n); + console.log(r[0], r[1]); + if (r[0]) console.log(r[1] + 1); + else console.log(r[1].length); +} + +function describe(r: Result): string { + const value = r[1]; + return `${r[0] ? "ok" : "err"}:${typeof value}:${value}`; +} +console.log(describe(pick(2)), describe(pick(0))); + +function isOk(r: Result): boolean { + return r[0]; +} +function payload(r: Result): T | E { + return r[1]; +} +console.log(isOk(pick(3)), isOk(pick(-3)), payload(pick(4)), payload(pick(-4))); + +type Triple = readonly [string, number, boolean] | readonly [number, string, boolean]; +const triples: Triple[] = [ + ["a", 1, true], + [2, "b", false], +]; +for (const t of triples) console.log(typeof t[0], typeof t[1], t[2]); + +// A read through a single tuple view of the union. +function flag(r: Result): boolean { + return (r as readonly [boolean, T | E])[0]; +} +console.log(flag(pick(1)), flag(pick(-1))); diff --git a/tests/corpus/var-object-filled-later.js b/tests/corpus/var-object-filled-later.js new file mode 100644 index 0000000000..009d8b54d2 --- /dev/null +++ b/tests/corpus/var-object-filled-later.js @@ -0,0 +1,48 @@ +// A function-scoped `var` initialized with an object literal that later +// assignments fill in: tsc types the binding by the members those +// assignments add, but the object only gains them after its initializer. +"use strict"; +function load(id) { + var p = {}; + console.log(p.count, "count" in p, Object.keys(p).length); + p.id = id; + p.count = id * 2; + p.label = "item-" + id; + return p; +} +const p = load(7); +console.log(p.id + p.count, p.label.toUpperCase(), JSON.stringify(p), Object.keys(p).join(",")); + +function partial(name) { + var entry = { name: name }; + entry.size = name.length; + entry.tags = [name, name + "!"]; + return entry; +} +const e = partial("abc"); +console.log(e.size * 10, e.tags.join("|"), JSON.stringify(e)); + +function nested(depth) { + var node = {}; + node.depth = depth; + node.child = depth > 0 ? nested(depth - 1) : null; + return node; +} +console.log(JSON.stringify(nested(2))); + +function rows(n) { + var out = []; + for (var i = 0; i < n; i++) { + var row = {}; + row.index = i; + row.square = i * i; + out.push(row); + } + return out; +} +console.log(rows(4).map(function (r) { return r.index + ":" + r.square; }).join(" ")); + +var top = {}; +top.ready = true; +top.total = rows(3).reduce(function (s, r) { return s + r.square; }, 0); +console.log(top.ready, top.total, JSON.stringify(top)); diff --git a/tests/coverage-fixtures/localized-failure.ts b/tests/coverage-fixtures/localized-failure.ts new file mode 100644 index 0000000000..04a5b8037e --- /dev/null +++ b/tests/coverage-fixtures/localized-failure.ts @@ -0,0 +1,68 @@ +// The localized-failure report of `scriptc coverage --explain-dynamic`: +// type failures that would become checked dynamic slots, and the uses +// each one rejects today. +interface Tool { + name: string; + hooks: Map string> | AbortSignal | (() => void) | Set; +} +type Progress = (done: number, total: number) => void; +type Callbacks = Progress | AbortSignal | Map string> | Set; + +// A field: rejects the class, so every use of an instance fails. +class Session { + private tools: Tool[] = []; + count(): number { + return this.tools.length; + } + label(): string { + return "s"; + } +} + +// A parameter: rejects the function, so every call fails. +function listAll(dir: string, progress?: Callbacks): number { + return dir.length; +} + +// The dyn class field policy. +class Holder { + state: any = {}; + get(): number { + return 1; + } +} + +// A local: rejects its declaration statement, so its uses fail. +function pick(n: number): boolean { + const callbacks: Callbacks | undefined = n > 1 ? new Set() : undefined; + return callbacks === undefined; +} + +// Refused: a value the compiler cannot produce stays an error. +class Cache { + tokens: WeakRef = new WeakRef({}); + size(): number { + return 0; + } +} + +const s = new Session(); +console.log(s.count(), s.label()); +console.log(listAll("abc")); +console.log(new Holder().get()); +console.log(pick(1), new Cache().size()); + +// A class extending one the dyn class field policy rejects: its rejection +// and the uses of the derived class are that field's cascade. +class Base { + state: any = {}; + get(): number { + return 1; + } +} +class Derived extends Base { + twice(): number { + return 2; + } +} +console.log(new Derived().twice()); diff --git a/tests/diagnostics/promise-all-generic-mixed-union.ts b/tests/diagnostics/promise-all-generic-mixed-union.ts new file mode 100644 index 0000000000..8324f6a232 --- /dev/null +++ b/tests/diagnostics/promise-all-generic-mixed-union.ts @@ -0,0 +1,8 @@ +// Promise.all over a generic array instantiated with a union of a value and +// a promise, with an element read of its result: the call is refused by name. +async function f(xs: V[]): Promise { + const values = await Promise.all(xs); + values[0]; +} +f>(["x"]); +export {}; diff --git a/tests/diagnostics/untyped-map-into-interface.ts b/tests/diagnostics/untyped-map-into-interface.ts new file mode 100644 index 0000000000..ba5e26a907 --- /dev/null +++ b/tests/diagnostics/untyped-map-into-interface.ts @@ -0,0 +1,12 @@ +// An untyped `new Map()` satisfies a record interface only through `any`; +// its flow into the interface-typed result is refused at build time, like a +// typed Map is. +interface Cache { + get(key: string): number | undefined; + set(key: string, value: number): unknown; +} +function make(c?: Cache): Cache { + return c ?? new Map(); +} +console.log(make().get("a")); +export {}; diff --git a/tests/fixtures/npm/cases/types-keyed-construction/main.ts b/tests/fixtures/npm/cases/types-keyed-construction/main.ts new file mode 100644 index 0000000000..b163f3d1c6 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/main.ts @@ -0,0 +1,12 @@ +// A plain-JavaScript package typed by an @types package builds its result +// key by key (`var p = {}; p.id = id; ...`): the object crosses the +// declaration boundary with every field, including the declared fields +// whose keys are not identifiers. +import { loadProfile, type Profile } from "profilekit"; + +const p: Profile = loadProfile(7); +console.log(p["a;b"].toUpperCase(), p["a;b"].length); +console.log(p["my-key"], p["my-key"].length, p["x y"].length); +console.log(p.id, p.count, JSON.stringify(p)); +const keys: string[] = Object.keys(p); +console.log(keys.join("|")); diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts new file mode 100644 index 0000000000..2da797940c --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/index.d.ts @@ -0,0 +1,8 @@ +export interface Profile { + id: number; + "a;b": string; + "my-key": string; + "x y": string; + count: number; +} +export declare function loadProfile(id: number): Profile; diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json new file mode 100644 index 0000000000..a7dd89b118 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/@types/profilekit/package.json @@ -0,0 +1 @@ +{ "name": "@types/profilekit", "version": "1.0.0", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js new file mode 100644 index 0000000000..8d8df93365 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/index.js @@ -0,0 +1,12 @@ +"use strict"; +// Plain JavaScript: the profile is built key by key. +function loadProfile(id) { + var p = {}; + p.id = id; + p["a;b"] = "semi" + id; + p["my-key"] = "dash"; + p["x y"] = "space"; + p.count = id * 2; + return p; +} +exports.loadProfile = loadProfile; diff --git a/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json new file mode 100644 index 0000000000..340a0a6c56 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/node_modules/profilekit/package.json @@ -0,0 +1 @@ +{ "name": "profilekit", "version": "1.0.0", "main": "index.js" } diff --git a/tests/fixtures/npm/cases/types-keyed-construction/package.json b/tests/fixtures/npm/cases/types-keyed-construction/package.json new file mode 100644 index 0000000000..24992c9539 --- /dev/null +++ b/tests/fixtures/npm/cases/types-keyed-construction/package.json @@ -0,0 +1 @@ +{ "name": "types-keyed-construction-case", "private": true, "type": "module" } diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts b/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts new file mode 100644 index 0000000000..d069face61 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/main.ts @@ -0,0 +1,20 @@ +// A package typed from its JavaScript returns profiles without a key its +// declarations promise ("a;b"). A literal-key read of that key answers +// undefined, as Node's does; it never reads another field. +import { loadProfiles } from "hostilekit"; + +const profiles = loadProfiles(); +for (const p of profiles) { + console.log(p.id, p["a;b"] === undefined, typeof p["a;b"], "a;b" in p); + try { + console.log(p["a;b"].toUpperCase()); + } catch (e) { + console.log("upper threw", e instanceof TypeError); + } + try { + console.log(p["a;b"].length); + } catch (e) { + console.log("length threw", e instanceof TypeError); + } + console.log(p.id.toFixed(1), JSON.stringify(p)); +} diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts new file mode 100644 index 0000000000..4bf22dfea0 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.d.ts @@ -0,0 +1,2 @@ +export type Profile = { id: number; "a;b": string }; +export declare function loadProfiles(): Profile[]; diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js new file mode 100644 index 0000000000..3165012566 --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/index.js @@ -0,0 +1,5 @@ +"use strict"; +function loadProfiles() { + return [{ id: 1 }]; +} +exports.loadProfiles = loadProfiles; diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json new file mode 100644 index 0000000000..2fdbc5b41e --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/node_modules/hostilekit/package.json @@ -0,0 +1 @@ +{ "name": "hostilekit", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json b/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json new file mode 100644 index 0000000000..afdb3cf5eb --- /dev/null +++ b/tests/fixtures/npm/cases/types-undeclared-quoted-key/package.json @@ -0,0 +1 @@ +{ "name": "types-undeclared-quoted-key-case", "private": true, "type": "module" } diff --git a/tests/harness/__snapshots__/coverage-explain-localized-failure.txt b/tests/harness/__snapshots__/coverage-explain-localized-failure.txt new file mode 100644 index 0000000000..31a933b263 --- /dev/null +++ b/tests/harness/__snapshots__/coverage-explain-localized-failure.txt @@ -0,0 +1,19 @@ + dynamic tier (tier assignment, report-only; --tier=auto) + 5 functions already operate on dynamic values (4 statements) + Base.constructor tests/coverage-fixtures/localized-failure.ts:57:1: dynObjLit at tests/coverage-fixtures/localized-failure.ts:58:16 + Session.constructor tests/coverage-fixtures/localized-failure.ts:12:1: dynArrLit at tests/coverage-fixtures/localized-failure.ts:13:27 + Session.count tests/coverage-fixtures/localized-failure.ts:14:3: %dyn.class.property at tests/coverage-fixtures/localized-failure.ts:15:12 + Holder.constructor tests/coverage-fixtures/localized-failure.ts:28:1: dynObjLit at tests/coverage-fixtures/localized-failure.ts:29:16 + %dyn.class.property.0 tests/coverage-fixtures/localized-failure.ts:15:12: %dyn.class.readPropertiesOrEmpty at tests/coverage-fixtures/localized-failure.ts:15:12 + + 2 demotion requests (refusals and fences the dynamic tier would replace) + + 0 would-be dynamic slots (localized failure, report-only: type failures a checked dynamic slot would hold) + 2 errors: 0 clear with declaration slots (L1-L2), 0 with member and expression slots too (L3-L4); 1 cascade attributed; 1 type failure stays refused + class Cache tests/coverage-fixtures/localized-failure.ts:42:1 (1 request) + <- SC1090 reflection at tests/coverage-fixtures/localized-failure.ts:53:22: constructing through a class value whose class has no lowering (the class declaration itself was rejected — see its own diagnostic) is not supported yet + -> function turns dynamic + field Cache.tokens tests/coverage-fixtures/localized-failure.ts:43:3 (1 request) + <- SC2020 operation at tests/coverage-fixtures/localized-failure.ts:43:3: 'WeakRef' is part of the standard library types but has no scriptc lowering yet + -> function turns dynamic + function tests/coverage-fixtures/localized-failure.ts:1:1 (2 sites inside) \ No newline at end of file diff --git a/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt b/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt new file mode 100644 index 0000000000..b0622d132f --- /dev/null +++ b/tests/harness/__snapshots__/promise-all-generic-mixed-union.ts.txt @@ -0,0 +1,8 @@ +promise-all-generic-mixed-union.ts:4:36 - error SC2020: 'Promise.all over this argument shape' is part of the standard library types but has no scriptc lowering yet (instantiating 'f' with | string>) + + 3 | async function f(xs: V[]): Promise { + 4 | const values = await Promise.all(xs); + | ^~ + 5 | values[0]; + + hint: an array of promises (Promise[]) is the supported form \ No newline at end of file diff --git a/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt b/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt new file mode 100644 index 0000000000..03b54a1368 --- /dev/null +++ b/tests/harness/__snapshots__/untyped-map-into-interface.ts.txt @@ -0,0 +1,8 @@ +untyped-map-into-interface.ts:9:10 - error SC2002: record shapes must match exactly or width-coerce: expected '{ get: (string) => number | undefined; set: (string, number) => unknown }', got 'Map' + + 8 | function make(c?: Cache): Cache { + 9 | return c ?? new Map(); + | ^~~~~~~~~~~~~~ + 10 | } + + hint: width subtyping compiles as a copy (each expected field copied off the source; extra fields drop) — this pair is outside the copy's rules; build a literal with exactly the expected fields instead \ No newline at end of file diff --git a/tests/harness/bootstrap-seed.test.ts b/tests/harness/bootstrap-seed.test.ts index 76b1f6de5a..2da27baea0 100644 --- a/tests/harness/bootstrap-seed.test.ts +++ b/tests/harness/bootstrap-seed.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { load } from "js-yaml"; import { expect, test } from "vitest"; -import { verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; +import { seedBuildEnvironment, verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; test("bootstrap artifacts require a complete seed from the same revision, lane, and path", () => { const directory = mkdtempSync(join(tmpdir(), "scriptc-seed-contract-")); @@ -36,6 +36,22 @@ test("bootstrap artifacts require a complete seed from the same revision, lane, } }); +test("seeds build without the function layout in both lanes, shared or not", () => { + for (const sanitize of [false, true]) + expect(seedBuildEnvironment(sanitize)).toEqual({ + SCRIPTC_SAN: sanitize ? "1" : "", + SCRIPTC_NATIVE_EMIT_IR: "1", + SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", + SCRIPTC_NATIVE_LAYOUT: "", + }); + // The bootstrap test builds its own seed when CI shares none. + const driver = readFileSync( + new URL("./self-hosting-native-driver.test.ts", import.meta.url), + "utf8", + ); + expect(driver).toContain("...seedBuildEnvironment(sanitize)"); +}); + test("CI restores each seed only into its matching instrumented bootstrap lane", () => { const workflow = load( readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"), diff --git a/tests/harness/corpus-costs/8008-string-index-paths.ts.json b/tests/harness/corpus-costs/8008-string-index-paths.ts.json new file mode 100644 index 0000000000..3a1b75cbc5 --- /dev/null +++ b/tests/harness/corpus-costs/8008-string-index-paths.ts.json @@ -0,0 +1 @@ +{ "plain": 4784, "san": 2792 } diff --git a/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json b/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json new file mode 100644 index 0000000000..981bcbb55b --- /dev/null +++ b/tests/harness/corpus-costs/8014-foreign-element-callbacks.ts.json @@ -0,0 +1 @@ +{ "plain": 5089, "san": 8118 } diff --git a/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json b/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json new file mode 100644 index 0000000000..6b0f0395b6 --- /dev/null +++ b/tests/harness/corpus-costs/8332-worker-shared-counted-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 4011, "san": 2208 } diff --git a/tests/harness/corpus-costs/constant-choice-chains.ts.json b/tests/harness/corpus-costs/constant-choice-chains.ts.json new file mode 100644 index 0000000000..6562c9db30 --- /dev/null +++ b/tests/harness/corpus-costs/constant-choice-chains.ts.json @@ -0,0 +1 @@ +{ "plain": 3921, "san": 5592 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json new file mode 100644 index 0000000000..8c0e66c3c2 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-fresh.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json new file mode 100644 index 0000000000..8c0e66c3c2 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-holders.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json b/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json new file mode 100644 index 0000000000..8c0e66c3c2 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity-workers.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-cast-identity.ts.json b/tests/harness/corpus-costs/dyn-cast-identity.ts.json new file mode 100644 index 0000000000..8c0e66c3c2 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-cast-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/dyn-literal-identity.ts.json b/tests/harness/corpus-costs/dyn-literal-identity.ts.json new file mode 100644 index 0000000000..8c0e66c3c2 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-literal-identity.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } diff --git a/tests/harness/corpus-costs/fs-readdir-recursive.ts.json b/tests/harness/corpus-costs/fs-readdir-recursive.ts.json new file mode 100644 index 0000000000..ad3232154e --- /dev/null +++ b/tests/harness/corpus-costs/fs-readdir-recursive.ts.json @@ -0,0 +1 @@ +{ "plain": 3521, "san": 40657 } diff --git a/tests/harness/corpus-costs/localized-failure-cycles.ts.json b/tests/harness/corpus-costs/localized-failure-cycles.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-cycles.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-declarations.ts.json b/tests/harness/corpus-costs/localized-failure-declarations.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-declarations.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json b/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-forward-capture.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-function-slots.ts.json b/tests/harness/corpus-costs/localized-failure-function-slots.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-function-slots.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-held-values.ts.json b/tests/harness/corpus-costs/localized-failure-held-values.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-held-values.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-locals.ts.json b/tests/harness/corpus-costs/localized-failure-locals.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-locals.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json b/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-pi-shapes.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json b/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json new file mode 100644 index 0000000000..1254aa6e43 --- /dev/null +++ b/tests/harness/corpus-costs/localized-failure-registered-statics.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } diff --git a/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json b/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json new file mode 100644 index 0000000000..1af81d8dfd --- /dev/null +++ b/tests/harness/corpus-costs/never-narrowed-literal-argument.ts.json @@ -0,0 +1 @@ +{ "plain": 2485, "san": 53450 } diff --git a/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json b/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json new file mode 100644 index 0000000000..0358eaf914 --- /dev/null +++ b/tests/harness/corpus-costs/rest-tuple-shorthand-property.ts.json @@ -0,0 +1 @@ +{ "plain": 2199, "san": 6086 } diff --git a/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json b/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json new file mode 100644 index 0000000000..47ead0063c --- /dev/null +++ b/tests/harness/corpus-costs/speculative-loop-view-headers.ts.json @@ -0,0 +1 @@ +{ "plain": 2400 } diff --git a/tests/harness/corpus-costs/string-plus-element-read-return.ts.json b/tests/harness/corpus-costs/string-plus-element-read-return.ts.json new file mode 100644 index 0000000000..17a596f9c7 --- /dev/null +++ b/tests/harness/corpus-costs/string-plus-element-read-return.ts.json @@ -0,0 +1 @@ +{ "plain": 2298, "san": 3533 } diff --git a/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json b/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json new file mode 100644 index 0000000000..917f53744a --- /dev/null +++ b/tests/harness/corpus-costs/symbol-keyed-recursive-record.ts.json @@ -0,0 +1 @@ +{ "plain": 2702, "san": 43257 } diff --git a/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json b/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json new file mode 100644 index 0000000000..8cfabb847b --- /dev/null +++ b/tests/harness/corpus-costs/undeclared-key-optional-chain/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 3000 } diff --git a/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json b/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json new file mode 100644 index 0000000000..3c750a2dc5 --- /dev/null +++ b/tests/harness/corpus-costs/union-record-kept-fields-arm.ts.json @@ -0,0 +1 @@ +{ "plain": 4793, "san": 7135 } diff --git a/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json b/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json new file mode 100644 index 0000000000..5e891c40e5 --- /dev/null +++ b/tests/harness/corpus-costs/union-record-kept-fields-no-discriminant.ts.json @@ -0,0 +1 @@ +{ "plain": 4466, "san": 5253 } diff --git a/tests/harness/corpus-costs/union-tuple-element-read.ts.json b/tests/harness/corpus-costs/union-tuple-element-read.ts.json new file mode 100644 index 0000000000..fcdd8c1759 --- /dev/null +++ b/tests/harness/corpus-costs/union-tuple-element-read.ts.json @@ -0,0 +1 @@ +{ "plain": 2301, "san": 3138 } diff --git a/tests/harness/corpus-costs/var-object-filled-later.js.json b/tests/harness/corpus-costs/var-object-filled-later.js.json new file mode 100644 index 0000000000..1e74b61e72 --- /dev/null +++ b/tests/harness/corpus-costs/var-object-filled-later.js.json @@ -0,0 +1 @@ +{ "plain": 3400 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index fde16a901b..f8fa0da038 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -428,6 +428,63 @@ test("--explain-dynamic attributes each refusal to the unit the dynamic tier wou ); }); +test("--explain-dynamic lists the type failures localized failure would turn into dynamic slots", async () => { + const { coverage } = analyze(fixture("localized-failure.ts"), { explainDynamic: true }); + const report = coverage.tiers!.localizedFailure; + // Locals, fields and parameters degrade (stages L1 and L2), and the + // 'any' field compiles; only the refused WeakRef field and its + // construction remain. + expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name])).toEqual([ + ["field", "tools"], + ["param", "progress"], + ["local", "callbacks"], + ]); + expect(report.errors).toBe(2); + expect(report.typeFailures).toEqual({ degradable: 0, refused: 1 }); + expect(report.slots).toEqual([]); + expect(report.cascades).toBe(1); + expect(report.remainingByCode).toEqual({ SC1090: 1, SC2020: 1 }); + const relative = JSON.parse( + JSON.stringify(coverage.tiers).replaceAll(repoRoot + "/", ""), + ) as typeof coverage.tiers; + await expect(renderExplainDynamic(relative!)).toMatchFileSnapshot( + "__snapshots__/coverage-explain-localized-failure.txt", + ); +}); + +test("a local or module global whose declared type does not compile becomes a dynamic slot", async () => { + const program = join(repoRoot, "tests/corpus/localized-failure-locals.ts"); + const { coverage, sourceTexts } = analyze(program); + expect(coverage.diagnostics).toEqual([]); + expect(coverage.dynamicSlots?.map((s) => [s.slot, s.name, s.declaredType, s.code])).toEqual([ + ["global", "current", "Callbacks | undefined", "SC2009"], + ["global", "all", "(number | Callbacks)[]", "SC2009"], + ["local", "local", "Callbacks | undefined", "SC2009"], + ]); + expect(coverage.performance?.filter((d) => d.code === "SC6104")).toHaveLength(3); + expect(coveragePasses(coverage, "blockers")).toBe(true); + expect(coveragePasses(coverage, "dynamic")).toBe(true); + expect(coveragePasses(coverage, "dynamic-slots")).toBe(false); + const envelope = coverageEnvelope(coverage, { compilerVersion: "test", sourceTexts }); + expect(envelope.dynamicSlots?.map((s) => [s.kind, s.name, s.line])).toEqual([ + ["global", "current", 10], + ["global", "all", 36], + ["local", "local", 14], + ]); + expect(envelope.diagnostics.filter((d) => d.code === "SC6104").map((d) => d.severity)).toEqual([ + "warning", + "warning", + "warning", + ]); + const text = renderCoverage(coverage, { sourceTexts, root: repoRoot }); + expect(text).toContain("dynamic slots 3"); + expect(text).toContain("module global 'current' (declared 'Callbacks | undefined')"); + // --tier=static-only keeps every refusal. + const strict = analyze(program, { tier: "static-only" }).coverage; + expect(strict.dynamicSlots).toBeUndefined(); + expect(strict.diagnostics.map((d) => d.code)).toContain("SC2009"); +}); + test("CLI: --tier and --explain-dynamic", () => { const scriptcCli = join(repoRoot, "packages/cli/src/main.ts"); const outDir = mkdtempSync(join(tmpdir(), "scriptc-tier-")); @@ -530,13 +587,18 @@ test("runtime-optional refined unions report blockers without crashing", () => { }); test("a rejected class reached only through a receiver probe reports its own diagnostic once", () => { - const { coverage } = analyze(join(repoRoot, "tests/diagnostics/deferred-class-receiver/main.ts")); + const program = join(repoRoot, "tests/diagnostics/deferred-class-receiver/main.ts"); + // The class's refusal, as the diagnostics corpus pins it (--tier=static-only). + const { coverage } = analyze(program, { tier: "static-only" }); const messages = coverage.diagnostics.map((d) => d.message); expect( messages.filter((m) => m.startsWith("'MapIterator'-typed class fields")), ).toHaveLength(1); expect(messages.some((m) => m.startsWith("constructing through a class value"))).toBe(true); expect(coverage.unreached?.diagnostics ?? []).toEqual([]); + // Localized failure retries the class where the probe's statement commits + // the flush, with the field policy lifted, and it compiles. + expect(analyze(program).coverage.diagnostics).toEqual([]); }); test("settled generic rest-order fences count each source statement once", () => { diff --git a/tests/harness/diagnostics.test.ts b/tests/harness/diagnostics.test.ts index 77fd5937da..895ed9647e 100644 --- a/tests/harness/diagnostics.test.ts +++ b/tests/harness/diagnostics.test.ts @@ -39,7 +39,14 @@ describe(`diagnostics corpus (${files.length} programs)`, () => { if (name.startsWith("checker-panic-")) process.env["SCRIPTC_CHECKER"] = "tsgo"; let result: Awaited>; try { - result = await compile(file, { outPath: join(outDir, "never"), outDir }); + // --tier=static-only: these programs pin today's refusals, which + // localized failure otherwise turns into checked dynamic slots + // (the snapshots double as its proof that static-only keeps them). + result = await compile(file, { + outPath: join(outDir, "never"), + outDir, + tier: "static-only", + }); } finally { if (checker === undefined) delete process.env["SCRIPTC_CHECKER"]; else process.env["SCRIPTC_CHECKER"] = checker; diff --git a/tests/harness/dyncheck.test.ts b/tests/harness/dyncheck.test.ts index dfd037570c..97def0cc69 100644 --- a/tests/harness/dyncheck.test.ts +++ b/tests/harness/dyncheck.test.ts @@ -169,6 +169,33 @@ describe(`dynamic-boundary checks (scriptc-only${sanitize ? ", sanitized" : ""}) ).toBe(""); }); + test("a boxed native value cast to another layout is copied (decision D3)", async () => { + // Node's `as` is a no-op, so these writes would show through every + // holder. scriptc copies a boxed native value into a cast's different + // layout for now (the object model's decision D3: copy, throw, or + // share storage); the copy keeps one identity per boxed value. + const r = await compileAndRun( + "native-layout-casts", + `type Point = { x: number; y: number }; +const dict: Record = { x: 1, y: 2 }; +const v: unknown = dict; +const p = v as Point; +p.x = 5; +console.log(dict.x, p.x, (v as Point) === p); +const nums: number[] = [1, 2]; +const u: unknown = nums; +console.log((u as number[]) === nums); +const wide = u as (number | string)[]; +wide.push("s"); +console.log(nums.length, wide.length); +console.log(JSON.stringify({ ...(v as Record) })); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(["1 5 true", "true", "2 3", '{"x":1,"y":2}', ""].join("\n")); + expect(r.stderr).toBe(""); + }); + test("wrong-typed field throws with the path", async () => { const r = await compileAndRun( "wrong-type", diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 1cc920bd2e..8cfd16eea3 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -1,4 +1,4 @@ -import { verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; +import { seedBuildEnvironment, verifyBootstrapSeed } from "../../scripts/bootstrap-seed.mjs"; import { execFile, spawnSync } from "node:child_process"; import { chmodSync, @@ -89,12 +89,7 @@ test( // seed avoids optimizing the huge compiler module before ASan can // exercise its complete traversal, serialization and LLVM emission. ...options, - env: { - ...process.env, - SCRIPTC_SAN: sanitize ? "1" : "", - SCRIPTC_NATIVE_EMIT_IR: "1", - SCRIPTC_NATIVE_OPTIMIZATION: sanitize ? "dev" : "release", - }, + env: { ...process.env, ...seedBuildEnvironment(sanitize) }, }, ), );